fix video state sync, fixes #250.

.docker/base/Dockerfile

@@ -79,7 +79,9 @@ RUN set -eux; \
         # Japanese fonts
         fonts-takao-mincho \
         # Chinese fonts
-        fonts-wqy-zenhei; \
+        fonts-wqy-zenhei xfonts-intl-chinese xfonts-wqy \
+        # Korean fonts
+        fonts-wqy-microhei; \
     #
     # create a non-root user
     groupadd --gid $USER_GID $USERNAME; \

.docker/base/Dockerfile.arm

@@ -85,7 +85,9 @@ RUN set -eux; \
         # Japanese fonts
         fonts-takao-mincho \
         # Chinese fonts
-        fonts-wqy-zenhei; \
+        fonts-wqy-zenhei xfonts-intl-chinese xfonts-wqy \
+        # Korean fonts
+        fonts-wqy-microhei; \
     #
     # create a non-root user
     groupadd --gid $USER_GID $USERNAME; \

.docker/base/Dockerfile.intel

@@ -88,7 +88,9 @@ RUN set -eux; \
         # Japanese fonts
         fonts-takao-mincho \
         # Chinese fonts
-        fonts-wqy-zenhei; \
+        fonts-wqy-zenhei xfonts-intl-chinese xfonts-wqy \
+        # Korean fonts
+        fonts-wqy-microhei; \
     #
     # create a non-root user
     groupadd --gid $USER_GID $USERNAME; \

.docker/base/Dockerfile.nvidia

@@ -97,7 +97,9 @@ RUN set -eux; \
         # Japanese fonts
         fonts-takao-mincho \
         # Chinese fonts
-        fonts-wqy-zenhei; \
+        fonts-wqy-zenhei xfonts-intl-chinese xfonts-wqy \
+        # Korean fonts
+        fonts-wqy-microhei; \
     #
     # create a non-root user
     groupadd --gid $USER_GID $USERNAME; \

.docker/chromium/Dockerfile.nvidia

@@ -4,9 +4,13 @@ FROM $BASE_IMAGE
 #
 # install neko chromium
 RUN set -eux; \
-    echo "deb http://ftp.de.debian.org/debian bookworm main" >> /etc/apt/sources.list; \
     apt-get update; \
-    apt-get install -y --no-install-recommends unzip chromium chromium-common chromium-sandbox openbox; \
+    apt-get install -y --no-install-recommends software-properties-common; \
+    # chromium-browser from default repo needs snap to be installed
+    # and nvidia base is ubuntu not debian
+    add-apt-repository ppa:system76/pop; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends unzip chromium openbox; \
     #
     # install widevine module
     CHROMIUM_DIR="/usr/lib/chromium"; \

.docker/rebuild-server

@@ -17,5 +17,9 @@ fi
 
 docker run --rm -it \
   -v "${PWD}/../server:/src" \
-  --entrypoint="go" \
-  neko_dev_server build -o "bin/neko" "cmd/neko/main.go"
+  -e GIT_COMMIT=`git rev-parse --short HEAD` \
+  -e GIT_BRANCH=`git rev-parse --symbolic-full-name --abbrev-ref HEAD` \
+  -e GIT_DIRTY=`git diff-index --quiet HEAD -- || echo "✗-"` \
+  --entrypoint="bash" \
+  --workdir="/src" \
+  neko_dev_server ./build

.docker/serve-client

@@ -16,7 +16,7 @@ if [ ! -d "${PWD}/../client/node_modules" ] || [ "$1" == "-i" ]; then
     -v "${PWD}/../client:/app" \
     --workdir="/app" \
     --entrypoint="npm" \
-    node:14-buster-slim install
+    node:18-bullseye-slim install
 fi
 
 docker run --rm -it \
@@ -25,5 +25,5 @@ docker run --rm -it \
   -e "VUE_APP_SERVER_PORT=${SERVER_PORT}" \
   --workdir="/app" \
   --entrypoint="npm" \
-  node:14-buster-slim run serve
+  node:18-bullseye-slim run serve
   

.docker/start-server

@@ -17,9 +17,22 @@ if [ ! -f "${BINARY_PATH}" ] || [ "$1" == "-r" ]; then
   ./rebuild-server
 fi
 
+# if image starts with nvidia- add --gpus all
+if [[ "${SERVER_TAG}" == "nvidia-"* ]]; then
+  GPU_FLAG="--gpus all"
+  echo "Nvidia GPU acceleration enabled"
+fi
+
+# if image starts with intel- add --device /dev/dri
+if [[ "${SERVER_TAG}" == "intel-"* ]]; then
+  GPU_FLAG="--device /dev/dri"
+  echo "Intel GPU acceleration enabled"
+fi
+
 # use --gpus all to enable GPU acceleration
 docker run --rm -it \
   --name "neko_dev" \
+  $GPU_FLAG \
   -p "${SERVER_PORT}:8080" \
   -p "${SERVER_EPR}:${SERVER_EPR}/udp" \
   -e "NEKO_SCREEN=1920x1080@60" \

client/src/app.vue

@@ -1,5 +1,5 @@
 <template>
-  <div id="neko" :class="[side ? 'expanded' : '']">
+  <div id="neko" :class="[!hideControls && side ? 'expanded' : '']">
     <template v-if="!$client.supported">
       <neko-unsupported />
     </template>

client/src/components/video.vue

@@ -22,10 +22,10 @@
           @mouseenter.stop.prevent="onMouseEnter"
           @mouseleave.stop.prevent="onMouseLeave"
         />
-        <div v-if="!playing && playable" class="player-overlay" @click.stop.prevent="toggle">
+        <div v-if="!playing && playable" class="player-overlay" @click.stop.prevent="playAndUnmute">
           <i class="fas fa-play-circle" />
         </div>
-        <div v-if="mutedOverlay && muted" class="player-overlay" @click.stop.prevent="unmute">
+        <div v-else-if="mutedOverlay && muted" class="player-overlay" @click.stop.prevent="unmute">
           <i class="fas fa-volume-up" />
         </div>
         <div ref="aspect" class="player-aspect" />
@@ -384,9 +384,16 @@
     }
 
     @Watch('playing')
-    onPlayingChanged(playing: boolean) {
+    async onPlayingChanged(playing: boolean) {
       if (this._video && this._video.paused && playing) {
-        this.play()
+        // if autoplay is disabled, play() will throw an error
+        // and we need to properly save the state otherwise we
+        // would be thinking we're playing when we're not
+        try {
+          await this._video.play()
+        } catch (err: any) {
+          this.$accessor.video.pause()
+        }
       }
 
       if (this._video && !this._video.paused && !playing) {
@@ -560,6 +567,11 @@
       }
     }
 
+    playAndUnmute() {
+      this.$accessor.video.play()
+      this.$accessor.video.setMuted(false)
+    }
+
     unmute() {
       this.$accessor.video.setMuted(false)
     }

client/src/neko/base.ts

@@ -203,11 +203,12 @@ export abstract class BaseClient extends EventEmitter<BaseEvents> {
       return
     }
 
-    this._peer = new RTCPeerConnection()
     if (lite !== true) {
       this._peer = new RTCPeerConnection({
         iceServers: servers,
       })
+    } else {
+      this._peer = new RTCPeerConnection()
     }
 
     this._peer.onconnectionstatechange = () => {
@@ -251,11 +252,28 @@ export abstract class BaseClient extends EventEmitter<BaseEvents> {
 
     this._peer.ontrack = this.onTrack.bind(this)
 
+    this._peer.onicecandidate = (event: RTCPeerConnectionIceEvent) => {
+      if (!event.candidate) {
+        this.emit('debug', `sent all local ICE candidates`)
+        return
+      }
+
+      const init = event.candidate.toJSON()
+      this.emit('debug', `sending local ICE candidate`, init)
+
+      this._ws!.send(
+        JSON.stringify({
+          event: EVENT.SIGNAL.CANDIDATE,
+          data: JSON.stringify(init),
+        }),
+      )
+    }
+
     this._peer.onnegotiationneeded = async () => {
       this.emit('warn', `negotiation is needed`)
 
       const d = await this._peer!.createOffer()
-      this._peer!.setLocalDescription(d)
+      await this._peer!.setLocalDescription(d)
 
       this._ws!.send(
         JSON.stringify({
@@ -277,10 +295,10 @@ export abstract class BaseClient extends EventEmitter<BaseEvents> {
       return
     }
 
-    this._peer.setRemoteDescription({ type: 'offer', sdp })
+    await this._peer.setRemoteDescription({ type: 'offer', sdp })
 
     for (const candidate of this._candidates) {
-      this._peer.addIceCandidate(candidate)
+      await this._peer.addIceCandidate(candidate)
     }
     this._candidates = []
 
@@ -310,7 +328,7 @@ export abstract class BaseClient extends EventEmitter<BaseEvents> {
       return
     }
 
-    this._peer.setRemoteDescription({ type: 'answer', sdp })
+    await this._peer.setRemoteDescription({ type: 'answer', sdp })
   }
 
   private async onMessage(e: MessageEvent) {

docs/changelog.md

@@ -12,10 +12,13 @@
 - Fixed stereo problem in chromium-based browsers, where it was only as mono by adding `stereo=1` to opus SDP to clients answer.
 - Fixed keysym mapping for unknown keycodes, which was causing some key combinations to not work on some keyboards.
 - Fixed a bug where `max_fps=0` would lead to an invalid pipeline.
+- Fixed client side webrtc ICE gathering, so that neko can be used without exposed ports, only with STUN and TURN servers.
+- Fixed play state synchronization, when autoplay is disabled.
 
 ### Misc
 - Updated to go 1.19 and Node 18, removed go-events as dependency (by @mbattista).
 - Added adaptive framerate which now streams in the framerate you selected from the dropdown.
+- Improved chinese and korean characters support.
 
 ## [n.eko v2.7](https://github.com/m1k1o/neko/releases/tag/v2.7)
 

docs/getting-started/README.md

@@ -162,6 +162,57 @@ services:
   "RestoreOnStartup": 1,
 ```
 
+### Nvidia GPU acceleration
+
+You need to have nvidia-docker installed, start the container with `--gpus all` flag and use images built for nvidia (see above).
+
+```bash
+docker run -d --gpus all \
+  -p 8080:8080 \
+  -p 56000-56100:56000-56100/udp \
+  -e NEKO_SCREEN=1920x1080@30 \
+  -e NEKO_PASSWORD=neko \
+  -e NEKO_PASSWORD_ADMIN=admin \
+  -e NEKO_EPR=56000-56100 \
+  -e NEKO_NAT1TO1=192.168.1.10 \
+  -e NEKO_ICELITE=1 \
+  --shm-size=2gb \
+  --cap-add=SYS_ADMIN \
+  --name neko \
+  ghcr.io/m1k1o/neko/nvidia-google-chrome:latest
+```
+
+If you want to use docker-compose, you can use this example:
+
+```yaml
+version: "3.4"
+services:
+  neko:
+    image: "ghcr.io/m1k1o/neko/nvidia-google-chrome:latest"
+    restart: "unless-stopped"
+    shm_size: "2gb"
+    ports:
+      - "8080:8080"
+      - "56000-56100:56000-56100/udp"
+    cap_add:
+      - SYS_ADMIN
+    environment:
+      NEKO_SCREEN: '1920x1080@30'
+      NEKO_PASSWORD: neko
+      NEKO_PASSWORD_ADMIN: admin
+      NEKO_EPR: 56000-56100
+      NEKO_NAT1TO1: 192.168.1.10
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: 1
+              capabilities: [gpu]
+```
+
+Note, currently only browser GPU acceleration is supported, not encoding.
+
 ### Want to use VPN for your n.eko browsing?
 - Check this out: https://github.com/m1k1o/neko-vpn
 

server/build

@@ -3,8 +3,22 @@
 set -ex
 
 BUILD_TIME=`date -u +'%Y-%m-%dT%H:%M:%SZ'`
+
+#
+# set git build variables if git exists
+if git status > /dev/null 2>&1 && [ -z $GIT_COMMIT ] && [ -z $GIT_BRANCH ] && [ -z $GIT_DIRTY ];
+then
     GIT_COMMIT=`git rev-parse --short HEAD`
     GIT_BRANCH=`git rev-parse --symbolic-full-name --abbrev-ref HEAD`
     GIT_DIRTY=`git diff-index --quiet HEAD -- || echo "✗-"`
+fi
 
-go build -o bin/neko -ldflags "-s -X 'm1k1o/neko.buildDate=${BUILD_TIME}' -X 'm1k1o/neko.gitCommit=${GIT_DIRTY}${GIT_COMMIT}' -X 'm1k1o/neko.gitBranch=${GIT_BRANCH}'" -i cmd/neko/main.go
+go build \
+    -o bin/neko \
+    -ldflags "
+        -s -w
+        -X 'm1k1o/neko.buildDate=${BUILD_TIME}'
+        -X 'm1k1o/neko.gitCommit=${GIT_DIRTY}${GIT_COMMIT}'
+        -X 'm1k1o/neko.gitBranch=${GIT_BRANCH}'
+    " \
+    cmd/neko/main.go;

server/internal/session/session.go

@@ -131,6 +131,17 @@ func (session *Session) SignalLocalAnswer(sdp string) error {
 	})
 }
 
+func (session *Session) SignalLocalCandidate(data string) error {
+	if session.socket == nil {
+		return nil
+	}
+	session.logger.Info().Msg("signal update - LocalCandidate")
+	return session.socket.Send(&message.SignalCandidate{
+		Event: event.SIGNAL_CANDIDATE,
+		Data:  data,
+	})
+}
+
 func (session *Session) SignalRemoteOffer(sdp string) error {
 	if session.peer == nil {
 		return nil
@@ -154,14 +165,12 @@ func (session *Session) SignalRemoteAnswer(sdp string) error {
 	return session.peer.SetAnswer(sdp)
 }
 
-func (session *Session) SignalCandidate(data string) error {
+func (session *Session) SignalRemoteCandidate(data string) error {
 	if session.socket == nil {
 		return nil
 	}
-	return session.socket.Send(&message.SignalCandidate{
-		Event: event.SIGNAL_CANDIDATE,
-		Data:  data,
-	})
+	session.logger.Info().Msg("signal update - RemoteCandidate")
+	return session.peer.SetCandidate(data)
 }
 
 func (session *Session) destroy() error {

server/internal/types/session.go

@@ -40,9 +40,10 @@ type Session interface {
 	Send(v interface{}) error
 	SignalLocalOffer(sdp string) error
 	SignalLocalAnswer(sdp string) error
+	SignalLocalCandidate(data string) error
 	SignalRemoteOffer(sdp string) error
 	SignalRemoteAnswer(sdp string) error
-	SignalCandidate(data string) error
+	SignalRemoteCandidate(data string) error
 }
 
 type SessionManager interface {

server/internal/types/webrtc.go

@@ -21,6 +21,7 @@ type Peer interface {
 	CreateAnswer() (string, error)
 	SetOffer(sdp string) error
 	SetAnswer(sdp string) error
+	SetCandidate(candidateString string) error
 	WriteData(v interface{}) error
 	Destroy() error
 }

server/internal/webrtc/peer.go

@@ -1,6 +1,7 @@
 package webrtc
 
 import (
+	"encoding/json"
 	"sync"
 
 	"github.com/pion/webrtc/v3"
@@ -49,6 +50,16 @@ func (peer *Peer) SetAnswer(sdp string) error {
 	return peer.connection.SetRemoteDescription(webrtc.SessionDescription{SDP: sdp, Type: webrtc.SDPTypeAnswer})
 }
 
+func (peer *Peer) SetCandidate(candidateString string) error {
+	var candidate webrtc.ICECandidateInit
+	err := json.Unmarshal([]byte(candidateString), &candidate)
+	if err != nil {
+		return err
+	}
+
+	return peer.connection.AddICECandidate(candidate)
+}
+
 func (peer *Peer) WriteData(v interface{}) error {
 	peer.mu.Lock()
 	defer peer.mu.Unlock()

server/internal/webrtc/webrtc.go

@@ -123,7 +123,6 @@ func (manager *WebRTCManager) initAPI() error {
 		LoggerFactory: logger,
 	}
 
-	_ = settings.SetEphemeralUDPPortRange(manager.config.EphemeralMin, manager.config.EphemeralMax)
 	settings.SetNAT1To1IPs(manager.config.NAT1To1IPs, webrtc.ICECandidateTypeHost)
 	settings.SetICETimeouts(6*time.Second, 6*time.Second, 3*time.Second)
 	settings.SetSRTPReplayProtectionWindow(512)
@@ -168,12 +167,15 @@ func (manager *WebRTCManager) initAPI() error {
 
 		networkType = append(networkType, webrtc.NetworkTypeUDP4)
 		manager.logger.Info().Int("port", manager.config.UDPMUX).Msg("using UDP MUX")
+	} else if manager.config.EphemeralMax != 0 {
+		_ = settings.SetEphemeralUDPPortRange(manager.config.EphemeralMin, manager.config.EphemeralMax)
+		networkType = append(networkType,
+			webrtc.NetworkTypeUDP4,
+			webrtc.NetworkTypeUDP6,
+		)
 	}
 
-	// Enable support for TCP and UDP ICE candidates
-	if len(networkType) > 0 {
 	settings.SetNetworkTypes(networkType)
-	}
 
 	// Create MediaEngine with selected codecs
 	engine := webrtc.MediaEngine{}
@@ -299,7 +301,7 @@ func (manager *WebRTCManager) CreatePeer(id string, session types.Session) (type
 			return
 		}
 
-		if err := session.SignalCandidate(string(candidateString)); err != nil {
+		if err := session.SignalLocalCandidate(string(candidateString)); err != nil {
 			manager.logger.Warn().Err(err).Msg("sending SignalCandidate failed")
 			return
 		}

server/internal/websocket/handler/handler.go

@@ -87,6 +87,12 @@ func (h *MessageHandler) Message(id string, raw []byte) error {
 			utils.Unmarshal(payload, raw, func() error {
 				return h.signalRemoteAnswer(id, session, payload)
 			}), "%s failed", header.Event)
+	case event.SIGNAL_CANDIDATE:
+		payload := &message.SignalCandidate{}
+		return errors.Wrapf(
+			utils.Unmarshal(payload, raw, func() error {
+				return h.signalRemoteCandidate(id, session, payload)
+			}), "%s failed", header.Event)
 
 	// Control Events
 	case event.CONTROL_RELEASE:

server/internal/websocket/handler/signal.go

@@ -45,3 +45,7 @@ func (h *MessageHandler) signalRemoteAnswer(id string, session types.Session, pa
 
 	return nil
 }
+
+func (h *MessageHandler) signalRemoteCandidate(id string, session types.Session, payload *message.SignalCandidate) error {
+	return session.SignalRemoteCandidate(payload.Data)
+}