live change resolution (WIP)

This commit is contained in:
Craig 2020-02-11 05:15:59 +00:00
parent 3d1341cfe1
commit 9e995233af
27 changed files with 747 additions and 127 deletions

View File

@ -10,9 +10,9 @@ RUN set -eux; apt-get update; apt-get install -y --no-install-recommends \
libpng-dev libpq-dev libreadline-dev libsqlite3-dev libssl-dev libtool libwebp-dev libxml2-dev libxslt-dev libyaml-dev \
make patch unzip xz-utils zlib1g-dev pkg-config \
build-essential perl python autopoint bison flex \
gettext openssl libopus-dev libvpx-dev libpulse-dev libx11-dev libxv-dev libxt-dev \
gettext openssl libopus-dev libvpx-dev libpulse-dev libx11-dev libxv-dev libxt-dev libxrandr-dev \
libxfixes-dev apt-utils x11vnc libxtst-dev dialog \
pulseaudio openbox chromium firefox-esr dbus-x11 xvfb xclip supervisor; \
pulseaudio openbox chromium firefox-esr dbus-x11 xserver-xorg-video-dummy supervisor; \
if ! command -v gpg > /dev/null; then \
apt-get install -y --no-install-recommends gnupg dirmngr; \
fi

View File

@ -1,7 +1,7 @@
# This xorg configuration file is meant to be used by xpra
# to start a dummy X11 server.
# For details, please see:
# https://xpra.org/Xdummy.html
# https://xpra.org/trac/wiki/Xdummy
Section "ServerFlags"
Option "DontVTSwitch" "true"
@ -41,8 +41,31 @@ Section "Monitor"
#NOTE: the highest modes will not work without increasing the VideoRam
# for the dummy video card.
# https://arachnoid.com/modelines/
# 1280x720 @ 30.00 Hz (GTF) hsync: 21.99 kHz; pclk: 33.78 MHz
Modeline "1280x720_30.00" 33.78 1280 1288 1408 1536 720 721 724 733 -HSync +Vsync
# 1280x720 @ 60.00 Hz (GTF) hsync: 44.76 kHz; pclk: 74.48 MHz
Modeline "1280x720_60.00" 74.48 1280 1336 1472 1664 720 721 724 746 -HSync +Vsync
# 1152x648 @ 60.00 Hz (GTF) hsync: 40.26 kHz; pclk: 59.91 MHz
Modeline "1152x648_60.00" 59.91 1152 1200 1320 1488 648 649 652 671 -HSync +Vsync
# 1024x576 @ 60.00 Hz (GTF) hsync: 35.82 kHz; pclk: 47.00 MHz
Modeline "1024x576_60.00" 47.00 1024 1064 1168 1312 576 577 580 597 -HSync +Vsync
# 960x720 @ 60.00 Hz (GTF) hsync: 44.76 kHz; pclk: 55.86 MHz
Modeline "960x720_60.00" 55.86 960 1008 1104 1248 720 721 724 746 -HSync +Vsync
# 800x600 @ 60.00 Hz (GTF) hsync: 37.32 kHz; pclk: 38.22 MHz
Modeline "800x600_60.00" 38.22 800 832 912 1024 600 601 604 622 -HSync +Vsync
# 1920x1080 @ 30.00 Hz (GTF) hsync: 32.97 kHz; pclk: 80.18 MHz
Modeline "1920x1080_30.00" 80.18 1920 1984 2176 2432 1080 1081 1084 1099 -HSync +Vsync
# 1152x648 @ 30.00 Hz (GTF) hsync: 19.80 kHz; pclk: 26.93 MHz
Modeline "1152x648_30.00" 26.93 1152 1144 1256 1360 648 649 652 660 -HSync +Vsync
# 1024x576 @ 30.00 Hz (GTF) hsync: 17.61 kHz; pclk: 20.85 MHz
Modeline "1024x576_30.00" 20.85 1024 1008 1104 1184 576 577 580 587 -HSync +Vsync
# 960x720 @ 30.00 Hz (GTF) hsync: 21.99 kHz; pclk: 25.33 MHz
Modeline "960x720_30.00" 25.33 960 960 1056 1152 720 721 724 733 -HSync +Vsync
# 800x600 @ 30.00 Hz (GTF) hsync: 18.33 kHz; pclk: 17.01 MHz
Modeline "800x600_30.00" 17.01 800 792 864 928 600 601 604 611 -HSync +Vsync
EndSection
Section "Screen"
@ -53,8 +76,7 @@ Section "Screen"
SubSectionSub "Display"
Viewport 0 0
Depth 24
Modes "1280x720_30.00"
Virtual 1280 720
Modes "1280x720_30.00" "1920x1080_60.00" "1280x720_60.00" "1152x648_60.00" "1024x576_60.00" "960x720_60.00" "800x600_60.00" "1920x1080_30.00" "1152x648_30.00" "1024x576_30.00" "960x720_30.00" "800x600_30.00"
EndSubSection
EndSection

View File

@ -29,7 +29,7 @@ RUN set -eux; apt-get update; \
# install neko dependencies
RUN set -eux; apt-get update; \
apt-get install -y --no-install-recommends wget ca-certificates pulseaudio openbox dbus-x11 xserver-xorg-video-dummy supervisor; \
apt-get install -y --no-install-recommends libxv1 libopus0 libvpx4; \
apt-get install -y --no-install-recommends libxrandr2 libxv1 libopus0 libvpx4; \
#
# create a non-root user
groupadd --gid $USER_GID $USERNAME; \

View File

@ -0,0 +1,121 @@
<template>
<vue-context class="context" ref="context">
<template v-for="(conf, i) in configurations">
<li v-for="(rate, j) in conf.rates" :key="`${i}-${j}`" @click="screenSet(conf.width, conf.height, rate)">
<i class="fas fa-desktop"></i>
<span>{{ conf.width }}x{{ conf.height }}</span>
<small>{{ rate }}</small>
</li>
</template>
</vue-context>
</template>
<style lang="scss" scoped>
.context {
background-color: $background-floating;
background-clip: padding-box;
border-radius: 0.25rem;
display: block;
margin: 0;
padding: 5px;
min-width: 150px;
z-index: 1500;
position: fixed;
list-style: none;
box-sizing: border-box;
max-height: calc(100% - 50px);
overflow-y: auto;
color: $interactive-normal;
user-select: none;
box-shadow: $elevation-high;
scrollbar-width: thin;
scrollbar-color: $background-secondary transparent;
&::-webkit-scrollbar {
width: 8px;
}
&::-webkit-scrollbar-track {
background-color: transparent;
}
&::-webkit-scrollbar-thumb {
background-color: $background-secondary;
border: 2px solid $background-floating;
border-radius: 4px;
}
&::-webkit-scrollbar-thumb:hover {
background-color: $background-floating;
}
> li {
margin: 0;
position: relative;
align-content: center;
display: flex;
flex-direction: row;
padding: 8px 5px;
cursor: pointer;
i {
margin-right: 10px;
}
span {
flex-grow: 1;
}
small {
font-size: 0.7em;
justify-self: flex-end;
align-self: flex-end;
}
&:hover,
&:focus {
text-decoration: none;
background-color: $background-modifier-hover;
color: $interactive-hover;
}
&:focus {
outline: 0;
}
}
&:focus {
outline: 0;
}
}
</style>
<script lang="ts">
import { Component, Ref, Watch, Vue } from 'vue-property-decorator'
import { Member } from '~/neko/types'
// @ts-ignore
import { VueContext } from 'vue-context'
@Component({
name: 'neko-resolution',
components: {
'vue-context': VueContext,
},
})
export default class extends Vue {
@Ref('context') readonly context!: any
get configurations() {
return this.$accessor.video.configurations
}
open(event: MouseEvent) {
this.context.open(event)
}
screenSet(width: number, height: number, rate: number) {
this.$accessor.video.screenSet({ width, height, rate })
}
}
</script>

View File

@ -28,7 +28,11 @@
</div>
<div ref="aspect" class="player-aspect" />
</div>
<i v-if="!fullscreen" @click.stop.prevent="requestFullscreen" class="expand fas fa-expand"></i>
<ul v-if="!fullscreen" class="video-menu">
<li><i @click.stop.prevent="requestFullscreen" class="fas fa-expand"></i></li>
<li v-if="admin"><i @click.stop.prevent="onResolution" class="fas fa-cog"></i></li>
</ul>
<neko-resolution ref="resolution" />
</div>
</div>
</template>
@ -44,10 +48,15 @@
justify-content: center;
align-items: center;
.expand {
.video-menu {
position: absolute;
right: 20px;
top: 15px;
li {
margin: 0 0 10px 0;
i {
width: 30px;
height: 30px;
background: rgba($color: #fff, $alpha: 0.2);
@ -58,6 +67,8 @@
color: rgba($color: #fff, $alpha: 0.6);
cursor: pointer;
}
}
}
.player-container {
position: relative;
@ -129,11 +140,13 @@
import ResizeObserver from 'resize-observer-polyfill'
import Emote from './emote.vue'
import Resolution from './resolution.vue'
@Component({
name: 'neko-video',
components: {
'neko-emote': Emote,
'neko-resolution': Resolution,
},
})
export default class extends Vue {
@ -143,11 +156,16 @@
@Ref('aspect') readonly _aspect!: HTMLElement
@Ref('player') readonly _player!: HTMLElement
@Ref('video') readonly _video!: HTMLVideoElement
@Ref('resolution') readonly _resolution!: any
private observer = new ResizeObserver(this.onResise.bind(this))
private focused = false
private fullscreen = false
get admin() {
return this.$accessor.user.admin
}
get connected() {
return this.$accessor.connected
}
@ -200,6 +218,38 @@
return this.$accessor.remote.clipboard
}
get width() {
return this.$accessor.video.width
}
get height() {
return this.$accessor.video.height
}
get rate() {
return this.$accessor.video.rate
}
get vertical() {
return this.$accessor.video.vertical
}
get horizontal() {
return this.$accessor.video.horizontal
}
@Watch('width')
onWidthChanged(width: number) {
const { videoWidth, videoHeight } = this._video
console.log({ videoWidth, videoHeight })
this.onResise()
}
@Watch('height')
onHeightChanged(height: number) {
this.onResise()
}
@Watch('volume')
onVolumeChanged(volume: number) {
if (this._video) {
@ -292,8 +342,6 @@
this._video
.play()
.then(() => {
const { videoWidth, videoHeight } = this._video
this.$accessor.video.setResolution({ width: videoWidth, height: videoHeight })
this.onResise()
})
.catch(err => console.log(err))
@ -418,8 +466,6 @@
}
onResise() {
const { horizontal, vertical } = this.$accessor.video
let height = 0
if (!this.fullscreen) {
const { offsetWidth, offsetHeight } = this._component
@ -431,8 +477,12 @@
height = offsetHeight
}
this._container.style.maxWidth = `${(horizontal / vertical) * height}px`
this._aspect.style.paddingBottom = `${(vertical / horizontal) * 100}%`
this._container.style.maxWidth = `${(this.horizontal / this.vertical) * height}px`
this._aspect.style.paddingBottom = `${(this.vertical / this.horizontal) * 100}%`
}
onResolution(event: MouseEvent) {
this._resolution.open(event)
}
}
</script>

View File

@ -36,6 +36,11 @@ export const EVENT = {
MESSAGE: 'chat/message',
EMOTE: 'chat/emote',
},
SCREEN: {
CONFIGURATIONS: 'screen/configurations',
RESOLUTION: 'screen/resolution',
SET: 'screen/set',
},
ADMIN: {
BAN: 'admin/ban',
KICK: 'admin/kick',
@ -58,6 +63,7 @@ export type WebSocketEvents =
| MemberEvents
| SignalEvents
| ChatEvents
| ScreenEvents
| AdminEvents
export type ControlEvents =
@ -72,6 +78,8 @@ export type IdentityEvents = typeof EVENT.IDENTITY.PROVIDE | typeof EVENT.IDENTI
export type MemberEvents = typeof EVENT.MEMBER.LIST | typeof EVENT.MEMBER.CONNECTED | typeof EVENT.MEMBER.DISCONNECTED
export type SignalEvents = typeof EVENT.SIGNAL.ANSWER | typeof EVENT.SIGNAL.PROVIDE
export type ChatEvents = typeof EVENT.CHAT.MESSAGE | typeof EVENT.CHAT.EMOTE
export type ScreenEvents = typeof EVENT.SCREEN.CONFIGURATIONS | typeof EVENT.SCREEN.RESOLUTION | typeof EVENT.SCREEN.SET
export type AdminEvents =
| typeof EVENT.ADMIN.BAN
| typeof EVENT.ADMIN.KICK

View File

@ -15,9 +15,11 @@ import {
ControlTargetPayload,
ChatPayload,
EmotePayload,
ControlClipboardPayload,
ScreenConfigurationsPayload,
ScreenResolutionPayload,
AdminPayload,
AdminTargetPayload,
ControlClipboardPayload,
} from './messages'
interface NekoEvents extends BaseEvents {}
@ -285,6 +287,33 @@ export class NekoClient extends BaseClient implements EventEmitter<NekoEvents> {
this.$accessor.chat.newEmote({ type: emote })
}
/////////////////////////////
// Screen Events
/////////////////////////////
protected [EVENT.SCREEN.CONFIGURATIONS]({ configurations }: ScreenConfigurationsPayload) {
this.$accessor.video.setConfigurations(configurations)
}
protected [EVENT.SCREEN.RESOLUTION]({ id, width, height, rate }: ScreenResolutionPayload) {
this.$accessor.video.setResolution({ width, height, rate })
if (!id) {
return
}
const member = this.member(id)
if (!member || member.ignored) {
return
}
this.$accessor.chat.newMessage({
id,
content: `chaned the resolution to ${width}x${height}@${rate}`,
type: 'event',
created: new Date(),
})
}
/////////////////////////////
// Admin Events
/////////////////////////////

View File

@ -7,9 +7,10 @@ import {
MemberEvents,
SignalEvents,
ChatEvents,
ScreenEvents,
AdminEvents,
} from './events'
import { Member } from './types'
import { Member, ScreenConfigurations } from './types'
export type WebSocketMessages =
| WebSocketMessage
@ -19,6 +20,8 @@ export type WebSocketMessages =
| MembeConnectMessage
| MembeDisconnectMessage
| ControlMessage
| ScreenResolutionMessage
| ScreenConfigurationsMessage
| ChatMessage
export type WebSocketPayloads =
@ -31,6 +34,8 @@ export type WebSocketPayloads =
| ChatPayload
| ChatSendPayload
| EmojiSendPayload
| ScreenResolutionPayload
| ScreenConfigurationsPayload
| AdminPayload
export interface WebSocketMessage {
@ -145,6 +150,28 @@ export interface EmojiSendPayload {
emote: string
}
/*
SCREEN PAYLOADS
*/
export interface ScreenResolutionMessage extends WebSocketMessage, ScreenResolutionPayload {
event: ScreenEvents
}
export interface ScreenResolutionPayload {
id?: string
width: number
height: number
rate: number
}
export interface ScreenConfigurationsMessage extends WebSocketMessage, ScreenConfigurationsPayload {
event: ScreenEvents
}
export interface ScreenConfigurationsPayload {
configurations: ScreenConfigurations
}
/*
ADMIN PAYLOADS
*/

View File

@ -6,3 +6,13 @@ export interface Member {
connected?: boolean
ignored?: boolean
}
export interface ScreenConfigurations {
[index: number]: ScreenConfiguration
}
export interface ScreenConfiguration {
width: string
height: string
rates: { [index: number]: number }
}

View File

@ -1,5 +1,8 @@
import { getterTree, mutationTree, actionTree } from 'typed-vuex'
import { get, set } from '~/utils/localstorage'
import { EVENT } from '~/neko/events'
import { ScreenConfigurations } from '~/neko/types'
import { accessor } from '~/store'
export const namespaced = true
@ -7,8 +10,10 @@ export const state = () => ({
index: -1,
tracks: [] as MediaStreamTrack[],
streams: [] as MediaStream[],
configurations: {} as ScreenConfigurations,
width: 1280,
height: 720,
rate: 30,
horizontal: 16,
vertical: 9,
volume: get<number>('volume', 100),
@ -54,9 +59,10 @@ export const mutations = mutationTree(state, {
state.playable = playable
},
setResolution(state, { width, height }: { width: number; height: number }) {
setResolution(state, { width, height, rate }: { width: number; height: number; rate: number }) {
state.width = width
state.height = height
state.rate = rate
if ((height == 0 && width == 0) || (height == 0 && width != 0) || (height != 0 && width == 0)) {
return
@ -92,6 +98,10 @@ export const mutations = mutationTree(state, {
state.vertical = height / gcd
},
setConfigurations(state, configurations: ScreenConfigurations) {
state.configurations = configurations
},
setVolume(state, volume: number) {
state.volume = volume
set('volume', volume)
@ -115,11 +125,42 @@ export const mutations = mutationTree(state, {
state.index = -1
state.tracks = []
state.streams = []
state.configurations = []
state.width = 1280
state.height = 720
state.rate = 30
state.horizontal = 16
state.vertical = 9
state.playing = false
state.playable = false
},
})
export const actions = actionTree(
{ state, getters, mutations },
{
screenConfiguations({ state }) {
if (!accessor.connected || !accessor.user.admin) {
return
}
$client.sendMessage(EVENT.SCREEN.CONFIGURATIONS)
},
screenGet({ state }) {
if (!accessor.connected) {
return
}
$client.sendMessage(EVENT.SCREEN.RESOLUTION)
},
screenSet({ state }, { width, height, rate }: { width: number; height: number; rate: number }) {
if (!accessor.connected || !accessor.user.admin) {
return
}
$client.sendMessage(EVENT.SCREEN.SET, { width, height, rate })
},
},
)

View File

@ -44,6 +44,7 @@ type Pipeline struct {
Sample chan types.Sample
CodecName string
ClockRate float32
Src string
id int
}
@ -55,6 +56,8 @@ const (
videoClockRate = 90000
audioClockRate = 48000
pcmClockRate = 8000
videoSrc = "ximagesrc xid=%s show-pointer=true use-damage=false ! video/x-raw ! videoconvert ! queue ! "
audioSrc = "pulsesrc device=%s ! audioconvert ! "
)
func init() {
@ -63,8 +66,9 @@ func init() {
}
// CreatePipeline creates a GStreamer Pipeline
func CreatePipeline(codecName string, pipelineSrc string) (*Pipeline, error) {
pipelineStr := "appsink name=appsink"
func CreatePipeline(codecName string, pipelineDevice string, pipelineSrc string) (*Pipeline, error) {
pipelineStr := " ! appsink name=appsink"
var clockRate float32
switch codecName {
@ -72,93 +76,119 @@ func CreatePipeline(codecName string, pipelineSrc string) (*Pipeline, error) {
// https://gstreamer.freedesktop.org/documentation/vpx/vp8enc.html?gi-language=c
// gstreamer1.0-plugins-good
// vp8enc error-resilient=partitions keyframe-max-dist=10 auto-alt-ref=true cpu-used=5 deadline=1
pipelineStr = pipelineSrc + " ! vp8enc cpu-used=8 threads=2 deadline=1 error-resilient=partitions keyframe-max-dist=10 auto-alt-ref=true ! " + pipelineStr
clockRate = videoClockRate
if err := CheckPlugins([]string{"ximagesrc", "vpx"}); err != nil {
return nil, err
}
clockRate = videoClockRate
if pipelineSrc != "" {
pipelineStr = fmt.Sprintf(pipelineSrc+pipelineStr, pipelineDevice)
} else {
pipelineStr = fmt.Sprintf(videoSrc+"vp8enc cpu-used=8 threads=2 deadline=1 error-resilient=partitions keyframe-max-dist=10 auto-alt-ref=true"+pipelineStr, pipelineDevice)
}
case webrtc.VP9:
// https://gstreamer.freedesktop.org/documentation/vpx/vp9enc.html?gi-language=c
// gstreamer1.0-plugins-good
// vp9enc
// Causes panic! not sure why...
pipelineStr = pipelineSrc + " ! vp9enc ! " + pipelineStr
clockRate = videoClockRate
if err := CheckPlugins([]string{"ximagesrc", "vpx"}); err != nil {
return nil, err
}
clockRate = videoClockRate
// Causes panic! not sure why...
if pipelineSrc != "" {
pipelineStr = fmt.Sprintf(pipelineSrc+pipelineStr, pipelineDevice)
} else {
pipelineStr = fmt.Sprintf(videoSrc+"vp9enc"+pipelineStr, pipelineDevice)
}
case webrtc.H264:
// https://gstreamer.freedesktop.org/documentation/openh264/openh264enc.html?gi-language=c#openh264enc
// gstreamer1.0-plugins-bad
// openh264enc multi-thread=4 complexity=high bitrate=3072000 max-bitrate=4096000
pipelineStr = pipelineSrc + " ! openh264enc multi-thread=4 complexity=high bitrate=3072000 max-bitrate=4096000 ! video/x-h264,stream-format=byte-stream ! " + pipelineStr
clockRate = videoClockRate
if err := CheckPlugins([]string{"ximagesrc"}); err != nil {
return nil, err
}
clockRate = videoClockRate
if pipelineSrc != "" {
pipelineStr = fmt.Sprintf(pipelineSrc+pipelineStr, pipelineDevice)
} else {
pipelineStr = fmt.Sprintf(videoSrc+"openh264enc multi-thread=4 complexity=high bitrate=3072000 max-bitrate=4096000 ! video/x-h264,stream-format=byte-stream"+pipelineStr, pipelineDevice)
// https://gstreamer.freedesktop.org/documentation/x264/index.html?gi-language=c
// gstreamer1.0-plugins-ugly
// video/x-raw,format=I420 ! x264enc bframes=0 key-int-max=60 byte-stream=true tune=zerolatency speed-preset=veryfast ! video/x-h264,stream-format=byte-stream
if err := CheckPlugins([]string{"openh264"}); err != nil {
pipelineStr = pipelineSrc + " ! video/x-raw,format=I420 ! x264enc bframes=0 key-int-max=60 byte-stream=true tune=zerolatency speed-preset=veryfast ! video/x-h264,stream-format=byte-stream ! " + pipelineStr
pipelineStr = fmt.Sprintf(videoSrc+"video/x-raw,format=I420 ! x264enc bframes=0 key-int-max=60 byte-stream=true tune=zerolatency speed-preset=veryfast ! video/x-h264,stream-format=byte-stream"+pipelineStr, pipelineDevice)
if err := CheckPlugins([]string{"x264"}); err != nil {
return nil, err
}
}
}
case webrtc.Opus:
// https://gstreamer.freedesktop.org/documentation/opus/opusenc.html
// gstreamer1.0-plugins-base
// opusenc
pipelineStr = pipelineSrc + " ! opusenc ! " + pipelineStr
clockRate = audioClockRate
if err := CheckPlugins([]string{"pulseaudio", "opus"}); err != nil {
return nil, err
}
clockRate = audioClockRate
if pipelineSrc != "" {
pipelineStr = fmt.Sprintf(pipelineSrc+pipelineStr, pipelineDevice)
} else {
pipelineStr = fmt.Sprintf(audioSrc+"opusenc"+pipelineStr, pipelineDevice)
}
case webrtc.G722:
// https://gstreamer.freedesktop.org/documentation/libav/avenc_g722.html?gi-language=c
// gstreamer1.0-libav
// avenc_g722
pipelineStr = pipelineSrc + " ! avenc_g722 ! " + pipelineStr
clockRate = audioClockRate
if err := CheckPlugins([]string{"pulseaudio", "libav"}); err != nil {
return nil, err
}
clockRate = audioClockRate
if pipelineSrc != "" {
pipelineStr = fmt.Sprintf(pipelineSrc+pipelineStr, pipelineDevice)
} else {
pipelineStr = fmt.Sprintf(audioSrc+"avenc_g722"+pipelineStr, pipelineDevice)
}
case webrtc.PCMU:
// https://gstreamer.freedesktop.org/documentation/mulaw/mulawenc.html?gi-language=c
// gstreamer1.0-plugins-good
// audio/x-raw, rate=8000 ! mulawenc
pipelineStr = pipelineSrc + " ! audio/x-raw, rate=8000 ! mulawenc ! " + pipelineStr
clockRate = pcmClockRate
if err := CheckPlugins([]string{"pulseaudio", "mulaw"}); err != nil {
return nil, err
}
clockRate = pcmClockRate
if pipelineSrc != "" {
pipelineStr = fmt.Sprintf(pipelineSrc+pipelineStr, pipelineDevice)
} else {
pipelineStr = fmt.Sprintf(audioSrc+"audio/x-raw, rate=8000 ! mulawenc"+pipelineStr, pipelineDevice)
}
case webrtc.PCMA:
// https://gstreamer.freedesktop.org/documentation/alaw/alawenc.html?gi-language=c
// gstreamer1.0-plugins-good
// audio/x-raw, rate=8000 ! alawenc
pipelineStr = pipelineSrc + " ! audio/x-raw, rate=8000 ! alawenc ! " + pipelineStr
clockRate = pcmClockRate
if err := CheckPlugins([]string{"pulseaudio", "alaw"}); err != nil {
return nil, err
}
clockRate = pcmClockRate
if pipelineSrc != "" {
pipelineStr = fmt.Sprintf(pipelineSrc+pipelineStr, pipelineDevice)
} else {
pipelineStr = fmt.Sprintf(audioSrc+"audio/x-raw, rate=8000 ! alawenc"+pipelineStr, pipelineDevice)
}
default:
return nil, errors.Errorf("unknown video codec %s", codecName)
}
@ -169,16 +199,17 @@ func CreatePipeline(codecName string, pipelineSrc string) (*Pipeline, error) {
pipelinesLock.Lock()
defer pipelinesLock.Unlock()
pipeline := &Pipeline{
p := &Pipeline{
Pipeline: C.gstreamer_send_create_pipeline(pipelineStrUnsafe),
Sample: make(chan types.Sample),
CodecName: codecName,
ClockRate: clockRate,
Src: pipelineStr,
id: len(pipelines),
}
pipelines[pipeline.id] = pipeline
return pipeline, nil
pipelines[p.id] = p
return p, nil
}
// Start starts the GStreamer Pipeline

View File

@ -1,33 +1,53 @@
package event
const SYSTEM_DISCONNECT = "system/disconnect"
const (
SYSTEM_DISCONNECT = "system/disconnect"
)
const SIGNAL_ANSWER = "signal/answer"
const SIGNAL_PROVIDE = "signal/provide"
const (
SIGNAL_ANSWER = "signal/answer"
SIGNAL_PROVIDE = "signal/provide"
)
const IDENTITY_PROVIDE = "identity/provide"
const IDENTITY_DETAILS = "identity/details"
const (
IDENTITY_PROVIDE = "identity/provide"
IDENTITY_DETAILS = "identity/details"
)
const MEMBER_LIST = "member/list"
const MEMBER_CONNECTED = "member/connected"
const MEMBER_DISCONNECTED = "member/disconnected"
const (
MEMBER_LIST = "member/list"
MEMBER_CONNECTED = "member/connected"
MEMBER_DISCONNECTED = "member/disconnected"
)
const CONTROL_LOCKED = "control/locked"
const CONTROL_RELEASE = "control/release"
const CONTROL_REQUEST = "control/request"
const CONTROL_REQUESTING = "control/requesting"
const CONTROL_GIVE = "control/give"
const CONTROL_CLIPBOARD = "control/clipboard"
const (
CONTROL_LOCKED = "control/locked"
CONTROL_RELEASE = "control/release"
CONTROL_REQUEST = "control/request"
CONTROL_REQUESTING = "control/requesting"
CONTROL_GIVE = "control/give"
CONTROL_CLIPBOARD = "control/clipboard"
)
const CHAT_MESSAGE = "chat/message"
const CHAT_EMOTE = "chat/emote"
const (
CHAT_MESSAGE = "chat/message"
CHAT_EMOTE = "chat/emote"
)
const ADMIN_BAN = "admin/ban"
const ADMIN_KICK = "admin/kick"
const ADMIN_LOCK = "admin/lock"
const ADMIN_MUTE = "admin/mute"
const ADMIN_UNLOCK = "admin/unlock"
const ADMIN_UNMUTE = "admin/unmute"
const ADMIN_CONTROL = "admin/control"
const ADMIN_RELEASE = "admin/release"
const ADMIN_GIVE = "admin/give"
const (
SCREEN_CONFIGURATIONS = "screen/configurations"
SCREEN_RESOLUTION = "screen/resolution"
SCREEN_SET = "screen/set"
)
const (
ADMIN_BAN = "admin/ban"
ADMIN_KICK = "admin/kick"
ADMIN_LOCK = "admin/lock"
ADMIN_MUTE = "admin/mute"
ADMIN_UNLOCK = "admin/unlock"
ADMIN_UNMUTE = "admin/unmute"
ADMIN_CONTROL = "admin/control"
ADMIN_RELEASE = "admin/release"
ADMIN_GIVE = "admin/give"
)

View File

@ -90,3 +90,16 @@ type AdminTarget struct {
Target string `json:"target"`
ID string `json:"id"`
}
type ScreenResolution struct {
Event string `json:"event"`
ID string `json:"id,omitempty"`
Width int `json:"width"`
Height int `json:"height"`
Rate int `json:"rate"`
}
type ScreenConfigurations struct {
Event string `json:"event"`
Configurations map[int]types.ScreenConfiguration `json:"configurations"`
}

View File

@ -9,6 +9,7 @@ type WebRTCManager interface {
Start()
Shutdown() error
CreatePeer(id string, sdp string) (string, Peer, error)
ChangeScreenSize(width int, height int, rate int) error
}
type Peer interface {

View File

@ -0,0 +1,13 @@
package types
type ScreenSize struct {
Width int `json:"width"`
Height int `json:"height"`
Rate int16 `json:"rate"`
}
type ScreenConfiguration struct {
Width int `json:"width"`
Height int `json:"height"`
Rates map[int]int16 `json:"rates"`
}

View File

@ -6,7 +6,7 @@ import (
"strconv"
"github.com/pion/webrtc/v2"
"n.eko.moe/neko/internal/hid"
"n.eko.moe/neko/internal/xorg"
)
const OP_MOVE = 0x01
@ -63,7 +63,7 @@ func (m *WebRTCManager) handle(id string, msg webrtc.DataChannelMessage) error {
return err
}
hid.Move(int(payload.X), int(payload.Y))
xorg.Move(int(payload.X), int(payload.Y))
break
case OP_SCROLL:
payload := &PayloadScroll{}
@ -77,7 +77,7 @@ func (m *WebRTCManager) handle(id string, msg webrtc.DataChannelMessage) error {
Str("y", strconv.Itoa(int(payload.Y))).
Msg("scroll")
hid.Scroll(int(payload.X), int(payload.Y))
xorg.Scroll(int(payload.X), int(payload.Y))
break
case OP_KEY_DOWN:
payload := &PayloadKey{}
@ -86,7 +86,7 @@ func (m *WebRTCManager) handle(id string, msg webrtc.DataChannelMessage) error {
}
if payload.Key < 8 {
button, err := hid.ButtonDown(int(payload.Key))
button, err := xorg.ButtonDown(int(payload.Key))
if err != nil {
m.logger.Warn().Err(err).Msg("key down failed")
return nil
@ -94,7 +94,7 @@ func (m *WebRTCManager) handle(id string, msg webrtc.DataChannelMessage) error {
m.logger.Debug().Msgf("button down %s(%d)", button.Name, payload.Key)
} else {
key, err := hid.KeyDown(int(payload.Key))
key, err := xorg.KeyDown(int(payload.Key))
if err != nil {
m.logger.Warn().Err(err).Msg("key down failed")
return nil
@ -112,7 +112,7 @@ func (m *WebRTCManager) handle(id string, msg webrtc.DataChannelMessage) error {
}
if payload.Key < 8 {
button, err := hid.ButtonUp(int(payload.Key))
button, err := xorg.ButtonUp(int(payload.Key))
if err != nil {
m.logger.Warn().Err(err).Msg("button up failed")
return nil
@ -120,7 +120,7 @@ func (m *WebRTCManager) handle(id string, msg webrtc.DataChannelMessage) error {
m.logger.Debug().Msgf("button up %s(%d)", button.Name, payload.Key)
} else {
key, err := hid.KeyUp(int(payload.Key))
key, err := xorg.KeyUp(int(payload.Key))
if err != nil {
m.logger.Warn().Err(err).Msg("keyup failed")
return nil

View File

@ -2,6 +2,7 @@ package webrtc
import (
"fmt"
"strings"
"time"
"github.com/pion/webrtc/v2"
@ -9,9 +10,9 @@ import (
"github.com/rs/zerolog/log"
"n.eko.moe/neko/internal/gst"
"n.eko.moe/neko/internal/hid"
"n.eko.moe/neko/internal/types"
"n.eko.moe/neko/internal/types/config"
"n.eko.moe/neko/internal/xorg"
)
func New(sessions types.SessionManager, config *config.WebRTC) *WebRTCManager {
@ -53,11 +54,12 @@ type WebRTCManager struct {
}
func (m *WebRTCManager) Start() {
hid.Display(m.config.Display)
xorg.Display(m.config.Display)
videoPipeline, err := gst.CreatePipeline(
m.config.VideoCodec,
fmt.Sprintf("ximagesrc xid=%s show-pointer=true use-damage=false ! video/x-raw,framerate=30/1 ! videoconvert ! queue", m.config.Display),
m.config.Display,
m.config.VideoParams,
)
if err != nil {
@ -66,7 +68,8 @@ func (m *WebRTCManager) Start() {
audioPipeline, err := gst.CreatePipeline(
m.config.AudioCodec,
fmt.Sprintf("pulsesrc device=%s ! audioconvert", m.config.Device),
m.config.Device,
m.config.AudioParams,
)
if err != nil {
@ -88,22 +91,22 @@ func (m *WebRTCManager) Start() {
select {
case <-m.shutdown:
return
case sample := <-videoPipeline.Sample:
case sample := <-m.videoPipeline.Sample:
if err := m.sessions.WriteVideoSample(sample); err != nil {
m.logger.Warn().Err(err).Msg("video pipeline failed to write")
}
case sample := <-audioPipeline.Sample:
case sample := <-m.audioPipeline.Sample:
if err := m.sessions.WriteAudioSample(sample); err != nil {
m.logger.Warn().Err(err).Msg("audio pipeline failed to write")
}
case <-m.cleanup.C:
hid.Check(time.Second * 10)
xorg.CheckKeys(time.Second * 10)
}
}
}()
m.sessions.OnHostCleared(func(id string) {
hid.Reset()
xorg.ResetKeys()
})
m.sessions.OnCreated(func(id string, session types.Session) {
@ -120,6 +123,10 @@ func (m *WebRTCManager) Start() {
Str("video_codec", m.config.VideoCodec).
Str("audio_device", m.config.Device).
Str("audio_codec", m.config.AudioCodec).
Str("ephemeral_port_range", fmt.Sprintf("%d-%d", m.config.EphemeralMin, m.config.EphemeralMax)).
Str("nat_ips", strings.Join(m.config.NAT1To1IPs, ",")).
Str("audio_pipeline_src", audioPipeline.Src).
Str("video_pipeline_src", videoPipeline.Src).
Msgf("webrtc streaming")
}
@ -180,7 +187,7 @@ func (m *WebRTCManager) CreatePeer(id string, sdp string) (string, types.Peer, e
return "", nil, err
}
// set remote description
// Set remote description
connection.SetRemoteDescription(description)
answer, err := connection.CreateAnswer(nil)
@ -222,3 +229,26 @@ func (m *WebRTCManager) CreatePeer(id string, sdp string) (string, types.Peer, e
connection: connection,
}, nil
}
func (m *WebRTCManager) ChangeScreenSize(width int, height int, rate int) error {
m.videoPipeline.Stop()
if err := xorg.ChangeScreenSize(width, height, rate); err != nil {
return err
}
videoPipeline, err := gst.CreatePipeline(
m.config.VideoCodec,
m.config.Display,
m.config.VideoParams,
)
if err != nil {
m.logger.Panic().Err(err).Msg("unable to create new video pipeline")
}
m.videoPipeline = videoPipeline
m.videoPipeline.Start()
return nil
}

View File

@ -1,10 +1,10 @@
package websocket
import (
"n.eko.moe/neko/internal/hid"
"n.eko.moe/neko/internal/types"
"n.eko.moe/neko/internal/types/event"
"n.eko.moe/neko/internal/types/message"
"n.eko.moe/neko/internal/xorg"
)
func (h *MessageHandler) controlRelease(id string, session types.Session) error {
@ -113,6 +113,6 @@ func (h *MessageHandler) controlClipboard(id string, session types.Session, payl
return nil
}
hid.WriteClipboard(payload.Text)
xorg.WriteClipboard(payload.Text)
return nil
}

View File

@ -103,6 +103,18 @@ func (h *MessageHandler) Message(id string, raw []byte) error {
return h.chatEmote(id, session, payload)
}), "%s failed", header.Event)
// Screen Events
case event.SCREEN_RESOLUTION:
return errors.Wrapf(h.screenResolution(id, session), "%s failed", header.Event)
case event.SCREEN_CONFIGURATIONS:
return errors.Wrapf(h.screenConfigurations(id, session), "%s failed", header.Event)
case event.SCREEN_SET:
payload := &message.ScreenResolution{}
return errors.Wrapf(
utils.Unmarshal(payload, raw, func() error {
return h.screenSet(id, session, payload)
}), "%s failed", header.Event)
// Admin Events
case event.ADMIN_LOCK:
return errors.Wrapf(h.adminLock(id, session), "%s failed", header.Event)

View File

@ -0,0 +1,67 @@
package websocket
import (
"n.eko.moe/neko/internal/types"
"n.eko.moe/neko/internal/types/event"
"n.eko.moe/neko/internal/types/message"
"n.eko.moe/neko/internal/xorg"
)
func (h *MessageHandler) screenSet(id string, session types.Session, payload *message.ScreenResolution) error {
if !session.Admin() {
h.logger.Debug().Msg("user not admin")
return nil
}
if err := h.webrtc.ChangeScreenSize(payload.Width, payload.Height, payload.Rate); err != nil {
h.logger.Warn().Err(err).Msgf("unable to change screen size")
return err
}
if err := h.sessions.Brodcast(
message.ScreenResolution{
Event: event.SCREEN_RESOLUTION,
ID: id,
Width: payload.Width,
Height: payload.Height,
Rate: payload.Rate,
}, nil); err != nil {
h.logger.Warn().Err(err).Msgf("brodcasting event %s has failed", event.SCREEN_RESOLUTION)
return err
}
return nil
}
func (h *MessageHandler) screenResolution(id string, session types.Session) error {
if size := xorg.GetScreenSize(); size != nil {
if err := session.Send(message.ScreenResolution{
Event: event.SCREEN_RESOLUTION,
Width: size.Width,
Height: size.Height,
Rate: int(size.Rate),
}); err != nil {
h.logger.Warn().Err(err).Msgf("sending event %s has failed", event.SCREEN_RESOLUTION)
return err
}
}
return nil
}
func (h *MessageHandler) screenConfigurations(id string, session types.Session) error {
if !session.Admin() {
h.logger.Debug().Msg("user not admin")
return nil
}
if err := session.Send(message.ScreenConfigurations{
Event: event.SCREEN_CONFIGURATIONS,
Configurations: xorg.ScreenConfigurations,
}); err != nil {
h.logger.Warn().Err(err).Msgf("sending event %s has failed", event.SCREEN_CONFIGURATIONS)
return err
}
return nil
}

View File

@ -14,6 +14,18 @@ func (h *MessageHandler) SessionCreated(id string, session types.Session) error
return err
}
// send screen current resolution
if err := h.screenResolution(id, session); err != nil {
return err
}
if session.Admin() {
// send screen configurations
if err := h.screenConfigurations(id, session); err != nil {
return err
}
}
return nil
}

View File

@ -9,12 +9,12 @@ import (
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"n.eko.moe/neko/internal/hid"
"n.eko.moe/neko/internal/types"
"n.eko.moe/neko/internal/types/config"
"n.eko.moe/neko/internal/types/event"
"n.eko.moe/neko/internal/types/message"
"n.eko.moe/neko/internal/utils"
"n.eko.moe/neko/internal/xorg"
)
func New(sessions types.SessionManager, webrtc types.WebRTCManager, conf *config.WebSocket) *WebSocketHandler {
@ -81,7 +81,7 @@ func (ws *WebSocketHandler) Start() error {
ws.logger.Info().Msg("shutdown")
}()
current := hid.ReadClipboard()
current := xorg.ReadClipboard()
for {
select {
@ -89,7 +89,7 @@ func (ws *WebSocketHandler) Start() error {
return
default:
if ws.sessions.HasHost() {
text := hid.ReadClipboard()
text := xorg.ReadClipboard()
if text != current {
session, ok := ws.sessions.GetHost()
if ok {

View File

@ -1,4 +1,4 @@
#include "hid.h"
#include "xorg.h"
static clipboard_c *CLIPBOARD = NULL;
static Display *DISPLAY = NULL;
@ -9,7 +9,7 @@ static int DIRTY = 0;
Display *getXDisplay(void) {
/* Close the display if displayName has changed */
if (DIRTY) {
closeXDisplay();
XDisplayClose();
DIRTY = 0;
}
@ -25,7 +25,7 @@ Display *getXDisplay(void) {
if (DISPLAY == NULL) {
fputs("Could not open main display\n", stderr);
} else if (!REGISTERED) {
atexit(&closeXDisplay);
atexit(&XDisplayClose);
REGISTERED = 1;
}
}
@ -40,14 +40,14 @@ clipboard_c *getClipboard(void) {
return CLIPBOARD;
}
void closeXDisplay(void) {
void XDisplayClose(void) {
if (DISPLAY != NULL) {
XCloseDisplay(DISPLAY);
DISPLAY = NULL;
}
}
void setXDisplay(char *input) {
void XDisplaySet(char *input) {
NAME = strdup(input);
DIRTY = 1;
}
@ -110,3 +110,41 @@ char *XClipboardGet() {
clipboard_c *cb = getClipboard();
return clipboard_text_ex(cb, NULL, 0);
}
void XGetScreenConfigurations() {
Display *display = getXDisplay();
Window root = RootWindow(display, 0);
XRRScreenSize *xrrs;
int num_sizes;
xrrs = XRRSizes(display, 0, &num_sizes);
for(int i = 0; i < num_sizes; i ++) {
short *rates;
int num_rates;
goCreateScreenSize(i, xrrs[i].width, xrrs[i].height, xrrs[i].mwidth, xrrs[i].mheight);
rates = XRRRates(display, 0, i, &num_rates);
for (int j = 0; j < num_rates; j ++) {
goSetScreenRates(i, j, rates[j]);
}
}
}
void XSetScreenConfiguration(int index, short rate) {
Display *display = getXDisplay();
Window root = RootWindow(display, 0);
XRRSetScreenConfigAndRate(display, XRRGetScreenInfo(display, root), root, index, RR_Rotate_0, rate, CurrentTime);
}
int XGetScreenSize() {
Display *display = getXDisplay();
XRRScreenConfiguration *conf = XRRGetScreenInfo(display, RootWindow(display, 0));
Rotation original_rotation;
return XRRConfigCurrentConfiguration(conf, &original_rotation);
}
short XGetScreenRate() {
Display *display = getXDisplay();
XRRScreenConfiguration *conf = XRRGetScreenInfo(display, RootWindow(display, 0));
return XRRConfigCurrentRate(conf);
}

View File

@ -4,13 +4,13 @@
// pretty sure this *isn't* thread safe either.... /shrug
// if you know a better way to get this done *please* make a pr <3
package hid
package xorg
/*
#cgo linux CFLAGS: -I/usr/src
#cgo linux LDFLAGS: -L/usr/src -lX11 -lXtst -lclipboard
#cgo linux LDFLAGS: -L/usr/src -lX11 -lXtst -lXrandr -lclipboard
#include "hid.h"
#include "xorg.h"
*/
import "C"
@ -18,10 +18,14 @@ import (
"fmt"
"sync"
"time"
"unsafe"
"n.eko.moe/neko/internal/hid/keycode"
"n.eko.moe/neko/internal/types"
"n.eko.moe/neko/internal/xorg/keycode"
)
var ScreenConfigurations = make(map[int]types.ScreenConfiguration)
var debounce = make(map[int]time.Time)
var buttons = make(map[int]keycode.Button)
var keys = make(map[int]keycode.Key)
@ -135,13 +139,18 @@ func init() {
buttons[keycode.SCROLL_DOWN_BUTTON.Code] = keycode.SCROLL_DOWN_BUTTON
buttons[keycode.SCROLL_LEFT_BUTTON.Code] = keycode.SCROLL_LEFT_BUTTON
buttons[keycode.SCROLL_RIGHT_BUTTON.Code] = keycode.SCROLL_RIGHT_BUTTON
C.XGetScreenConfigurations()
}
func Display(display string) {
mu.Lock()
defer mu.Unlock()
C.setXDisplay(C.CString(display))
displayUnsafe := C.CString(display)
defer C.free(unsafe.Pointer(displayUnsafe))
C.XDisplaySet(displayUnsafe)
}
func Move(x, y int) {
@ -238,17 +247,23 @@ func ReadClipboard() string {
mu.Lock()
defer mu.Unlock()
return C.GoString(C.XClipboardGet())
clipboardUnsafe := C.XClipboardGet()
defer C.free(unsafe.Pointer(clipboardUnsafe))
return C.GoString(clipboardUnsafe)
}
func WriteClipboard(data string) {
mu.Lock()
defer mu.Unlock()
C.XClipboardSet(C.CString(data))
clipboardUnsafe := C.CString(data)
defer C.free(unsafe.Pointer(clipboardUnsafe))
C.XClipboardSet(clipboardUnsafe)
}
func Reset() {
func ResetKeys() {
for key := range debounce {
if key < 8 {
ButtonUp(key)
@ -260,7 +275,7 @@ func Reset() {
}
}
func Check(duration time.Duration) {
func CheckKeys(duration time.Duration) {
t := time.Now()
for key, start := range debounce {
if t.Sub(start) < duration {
@ -276,3 +291,53 @@ func Check(duration time.Duration) {
delete(debounce, key)
}
}
func ChangeScreenSize(width int, height int, rate int) error {
mu.Lock()
defer mu.Unlock()
for index, size := range ScreenConfigurations {
if size.Width == width && size.Height == height {
for _, srate := range size.Rates {
if int16(rate) == srate {
C.XSetScreenConfiguration(C.int(index), C.short(srate))
return nil
}
}
}
}
return fmt.Errorf("unknown configuration")
}
func GetScreenSize() *types.ScreenSize {
mu.Lock()
defer mu.Unlock()
index := int(C.XGetScreenSize())
rate := int16(C.XGetScreenRate())
if conf, ok := ScreenConfigurations[index]; ok {
return &types.ScreenSize{
Width: conf.Width,
Height: conf.Height,
Rate: rate,
}
}
return nil
}
//export goCreateScreenSize
func goCreateScreenSize(index C.int, width C.int, height C.int, mwidth C.int, mheight C.int) {
ScreenConfigurations[int(index)] = types.ScreenConfiguration{
Width: int(width),
Height: int(height),
Rates: make(map[int]int16),
}
}
//export goSetScreenRates
func goSetScreenRates(index C.int, rate_index C.int, rate C.short) {
ScreenConfigurations[int(index)].Rates[int(rate_index)] = int16(rate)
}

View File

@ -4,6 +4,7 @@
#define XDISPLAY_H
#include <X11/Xlib.h>
#include <X11/extensions/Xrandr.h>
#include <X11/extensions/XTest.h>
#include <libclipboard.h>
#include <stdint.h>
@ -11,6 +12,9 @@
#include <stdio.h> /* For fputs() */
#include <string.h> /* For strdup() */
extern void goCreateScreenSize(int index, int width, int height, int mwidth, int mheight);
extern void goSetScreenRates(int index, int rate_index, short rate);
/* Returns the main display, closed either on exit or when closeMainDisplay()
* is invoked. This removes a bit of the overhead of calling XOpenDisplay() &
* XCloseDisplay() everytime the main display needs to be used.
@ -19,14 +23,20 @@
Display *getXDisplay(void);
clipboard_c *getClipboard(void);
void XClipboardSet(char *src);
char *XClipboardGet();
void XMove(int x, int y);
void XScroll(int x, int y);
void XButton(unsigned int button, int down);
void XKey(unsigned long key, int down);
void closeXDisplay(void);
void setXDisplay(char *input);
void XClipboardSet(char *src);
char *XClipboardGet();
void XGetScreenConfigurations();
void XSetScreenConfiguration(int index, short rate);
int XGetScreenSize();
short XGetScreenRate();
void XDisplayClose(void);
void XDisplaySet(char *input);
#endif