major refactor.

internal/capture/broadcast.go

@@ -0,0 +1,60 @@
+package capture
+
+import (
+	"demodesk/neko/internal/capture/gst"
+)
+
+func (manager *CaptureManagerCtx) StartBroadcastPipeline() {
+	var err error
+
+	if manager.IsBoradcasting() || manager.broadcast_url == "" {
+		return
+	}
+
+	manager.logger.Info().
+		Str("audio_device", manager.config.Device).
+		Str("video_display", manager.config.Display).
+		Str("rtmp_pipeline_src", manager.broadcast.Src).
+		Msgf("Creating broadcast pipeline...")
+	
+	manager.broadcast, err = gst.CreateRTMPPipeline(
+		manager.config.Device,
+		manager.config.Display,
+		manager.config.BroadcastPipeline,
+		manager.broadcast_url,
+	)
+
+	if err != nil {
+		manager.logger.Panic().Err(err).Msg("unable to create broadcast pipeline")
+	}
+
+	manager.broadcast.Play()
+	manager.logger.Info().Msgf("Starting broadcast pipeline...")
+}
+
+func (manager *CaptureManagerCtx) StopBroadcastPipeline() {
+	if !manager.IsBoradcasting() {
+		return
+	}
+
+	manager.broadcast.DestroyPipeline()
+	manager.broadcast = nil
+}
+
+func (manager *CaptureManagerCtx) StartBroadcast(url string) {
+	manager.broadcast_url = url
+	manager.StartBroadcastPipeline()
+}
+
+func (manager *CaptureManagerCtx) StopBroadcast() {
+	manager.broadcast_url = ""
+	manager.StopBroadcastPipeline()
+}
+
+func (manager *CaptureManagerCtx) IsBoradcasting() bool {
+	return manager.broadcast != nil
+}
+
+func (manager *CaptureManagerCtx) BroadcastUrl() string {
+	return manager.broadcast_url
+}

internal/capture/gst/gst.c

@@ -0,0 +1,98 @@
+#include "gst.h"
+
+#include <gst/app/gstappsrc.h>
+
+typedef struct SampleHandlerUserData {
+  int pipelineId;
+} SampleHandlerUserData;
+
+void gstreamer_init(void) {
+  gst_init(NULL, NULL);
+}
+
+GMainLoop *gstreamer_send_main_loop = NULL;
+void gstreamer_send_start_mainloop(void) {
+  gstreamer_send_main_loop = g_main_loop_new(NULL, FALSE);
+
+  g_main_loop_run(gstreamer_send_main_loop);
+}
+
+static gboolean gstreamer_send_bus_call(GstBus *bus, GstMessage *msg, gpointer data) {
+  switch (GST_MESSAGE_TYPE(msg)) {
+
+  case GST_MESSAGE_EOS:
+    g_print("End of stream\n");
+    exit(1);
+    break;
+
+  case GST_MESSAGE_ERROR: {
+    gchar *debug;
+    GError *error;
+
+    gst_message_parse_error(msg, &error, &debug);
+    g_free(debug);
+
+    g_printerr("Error: %s\n", error->message);
+    g_error_free(error);
+    exit(1);
+  }
+  default:
+    break;
+  }
+
+  return TRUE;
+}
+
+GstFlowReturn gstreamer_send_new_sample_handler(GstElement *object, gpointer user_data) {
+  GstSample *sample = NULL;
+  GstBuffer *buffer = NULL;
+  gpointer copy = NULL;
+  gsize copy_size = 0;
+  SampleHandlerUserData *s = (SampleHandlerUserData *)user_data;
+
+  g_signal_emit_by_name(object, "pull-sample", &sample);
+  if (sample) {
+    buffer = gst_sample_get_buffer(sample);
+    if (buffer) {
+      gst_buffer_extract_dup(buffer, 0, gst_buffer_get_size(buffer), &copy, &copy_size);
+      goHandlePipelineBuffer(copy, copy_size, GST_BUFFER_DURATION(buffer), s->pipelineId);
+    }
+    gst_sample_unref(sample);
+  }
+
+  return GST_FLOW_OK;
+}
+
+GstElement *gstreamer_send_create_pipeline(char *pipeline) {
+  GError *error = NULL;
+  return gst_parse_launch(pipeline, &error);
+}
+
+void gstreamer_send_destroy_pipeline(GstElement *pipeline) {
+  gst_element_set_state(pipeline, GST_STATE_NULL);
+  gst_object_unref(pipeline);
+}
+
+void gstreamer_send_start_pipeline(GstElement *pipeline, int pipelineId) {
+  SampleHandlerUserData *s = calloc(1, sizeof(SampleHandlerUserData));
+  s->pipelineId = pipelineId;
+
+  GstBus *bus = gst_pipeline_get_bus(GST_PIPELINE(pipeline));
+  gst_bus_add_watch(bus, gstreamer_send_bus_call, NULL);
+  gst_object_unref(bus);
+
+  GstElement *appsink = gst_bin_get_by_name(GST_BIN(pipeline), "appsink");
+  g_object_set(appsink, "emit-signals", TRUE, NULL);
+  g_signal_connect(appsink, "new-sample", G_CALLBACK(gstreamer_send_new_sample_handler), s);
+  gst_object_unref(appsink);
+
+  gst_element_set_state(pipeline, GST_STATE_PLAYING);
+}
+
+void gstreamer_send_play_pipeline(GstElement *pipeline) {
+  gst_element_set_state(pipeline, GST_STATE_PLAYING);
+}
+
+void gstreamer_send_stop_pipeline(GstElement *pipeline) {
+  gst_element_set_state(pipeline, GST_STATE_NULL);
+}

internal/capture/gst/gst.go

@@ -0,0 +1,261 @@
+package gst
+
+/*
+#cgo pkg-config: gstreamer-1.0 gstreamer-app-1.0
+
+#include "gst.h"
+*/
+import "C"
+import (
+	"fmt"
+	"sync"
+	"unsafe"
+
+	"github.com/pion/webrtc/v2"
+
+	"demodesk/neko/internal/types"
+)
+
+// Pipeline is a wrapper for a GStreamer Pipeline
+type Pipeline struct {
+	Pipeline  *C.GstElement
+	Sample    chan types.Sample
+	CodecName string
+	ClockRate float32
+	Src       string
+	id        int
+}
+
+var pipelines = make(map[int]*Pipeline)
+var pipelinesLock sync.Mutex
+var registry *C.GstRegistry
+
+const (
+	videoClockRate = 90000
+	audioClockRate = 48000
+	pcmClockRate   = 8000
+	videoSrc       = "ximagesrc xid=%s show-pointer=true use-damage=false ! video/x-raw ! videoconvert ! queue ! "
+	audioSrc       = "pulsesrc device=%s ! audio/x-raw,channels=2 ! audioconvert ! "
+)
+
+func init() {
+	C.gstreamer_init()
+	registry = C.gst_registry_get()
+}
+
+// CreateRTMPPipeline creates a GStreamer Pipeline
+func CreateRTMPPipeline(pipelineDevice string, pipelineDisplay string, pipelineSrc string, pipelineRTMP string) (*Pipeline, error) {
+	video := fmt.Sprintf(videoSrc, pipelineDisplay)
+	audio := fmt.Sprintf(audioSrc, pipelineDevice)
+
+	var pipelineStr string
+	if pipelineSrc != "" {
+		pipelineStr = fmt.Sprintf(pipelineSrc, pipelineRTMP, pipelineDevice, pipelineDisplay)
+	} else {
+		pipelineStr = fmt.Sprintf("flvmux name=mux ! rtmpsink location='%s live=1' %s audio/x-raw,channels=2 ! audioconvert ! voaacenc ! mux. %s x264enc bframes=0 key-int-max=60 byte-stream=true tune=zerolatency speed-preset=veryfast ! mux.", pipelineRTMP, audio, video)
+	}
+
+	return CreatePipeline(pipelineStr, "", 0)
+}
+
+// CreateAppPipeline creates a GStreamer Pipeline
+func CreateAppPipeline(codecName string, pipelineDevice string, pipelineSrc string) (*Pipeline, error) {
+	pipelineStr := " ! appsink name=appsink"
+
+	var clockRate float32
+
+	switch codecName {
+	case webrtc.VP8:
+		// https://gstreamer.freedesktop.org/documentation/vpx/vp8enc.html?gi-language=c
+		// gstreamer1.0-plugins-good
+		// vp8enc error-resilient=partitions keyframe-max-dist=10 auto-alt-ref=true cpu-used=5 deadline=1
+		if err := CheckPlugins([]string{"ximagesrc", "vpx"}); err != nil {
+			return nil, err
+		}
+
+		clockRate = videoClockRate
+
+		if pipelineSrc != "" {
+			pipelineStr = fmt.Sprintf(pipelineSrc+pipelineStr, pipelineDevice)
+		} else {
+			pipelineStr = fmt.Sprintf(videoSrc+"vp8enc cpu-used=8 threads=2 deadline=1 error-resilient=partitions keyframe-max-dist=10 auto-alt-ref=true"+pipelineStr, pipelineDevice)
+		}
+	case webrtc.VP9:
+		// https://gstreamer.freedesktop.org/documentation/vpx/vp9enc.html?gi-language=c
+		// gstreamer1.0-plugins-good
+		// vp9enc
+		if err := CheckPlugins([]string{"ximagesrc", "vpx"}); err != nil {
+			return nil, err
+		}
+
+		clockRate = videoClockRate
+
+		// Causes panic! not sure why...
+		if pipelineSrc != "" {
+			pipelineStr = fmt.Sprintf(pipelineSrc+pipelineStr, pipelineDevice)
+		} else {
+			pipelineStr = fmt.Sprintf(videoSrc+"vp9enc"+pipelineStr, pipelineDevice)
+		}
+	case webrtc.H264:
+		// https://gstreamer.freedesktop.org/documentation/openh264/openh264enc.html?gi-language=c#openh264enc
+		// gstreamer1.0-plugins-bad
+		// openh264enc multi-thread=4 complexity=high bitrate=3072000 max-bitrate=4096000
+		if err := CheckPlugins([]string{"ximagesrc"}); err != nil {
+			return nil, err
+		}
+
+		clockRate = videoClockRate
+
+		if pipelineSrc != "" {
+			pipelineStr = fmt.Sprintf(pipelineSrc+pipelineStr, pipelineDevice)
+		} else {
+			pipelineStr = fmt.Sprintf(videoSrc+"openh264enc multi-thread=4 complexity=high bitrate=3072000 max-bitrate=4096000 ! video/x-h264,stream-format=byte-stream"+pipelineStr, pipelineDevice)
+
+			// https://gstreamer.freedesktop.org/documentation/x264/index.html?gi-language=c
+			// gstreamer1.0-plugins-ugly
+			// video/x-raw,format=I420 ! x264enc bframes=0 key-int-max=60 byte-stream=true tune=zerolatency speed-preset=veryfast ! video/x-h264,stream-format=byte-stream
+			if err := CheckPlugins([]string{"openh264"}); err != nil {
+				pipelineStr = fmt.Sprintf(videoSrc+"video/x-raw,format=I420 ! x264enc bframes=0 key-int-max=60 byte-stream=true tune=zerolatency speed-preset=veryfast ! video/x-h264,stream-format=byte-stream"+pipelineStr, pipelineDevice)
+
+				if err := CheckPlugins([]string{"x264"}); err != nil {
+					return nil, err
+				}
+			}
+		}
+	case webrtc.Opus:
+		// https://gstreamer.freedesktop.org/documentation/opus/opusenc.html
+		// gstreamer1.0-plugins-base
+		// opusenc
+		if err := CheckPlugins([]string{"pulseaudio", "opus"}); err != nil {
+			return nil, err
+		}
+
+		clockRate = audioClockRate
+
+		if pipelineSrc != "" {
+			pipelineStr = fmt.Sprintf(pipelineSrc+pipelineStr, pipelineDevice)
+		} else {
+			pipelineStr = fmt.Sprintf(audioSrc+"opusenc"+pipelineStr, pipelineDevice)
+		}
+	case webrtc.G722:
+		// https://gstreamer.freedesktop.org/documentation/libav/avenc_g722.html?gi-language=c
+		// gstreamer1.0-libav
+		// avenc_g722
+		if err := CheckPlugins([]string{"pulseaudio", "libav"}); err != nil {
+			return nil, err
+		}
+
+		clockRate = audioClockRate
+
+		if pipelineSrc != "" {
+			pipelineStr = fmt.Sprintf(pipelineSrc+pipelineStr, pipelineDevice)
+		} else {
+			pipelineStr = fmt.Sprintf(audioSrc+"avenc_g722"+pipelineStr, pipelineDevice)
+		}
+	case webrtc.PCMU:
+		// https://gstreamer.freedesktop.org/documentation/mulaw/mulawenc.html?gi-language=c
+		// gstreamer1.0-plugins-good
+		// audio/x-raw, rate=8000 ! mulawenc
+		if err := CheckPlugins([]string{"pulseaudio", "mulaw"}); err != nil {
+			return nil, err
+		}
+
+		clockRate = pcmClockRate
+
+		if pipelineSrc != "" {
+			pipelineStr = fmt.Sprintf(pipelineSrc+pipelineStr, pipelineDevice)
+		} else {
+			pipelineStr = fmt.Sprintf(audioSrc+"audio/x-raw, rate=8000 ! mulawenc"+pipelineStr, pipelineDevice)
+		}
+	case webrtc.PCMA:
+		// https://gstreamer.freedesktop.org/documentation/alaw/alawenc.html?gi-language=c
+		// gstreamer1.0-plugins-good
+		// audio/x-raw, rate=8000 ! alawenc
+		if err := CheckPlugins([]string{"pulseaudio", "alaw"}); err != nil {
+			return nil, err
+		}
+
+		clockRate = pcmClockRate
+
+		if pipelineSrc != "" {
+			pipelineStr = fmt.Sprintf(pipelineSrc+pipelineStr, pipelineDevice)
+		} else {
+			pipelineStr = fmt.Sprintf(audioSrc+"audio/x-raw, rate=8000 ! alawenc"+pipelineStr, pipelineDevice)
+		}
+	default:
+		return nil, fmt.Errorf("unknown codec %s", codecName)
+	}
+
+	return CreatePipeline(pipelineStr, codecName, clockRate)
+}
+
+// CreatePipeline creates a GStreamer Pipeline
+func CreatePipeline(pipelineStr string, codecName string, clockRate float32) (*Pipeline, error) {
+	pipelineStrUnsafe := C.CString(pipelineStr)
+	defer C.free(unsafe.Pointer(pipelineStrUnsafe))
+
+	pipelinesLock.Lock()
+	defer pipelinesLock.Unlock()
+
+	p := &Pipeline{
+		Pipeline:  C.gstreamer_send_create_pipeline(pipelineStrUnsafe),
+		Sample:    make(chan types.Sample),
+		CodecName: codecName,
+		ClockRate: clockRate,
+		Src:       pipelineStr,
+		id:        len(pipelines),
+	}
+
+	pipelines[p.id] = p
+	return p, nil
+}
+
+// Destroy GStreamer Pipeline
+func (p *Pipeline) DestroyPipeline() {
+	C.gstreamer_send_destroy_pipeline(p.Pipeline)
+}
+
+// Start starts the GStreamer Pipeline
+func (p *Pipeline) Start() {
+	C.gstreamer_send_start_pipeline(p.Pipeline, C.int(p.id))
+}
+
+// Play starts the GStreamer Pipeline
+func (p *Pipeline) Play() {
+	C.gstreamer_send_play_pipeline(p.Pipeline)
+}
+
+// Stop stops the GStreamer Pipeline
+func (p *Pipeline) Stop() {
+	C.gstreamer_send_stop_pipeline(p.Pipeline)
+}
+
+// gst-inspect-1.0
+func CheckPlugins(plugins []string) error {
+	var plugin *C.GstPlugin
+	for _, pluginstr := range plugins {
+		plugincstr := C.CString(pluginstr)
+		plugin = C.gst_registry_find_plugin(registry, plugincstr)
+		C.free(unsafe.Pointer(plugincstr))
+		if plugin == nil {
+			return fmt.Errorf("required gstreamer plugin %s not found", pluginstr)
+		}
+	}
+
+	return nil
+}
+
+//export goHandlePipelineBuffer
+func goHandlePipelineBuffer(buffer unsafe.Pointer, bufferLen C.int, duration C.int, pipelineID C.int) {
+	pipelinesLock.Lock()
+	pipeline, ok := pipelines[int(pipelineID)]
+	pipelinesLock.Unlock()
+
+	if ok {
+		samples := uint32(pipeline.ClockRate * (float32(duration) / 1000000000))
+		pipeline.Sample <- types.Sample{Data: C.GoBytes(buffer, bufferLen), Samples: samples}
+	} else {
+		fmt.Printf("discarding buffer, no pipeline with id %d", int(pipelineID))
+	}
+	C.free(buffer)
+}

internal/capture/gst/gst.h

@@ -0,0 +1,20 @@
+#ifndef GST_H
+#define GST_H
+
+#include <glib.h>
+#include <gst/gst.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+extern void goHandlePipelineBuffer(void *buffer, int bufferLen, int samples, int pipelineId);
+
+GstElement *gstreamer_send_create_pipeline(char *pipeline);
+void gstreamer_send_destroy_pipeline(GstElement *pipeline);
+
+void gstreamer_send_start_pipeline(GstElement *pipeline, int pipelineId);
+void gstreamer_send_play_pipeline(GstElement *pipeline);
+void gstreamer_send_stop_pipeline(GstElement *pipeline);
+void gstreamer_send_start_mainloop(void);
+void gstreamer_init(void);
+
+#endif

internal/capture/manager.go

@@ -0,0 +1,175 @@
+package capture
+
+import (
+	"fmt"
+
+	"github.com/kataras/go-events"
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+
+	"demodesk/neko/internal/types"
+	"demodesk/neko/internal/config"
+	"demodesk/neko/internal/capture/gst"
+)
+
+type CaptureManagerCtx struct {
+	logger        zerolog.Logger
+	video         *gst.Pipeline
+	audio         *gst.Pipeline
+	broadcast     *gst.Pipeline
+	config        *config.Capture
+	shutdown      chan bool
+	emmiter       events.EventEmmiter
+	streaming     bool
+	broadcast_url string
+	desktop       types.DesktopManager
+}
+
+func New(desktop types.DesktopManager, config *config.Capture) *CaptureManagerCtx {
+	return &CaptureManagerCtx{
+		logger:        log.With().Str("module", "capture").Logger(),
+		shutdown:      make(chan bool),
+		emmiter:       events.New(),
+		config:        config,
+		streaming:     false,
+		broadcast_url: "",
+		desktop:       desktop,
+	}
+}
+
+func (manager *CaptureManagerCtx) Start() {
+	manager.logger.Info().
+		Str("screen_resolution", fmt.Sprintf("%dx%d@%d", manager.config.ScreenWidth, manager.config.ScreenHeight, manager.config.ScreenRate)).
+		Msgf("Setting screen resolution...")
+
+	if err := manager.desktop.ChangeScreenSize(manager.config.ScreenWidth, manager.config.ScreenHeight, manager.config.ScreenRate); err != nil {
+		manager.logger.Warn().Err(err).Msg("unable to change screen size")
+	}
+
+	manager.CreateVideoPipeline()
+	manager.CreateAudioPipeline()
+	manager.StartBroadcastPipeline()
+
+	go func() {
+		defer func() {
+			manager.logger.Info().Msg("shutdown")
+		}()
+
+		for {
+			select {
+			case <-manager.shutdown:
+				return
+			case sample := <-manager.video.Sample:
+				manager.emmiter.Emit("video", sample)
+			case sample := <-manager.audio.Sample:
+				manager.emmiter.Emit("audio", sample)
+			}
+		}
+	}()
+}
+
+func (manager *CaptureManagerCtx) Shutdown() error {
+	manager.logger.Info().Msgf("capture shutting down")
+	manager.video.DestroyPipeline()
+	manager.audio.DestroyPipeline()
+	manager.StopBroadcastPipeline()
+
+	manager.shutdown <- true
+	return nil
+}
+
+func (manager *CaptureManagerCtx) VideoCodec() string {
+	return manager.config.VideoCodec
+}
+
+func (manager *CaptureManagerCtx) AudioCodec() string {
+	return manager.config.AudioCodec
+}
+
+func (manager *CaptureManagerCtx) OnVideoFrame(listener func(sample types.Sample)) {
+	manager.emmiter.On("video", func(payload ...interface{}) {
+		listener(payload[0].(types.Sample))
+	})
+}
+
+func (manager *CaptureManagerCtx) OnAudioFrame(listener func(sample types.Sample)) {
+	manager.emmiter.On("audio", func(payload ...interface{}) {
+		listener(payload[0].(types.Sample))
+	})
+}
+
+func (manager *CaptureManagerCtx) StartStream() {
+	manager.logger.Info().Msgf("Pipelines starting...")
+
+	manager.video.Start()
+	manager.audio.Start()
+	manager.streaming = true
+}
+
+func (manager *CaptureManagerCtx) StopStream() {
+	manager.logger.Info().Msgf("Pipelines shutting down...")
+
+	manager.video.Stop()
+	manager.audio.Stop()
+	manager.streaming = false
+}
+
+func (manager *CaptureManagerCtx) Streaming() bool {
+	return manager.streaming
+}
+
+func (manager *CaptureManagerCtx) CreateVideoPipeline() {
+	var err error
+
+	manager.logger.Info().
+		Str("video_codec", manager.config.VideoCodec).
+		Str("video_display", manager.config.Display).
+		Str("video_params", manager.config.VideoParams).
+		Msgf("Creating video pipeline...")
+
+	manager.video, err = gst.CreateAppPipeline(
+		manager.config.VideoCodec,
+		manager.config.Display,
+		manager.config.VideoParams,
+	)
+
+	if err != nil {
+		manager.logger.Panic().Err(err).Msg("unable to create video pipeline")
+	}
+}
+
+func (manager *CaptureManagerCtx) CreateAudioPipeline() {
+	var err error
+
+	manager.logger.Info().
+		Str("audio_codec", manager.config.AudioCodec).
+		Str("audio_display", manager.config.Device).
+		Str("audio_params", manager.config.AudioParams).
+		Msgf("Creating audio pipeline...")
+
+	manager.audio, err = gst.CreateAppPipeline(
+		manager.config.AudioCodec,
+		manager.config.Device,
+		manager.config.AudioParams,
+	)
+
+	if err != nil {
+		manager.logger.Panic().Err(err).Msg("unable to create audio pipeline")
+	}
+}
+
+func (manager *CaptureManagerCtx) ChangeResolution(width int, height int, rate int) error {
+	manager.video.DestroyPipeline()
+	manager.StopBroadcastPipeline()
+
+	defer func() {
+		manager.CreateVideoPipeline()
+	
+		manager.video.Start()
+		manager.logger.Info().Msg("starting video pipeline...")
+	
+		manager.StartBroadcastPipeline()
+	}()
+	
+	return manager.desktop.ChangeScreenSize(width, height, rate)
+}