Skip to content

Commit 0cd3a36

Browse files
committed
Add Wayland screencopy video capture
1 parent 148bc06 commit 0cd3a36

7 files changed

Lines changed: 277 additions & 6 deletions

File tree

server/internal/capture/manager.go

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,9 @@ func New(desktop types.DesktopManager, config *config.Capture) *CaptureManagerCt
3939

4040
createPipeline := func() (string, error) {
4141
if pipelineConf.GstPipeline != "" {
42+
if config.Wayland {
43+
return "", errors.New("custom video pipelines are not supported with Wayland capture")
44+
}
4245
// replace {display} with valid display
4346
return strings.Replace(pipelineConf.GstPipeline, "{display}", config.Display, 1), nil
4447
}
@@ -49,6 +52,18 @@ func New(desktop types.DesktopManager, config *config.Capture) *CaptureManagerCt
4952
return "", err
5053
}
5154

55+
if config.Wayland {
56+
fps := screen.Rate
57+
if fps <= 0 {
58+
fps = 25
59+
}
60+
return fmt.Sprintf(
61+
"appsrc name=appsrc is-live=true format=time do-timestamp=true "+
62+
"caps=video/x-raw,format=BGRx,width=%d,height=%d,framerate=%d/1 "+
63+
"%s ! appsink name=appsink", screen.Width, screen.Height, fps, pipeline,
64+
), nil
65+
}
66+
5267
return fmt.Sprintf(
5368
"ximagesrc display-name=%s show-pointer=%v use-damage=false "+
5469
"%s ! appsink name=appsink", config.Display, pipelineConf.ShowPointer, pipeline,
@@ -69,7 +84,13 @@ func New(desktop types.DesktopManager, config *config.Capture) *CaptureManagerCt
6984
Msg("syntax check for video stream pipeline passed")
7085

7186
// append to videos
72-
videos[video_id] = streamSinkNew(config.VideoCodec, createPipeline, video_id)
87+
video := streamSinkNew(config.VideoCodec, createPipeline, video_id)
88+
if config.Wayland {
89+
video.SetFrameSourceFactory(func() (frameSource, error) {
90+
return newWaylandFrameSource(config.WaylandRecorder, desktop.GetScreenSize()), nil
91+
})
92+
}
93+
videos[video_id] = video
7394
}
7495

7596
return &CaptureManagerCtx{

server/internal/capture/streamsink.go

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,12 @@ type StreamSinkManagerCtx struct {
3232
mu sync.Mutex
3333
wg sync.WaitGroup
3434

35-
codec codec.RTPCodec
36-
pipeline gst.Pipeline
37-
pipelineMu sync.Mutex
38-
pipelineFn func() (string, error)
35+
codec codec.RTPCodec
36+
pipeline gst.Pipeline
37+
pipelineMu sync.Mutex
38+
pipelineFn func() (string, error)
39+
frameSourceFn func() (frameSource, error)
40+
frameSource frameSource
3941

4042
listeners map[uintptr]types.SampleListener
4143
listenersKf map[uintptr]types.SampleListener // keyframe lobby
@@ -142,6 +144,10 @@ func (manager *StreamSinkManagerCtx) ID() string {
142144
return manager.id
143145
}
144146

147+
func (manager *StreamSinkManagerCtx) SetFrameSourceFactory(factory func() (frameSource, error)) {
148+
manager.frameSourceFn = factory
149+
}
150+
145151
func (manager *StreamSinkManagerCtx) Bitrate() uint64 {
146152
manager.listenersMu.Lock()
147153
defer manager.listenersMu.Unlock()
@@ -325,9 +331,27 @@ func (manager *StreamSinkManagerCtx) CreatePipeline() error {
325331
return err
326332
}
327333

334+
if manager.frameSourceFn != nil {
335+
manager.frameSource, err = manager.frameSourceFn()
336+
if err != nil {
337+
manager.pipeline.Destroy()
338+
manager.pipeline = nil
339+
return err
340+
}
341+
manager.pipeline.AttachAppsrc("appsrc")
342+
}
328343
manager.pipeline.AttachAppsink("appsink")
329344
manager.pipeline.Play()
330345

346+
if manager.frameSource != nil {
347+
if err := manager.frameSource.Start(manager.pipeline.Push); err != nil {
348+
manager.pipeline.Destroy()
349+
manager.pipeline = nil
350+
manager.frameSource = nil
351+
return err
352+
}
353+
}
354+
331355
manager.wg.Add(1)
332356
pipeline := manager.pipeline
333357

@@ -405,6 +429,10 @@ func (manager *StreamSinkManagerCtx) DestroyPipeline() {
405429
return
406430
}
407431

432+
if manager.frameSource != nil {
433+
manager.frameSource.Stop()
434+
manager.frameSource = nil
435+
}
408436
manager.pipeline.Destroy()
409437
manager.logger.Info().Msgf("destroying pipeline")
410438
manager.pipeline = nil

server/internal/capture/wayland.go

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
package capture
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"io"
7+
"os"
8+
"os/exec"
9+
"strconv"
10+
"sync"
11+
12+
"github.com/rs/zerolog/log"
13+
14+
"github.com/m1k1o/neko/server/pkg/types"
15+
)
16+
17+
type frameSource interface {
18+
Start(func([]byte)) error
19+
Stop()
20+
}
21+
22+
type waylandFrameSource struct {
23+
recorder string
24+
width int
25+
height int
26+
fps int
27+
28+
mu sync.Mutex
29+
cancel context.CancelFunc
30+
done chan struct{}
31+
}
32+
33+
func newWaylandFrameSource(recorder string, screen types.ScreenSize) *waylandFrameSource {
34+
fps := int(screen.Rate)
35+
if fps <= 0 {
36+
fps = 25
37+
}
38+
39+
return &waylandFrameSource{
40+
recorder: recorder,
41+
width: screen.Width,
42+
height: screen.Height,
43+
fps: fps,
44+
}
45+
}
46+
47+
func (source *waylandFrameSource) frameSize() int {
48+
return source.width * source.height * 4
49+
}
50+
51+
func (source *waylandFrameSource) args() []string {
52+
return []string{
53+
"--no-damage",
54+
"--no-dmabuf",
55+
"--framerate", strconv.Itoa(source.fps),
56+
"--muxer", "rawvideo",
57+
"--codec", "rawvideo",
58+
"--pixel-format", "bgr0",
59+
"--file", "/dev/stdout",
60+
"--overwrite",
61+
}
62+
}
63+
64+
func (source *waylandFrameSource) command() *exec.Cmd {
65+
return exec.Command(source.recorder, source.args()...)
66+
}
67+
68+
func (source *waylandFrameSource) Start(push func([]byte)) error {
69+
if push == nil {
70+
return fmt.Errorf("frame push callback is required")
71+
}
72+
if source.recorder == "" {
73+
return fmt.Errorf("Wayland recorder executable is required")
74+
}
75+
if source.width <= 0 || source.height <= 0 {
76+
return fmt.Errorf("invalid Wayland output size: %dx%d", source.width, source.height)
77+
}
78+
79+
ctx, cancel := context.WithCancel(context.Background())
80+
cmd := exec.CommandContext(ctx, source.recorder, source.args()...)
81+
cmd.Stderr = os.Stderr
82+
83+
stdout, err := cmd.StdoutPipe()
84+
if err != nil {
85+
cancel()
86+
return fmt.Errorf("create Wayland recorder pipe: %w", err)
87+
}
88+
if err := cmd.Start(); err != nil {
89+
cancel()
90+
return fmt.Errorf("start Wayland recorder: %w", err)
91+
}
92+
93+
done := make(chan struct{})
94+
source.mu.Lock()
95+
source.cancel = cancel
96+
source.done = done
97+
source.mu.Unlock()
98+
99+
go func() {
100+
defer close(done)
101+
defer stdout.Close()
102+
103+
frame := make([]byte, source.frameSize())
104+
for {
105+
if _, err := io.ReadFull(stdout, frame); err != nil {
106+
if err != io.EOF && err != io.ErrUnexpectedEOF {
107+
log.Warn().Err(err).Msg("Wayland recorder stopped while reading a frame")
108+
}
109+
break
110+
}
111+
112+
push(frame)
113+
}
114+
115+
if err := cmd.Wait(); err != nil && ctx.Err() == nil {
116+
log.Warn().Err(err).Msg("Wayland recorder exited")
117+
}
118+
}()
119+
120+
return nil
121+
}
122+
123+
func (source *waylandFrameSource) Stop() {
124+
source.mu.Lock()
125+
cancel := source.cancel
126+
done := source.done
127+
source.cancel = nil
128+
source.done = nil
129+
source.mu.Unlock()
130+
131+
if cancel == nil {
132+
return
133+
}
134+
cancel()
135+
<-done
136+
}
Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
package capture
2+
3+
import (
4+
"reflect"
5+
"testing"
6+
7+
"github.com/m1k1o/neko/server/pkg/types"
8+
)
9+
10+
func TestWaylandFrameSourceCommand(t *testing.T) {
11+
source := newWaylandFrameSource("wf-recorder", types.ScreenSize{
12+
Width: 1920,
13+
Height: 1080,
14+
Rate: 25,
15+
})
16+
17+
got := source.command().Args
18+
want := []string{
19+
"wf-recorder",
20+
"--no-damage",
21+
"--no-dmabuf",
22+
"--framerate", "25",
23+
"--muxer", "rawvideo",
24+
"--codec", "rawvideo",
25+
"--pixel-format", "bgr0",
26+
"--file", "/dev/stdout",
27+
"--overwrite",
28+
}
29+
if !reflect.DeepEqual(got, want) {
30+
t.Fatalf("command args = %#v, want %#v", got, want)
31+
}
32+
}
33+
34+
func TestWaylandFrameSourceDefaultsFrameRate(t *testing.T) {
35+
source := newWaylandFrameSource("wf-recorder", types.ScreenSize{
36+
Width: 10,
37+
Height: 20,
38+
})
39+
40+
if source.fps != 25 {
41+
t.Fatalf("fps = %d, want 25", source.fps)
42+
}
43+
if source.frameSize() != 800 {
44+
t.Fatalf("frame size = %d, want 800", source.frameSize())
45+
}
46+
}

server/internal/config/capture.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@ const (
2828
type Capture struct {
2929
Display string
3030

31+
Wayland bool
32+
WaylandRecorder string
33+
3134
VideoCodec codec.RTPCodec
3235
VideoIDs []string
3336
VideoPipelines map[string]types.VideoConfig
@@ -80,6 +83,16 @@ func (Capture) Init(cmd *cobra.Command) error {
8083
return err
8184
}
8285

86+
cmd.PersistentFlags().Bool("capture.video.wayland", false, "capture a Wayland compositor output")
87+
if err := viper.BindPFlag("capture.video.wayland", cmd.PersistentFlags().Lookup("capture.video.wayland")); err != nil {
88+
return err
89+
}
90+
91+
cmd.PersistentFlags().String("capture.video.wayland_recorder", "wf-recorder", "Wayland screencopy recorder executable")
92+
if err := viper.BindPFlag("capture.video.wayland_recorder", cmd.PersistentFlags().Lookup("capture.video.wayland_recorder")); err != nil {
93+
return err
94+
}
95+
8396
cmd.PersistentFlags().String("capture.video.codec", "vp8", "video codec to be used")
8497
if err := viper.BindPFlag("capture.video.codec", cmd.PersistentFlags().Lookup("capture.video.codec")); err != nil {
8598
return err
@@ -326,6 +339,9 @@ func (s *Capture) Set() {
326339
s.Display = os.Getenv("DISPLAY")
327340
}
328341

342+
s.Wayland = viper.GetBool("capture.video.wayland")
343+
s.WaylandRecorder = viper.GetString("capture.video.wayland_recorder")
344+
329345
// video
330346
videoCodec := viper.GetString("capture.video.codec")
331347
s.VideoCodec, ok = codec.ParseStr(videoCodec)

webpage/docs/configuration/capture.md

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,14 +31,18 @@ The Gstreamer pipeline is started when the first client requests the video strea
3131

3232
<ConfigurationTab options={configOptions} filter={[
3333
"capture.video.display",
34+
"capture.video.wayland",
35+
"capture.video.wayland_recorder",
3436
"capture.video.codec",
3537
"capture.video.ids",
3638
"capture.video.pipeline",
3739
"capture.video.pipelines",
3840
]} comments={false} />
3941

4042
- <Def id="video.display" /> is the name of the [X display](https://www.x.org/wiki/) that you want to capture. If not specified, the environment variable `DISPLAY` will be used.
41-
- <Def id="video.codec" /> available codecs are `vp8`, `vp9`, `av1`, `h264`. [Supported video codecs](https://developer.mozilla.org/en-US/docs/Web/Media/Guides/Formats/WebRTC_codecs#supported_video_codecs) are dependent on the WebRTC implementation used by the client, `vp8` and `h264` are supported by all WebRTC implementations.
43+
- <Def id="video.wayland" /> switches the video source from `ximagesrc` to a `wf-recorder` process using the compositor's `wlr-screencopy-unstable-v1` protocol. It requires a Wayland compositor that exposes that protocol and an executable <Def id="video.wayland_recorder" />. Custom `gst_pipeline` values are not supported in this mode.
44+
- <Def id="video.wayland_recorder" /> is the executable used to produce raw `BGRx` frames on stdout. The default is `wf-recorder`.
45+
- <Def id="video.codec" /> available codecs are `vp8`, `vp9`, `av1`, `h264`. [Supported video codecs](https://developer.mozilla.org/en-US/docs/Web/Media/Formats/WebRTC_codecs#supported_video_codecs) are dependent on the WebRTC implementation used by the client, `vp8` and `h264` are supported by all WebRTC implementations.
4246
- <Def id="video.ids" /> is a list of pipeline ids that are defined in the <Opt id="video.pipelines" /> section. The first pipeline in the list will be the default pipeline.
4347
- <Def id="video.pipeline" /> is a shorthand for defining [Gstreamer pipeline description](#video.gst_pipeline) for a single pipeline. This is option is ignored if <Opt id="video.pipelines" /> is defined.
4448
- <Def id="video.pipelines" /> is a dictionary of pipeline configurations. Each pipeline configuration is defined by a unique pipeline id. They can be defined in two ways: either by building the pipeline dynamically using [Expression-Driven Configuration](#video.expression) or by defining the pipeline using a [Gstreamer Pipeline Description](#video.gst_pipeline).

webpage/docs/configuration/help.json

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,26 @@
164164
"type": "string",
165165
"description": "X display to capture"
166166
},
167+
{
168+
"key": [
169+
"capture",
170+
"video",
171+
"wayland"
172+
],
173+
"type": "boolean",
174+
"defaultValue": false,
175+
"description": "capture a Wayland compositor output"
176+
},
177+
{
178+
"key": [
179+
"capture",
180+
"video",
181+
"wayland_recorder"
182+
],
183+
"type": "string",
184+
"defaultValue": "wf-recorder",
185+
"description": "Wayland screencopy recorder executable"
186+
},
167187
{
168188
"key": [
169189
"capture",

0 commit comments

Comments
 (0)