feat: add PipeWire screen and system audio capture (M2)
Capture the Wayland desktop via xdg-desktop-portal + PipeWire using go2tv.app/screencast (MIT), and stream it to OBS: - internal/capture: Capture/FrameSource/AudioSource interfaces and the PipeWire backend (BGRA frames at monitor resolution, S16 48 kHz stereo system audio) - protocol: EncodeBGRA fast path producing 4:2:0 YCbCr JPEGs - cmd: --source screen|pattern, --audio, --stream-index flags; real capture feeds the existing sender - share one wall-clock reference between the audio and video loops so OBS receives aligned A/V timestamps (avoids multi-second latency) Verified end-to-end: real desktop at 30 fps renders in OBS with sub-second latency.
This commit is contained in:
@@ -0,0 +1,85 @@
|
||||
// Package capture abstracts the screen and audio sources that feed the
|
||||
// Teleport sender. Implementations are platform-specific; the PipeWire
|
||||
// backend (pipewire.go) handles Wayland via xdg-desktop-portal.
|
||||
//
|
||||
// The core only knows about two things: a stream of video frames and a
|
||||
// stream of audio samples. Everything else (formats, portal negotiation)
|
||||
// stays behind this interface so the sender can later run headless or be
|
||||
// driven by a GUI without coupling.
|
||||
package capture
|
||||
|
||||
import (
|
||||
"io"
|
||||
"time"
|
||||
)
|
||||
|
||||
// VideoFrame is one captured screen frame. Pix holds BGRA (blue, green,
|
||||
// red, alpha) bytes in row-major order; Stride is the byte offset between
|
||||
// consecutive rows.
|
||||
type VideoFrame struct {
|
||||
Pix []byte
|
||||
Width int
|
||||
Height int
|
||||
Stride int
|
||||
}
|
||||
|
||||
// Capture is the combined screen + audio source. Close releases the
|
||||
// underlying capture session (and portal resources).
|
||||
type Capture interface {
|
||||
// Video returns the frame source. Frames arrive at the compositor's
|
||||
// refresh rate and are consumed one at a time via NextFrame.
|
||||
Video() FrameSource
|
||||
// Audio returns the audio source, or nil if audio capture is disabled
|
||||
// or unavailable.
|
||||
Audio() AudioSource
|
||||
io.Closer
|
||||
}
|
||||
|
||||
// FrameSource yields consecutive captured video frames.
|
||||
type FrameSource interface {
|
||||
// NextFrame blocks until the next frame is available and returns it.
|
||||
NextFrame() (*VideoFrame, error)
|
||||
}
|
||||
|
||||
// AudioSource yields raw interleaved PCM samples (signed 16-bit
|
||||
// little-endian, 48 kHz, stereo) read from the system's default output.
|
||||
type AudioSource interface {
|
||||
io.ReadCloser
|
||||
}
|
||||
|
||||
// ErrNoAudio is returned when the underlying backend cannot provide system
|
||||
// audio capture (e.g. sandboxed Flatpak without a direct PipeWire link).
|
||||
var ErrNoAudio = &AudioUnavailableError{}
|
||||
|
||||
// AudioUnavailableError signals that audio capture could not be started.
|
||||
type AudioUnavailableError struct{}
|
||||
|
||||
func (e *AudioUnavailableError) Error() string {
|
||||
return "capture: system audio unavailable"
|
||||
}
|
||||
|
||||
// silenceStep is the pacing interval between silence chunk reads.
|
||||
const silenceStep = 10 * time.Millisecond
|
||||
|
||||
// SilenceSource yields a continuous stream of digital silence, paced like a
|
||||
// real audio capture. It keeps OBS's audio pipeline alive when no system
|
||||
// audio is available or a synthetic source is in use.
|
||||
type silenceSource struct{}
|
||||
|
||||
// NewSilenceSource creates a silence-generating audio source.
|
||||
func NewSilenceSource() AudioSource {
|
||||
return &silenceSource{}
|
||||
}
|
||||
|
||||
func (s *silenceSource) Read(p []byte) (int, error) {
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
// Emit a chunk of silence at a real-audio cadence. The reader is 1:1
|
||||
// stereo S16, so 48000 * 10ms * 2ch * 2 bytes = 1920 bytes per read.
|
||||
clear(p)
|
||||
time.Sleep(silenceStep)
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func (s *silenceSource) Close() error { return nil }
|
||||
@@ -0,0 +1,84 @@
|
||||
// Package capture provides the PipeWire backend for Wayland screen + audio
|
||||
// capture.
|
||||
//
|
||||
// This uses go2tv.app/screencast (MIT) which implements the full
|
||||
// xdg-desktop-portal ScreenCast negotiation and then receives frames over
|
||||
// PipeWire. The portal session is responsible for the screen-share consent
|
||||
// dialog presented by the compositor (Hyprland in our dev environment).
|
||||
//
|
||||
// Frames arrive as raw BGRA at the monitor's native resolution/refresh rate.
|
||||
// Audio is signed 16-bit, 48 kHz, stereo interleaved PCM from the system's
|
||||
// default output.
|
||||
package capture
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
|
||||
"go2tv.app/screencast/capture"
|
||||
)
|
||||
|
||||
// PipeWire implements Capture on top of the screencast library.
|
||||
type PipeWire struct {
|
||||
stream *capture.Stream
|
||||
}
|
||||
|
||||
// OpenPipeWire opens a PipeWire capture session. streamIndex selects which
|
||||
// monitor to capture when multiple are present. Triggering the portal
|
||||
// consent dialog is expected; the compositor decides whether to show it.
|
||||
func OpenPipeWire(streamIndex int, audio bool) (*PipeWire, error) {
|
||||
s, err := capture.Open(&capture.Options{
|
||||
StreamIndex: streamIndex,
|
||||
IncludeAudio: audio,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &PipeWire{stream: s}, nil
|
||||
}
|
||||
|
||||
// Video returns the BGRA frame source.
|
||||
func (p *PipeWire) Video() FrameSource {
|
||||
return &pipewireVideo{stream: p.stream}
|
||||
}
|
||||
|
||||
// Audio returns the system audio source, or nil if unavailable.
|
||||
func (p *PipeWire) Audio() AudioSource {
|
||||
if p.stream.Audio == nil {
|
||||
return nil
|
||||
}
|
||||
return p.stream.Audio
|
||||
}
|
||||
|
||||
// Close releases the capture session and portal resources.
|
||||
func (p *PipeWire) Close() error {
|
||||
return p.stream.Close()
|
||||
}
|
||||
|
||||
// pipewireVideo adapts the screencast io.ReadCloser into FrameSource.
|
||||
type pipewireVideo struct {
|
||||
stream *capture.Stream
|
||||
frame *VideoFrame
|
||||
}
|
||||
|
||||
// NextFrame blocks until the next full frame is delivered. The screencast
|
||||
// library writes one complete BGRA frame per Read, so we assemble it with
|
||||
// ReadFull and reuse the underlying buffer across calls.
|
||||
func (v *pipewireVideo) NextFrame() (*VideoFrame, error) {
|
||||
w, h := int(v.stream.Width), int(v.stream.Height)
|
||||
if v.frame == nil {
|
||||
v.frame = &VideoFrame{
|
||||
Pix: make([]byte, w*h*4),
|
||||
Width: w,
|
||||
Height: h,
|
||||
Stride: w * 4,
|
||||
}
|
||||
}
|
||||
if _, err := io.ReadFull(v.stream, v.frame.Pix); err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
return nil, io.ErrClosedPipe
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return v.frame, nil
|
||||
}
|
||||
@@ -74,6 +74,47 @@ func (e *JPEGEncoder) Encode(img image.Image, quality int) ([]byte, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// EncodeBGRA compresses a raw BGRA (blue, green, red, alpha) pixel buffer of
|
||||
// the given dimensions. This is the fast path for the PipeWire screen-capture
|
||||
// backend, which delivers frames in BGRA byte order. Subsampling defaults to
|
||||
// 4:2:0 (a YCbCr JPEG), so the OBS receiver decodes it as I420 — the same
|
||||
// layout obs-teleport produces for OBS-native YCbCr frames.
|
||||
func (e *JPEGEncoder) EncodeBGRA(pix []byte, width, height, quality int) ([]byte, error) {
|
||||
if quality < 1 {
|
||||
quality = 1
|
||||
}
|
||||
if quality > 100 {
|
||||
quality = 100
|
||||
}
|
||||
|
||||
want := width * height * 4
|
||||
if len(pix) < want {
|
||||
return nil, errors.New("turbojpeg: BGRA buffer too small")
|
||||
}
|
||||
|
||||
C.tj3Set(e.ctx, C.TJPARAM_QUALITY, C.int(quality))
|
||||
C.tj3Set(e.ctx, C.TJPARAM_SUBSAMP, C.TJSAMP_420)
|
||||
C.tj3Set(e.ctx, C.TJPARAM_COLORSPACE, C.TJCS_YCbCr)
|
||||
|
||||
size := C.tj3JPEGBufSize(C.int(width), C.int(height), C.TJSAMP_420)
|
||||
buf := make([]byte, int(size))
|
||||
|
||||
srcPtr := unsafe.Pointer(&pix[0])
|
||||
dstPtr := (*C.uchar)(&buf[0])
|
||||
|
||||
var pin runtime.Pinner
|
||||
pin.Pin(srcPtr)
|
||||
pin.Pin(dstPtr)
|
||||
defer pin.Unpin()
|
||||
|
||||
jpegSize := size
|
||||
rc := C.tj3Compress8(e.ctx, (*C.uchar)(srcPtr), C.int(width), 0, C.int(height), C.TJPF_BGRA, &dstPtr, &jpegSize)
|
||||
if rc != 0 {
|
||||
return nil, errors.New("turbojpeg BGRA compress failed")
|
||||
}
|
||||
return buf[:int(jpegSize)], nil
|
||||
}
|
||||
|
||||
// encodeRGBA compresses a Go RGBA image (pixel layout [R,G,B,A] per 4 bytes).
|
||||
// TJPF_RGBA tells turbojpeg the exact layout; colourspace is RGB.
|
||||
func (e *JPEGEncoder) encodeRGBA(img *image.RGBA) ([]byte, error) {
|
||||
|
||||
@@ -120,3 +120,54 @@ func TestJPEGEncodeQualityClamp(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestJPEGEncodeBGRA checks the raw-BGRA fast path produces a decodable
|
||||
// JPEG with correct SOI/EOI markers.
|
||||
func TestJPEGEncodeBGRA(t *testing.T) {
|
||||
enc, err := NewJPEGEncoder()
|
||||
if err != nil {
|
||||
t.Fatalf("NewJPEGEncoder: %v", err)
|
||||
}
|
||||
defer enc.Close()
|
||||
|
||||
// 16x8 BGRA: top half red (R,G,B=255,0,0), bottom half blue (0,0,255).
|
||||
w, h := 16, 8
|
||||
pix := make([]byte, w*h*4)
|
||||
for y := 0; y < h; y++ {
|
||||
for x := 0; x < w; x++ {
|
||||
off := (y*w + x) * 4
|
||||
if y < h/2 {
|
||||
pix[off], pix[off+1], pix[off+2], pix[off+3] = 0, 0, 255, 255 // B,G,R,A = blue,red
|
||||
} else {
|
||||
pix[off], pix[off+1], pix[off+2], pix[off+3] = 255, 0, 0, 255 // B,G,R,A = red,blue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
buf, err := enc.EncodeBGRA(pix, w, h, 85)
|
||||
if err != nil {
|
||||
t.Fatalf("EncodeBGRA: %v", err)
|
||||
}
|
||||
if len(buf) == 0 {
|
||||
t.Fatal("empty JPEG output")
|
||||
}
|
||||
if !bytes.Equal(buf[:2], []byte{0xFF, 0xD8}) {
|
||||
t.Errorf("bad JPEG SOI marker: %x", buf[:2])
|
||||
}
|
||||
if !bytes.Equal(buf[len(buf)-2:], []byte{0xFF, 0xD9}) {
|
||||
t.Errorf("bad JPEG EOI marker")
|
||||
}
|
||||
}
|
||||
|
||||
// TestJPEGEncodeBGRAShort rejects truncated pixel buffers.
|
||||
func TestJPEGEncodeBGRAShort(t *testing.T) {
|
||||
enc, err := NewJPEGEncoder()
|
||||
if err != nil {
|
||||
t.Fatalf("NewJPEGEncoder: %v", err)
|
||||
}
|
||||
defer enc.Close()
|
||||
|
||||
if _, err := enc.EncodeBGRA(make([]byte, 10), 16, 8, 80); err == nil {
|
||||
t.Error("short BGRA buffer accepted")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user