diff --git a/README.md b/README.md index dfe519b..821dba4 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ view inside the editor. ## Features -- WebTransport streaming server in Rust, plus experimental WebRTC for runner previews, using hardware encoded HEVC/H.264 video +- WebTransport streaming server in Rust, plus experimental WebRTC for runner previews, using HEVC/H.264 video or full-resolution JPEG on CI runners - Simulator control & inspection using private accessibility APIs - CoreSimulator chrome asset rendering for device bezels - NativeScript and React Native runtime inspector plugins, plus a native UIKit inspector framework for other apps @@ -56,6 +56,62 @@ simdeck tap 0.5 0.5 --normalized simdeck describe --format agent --max-depth 2 ``` +## Daemon + +Manage the project daemon explicitly when needed: + +```sh +simdeck daemon start +simdeck daemon status +simdeck daemon stop +``` + +`simdeck daemon` manages the normal per-project warm process. For an always-on +daemon that is available after login, use the macOS user service commands: + +```sh +simdeck service on +simdeck service off +``` + +This uses a LaunchAgent, keeps the server bound to localhost by default, and is +best for agents or editor integrations that should be able to open SimDeck +without first starting a project daemon. + +Use software H.264 when macOS screen recording starves the hardware encoder: + +```sh +simdeck daemon start --video-codec h264-software +``` + +On GitHub Actions macOS runners where VideoToolbox hardware encode is not +available, use the experimental full-resolution JPEG data-channel stream: + +```sh +simdeck daemon start --video-codec jpeg +# open http://127.0.0.1:4310?transport=webrtc-data +``` + +For LAN browser access: + +```sh +simdeck ui --bind 0.0.0.0 --advertise-host 192.168.1.50 --open +``` + +Restart the CoreSimulator service layer when `simctl` reports a stale service +version or the live display gets stuck before the first frame: + +```sh +simdeck core-simulator restart +``` + +You can also start or stop the CoreSimulator service layer explicitly: + +```sh +simdeck core-simulator start +simdeck core-simulator shutdown +``` + ## CLI ```sh diff --git a/cli/XCWH264Encoder.m b/cli/XCWH264Encoder.m index 2b65b43..2da242d 100644 --- a/cli/XCWH264Encoder.m +++ b/cli/XCWH264Encoder.m @@ -1,6 +1,7 @@ #import "XCWH264Encoder.h" #import +#import #import #import #import @@ -15,6 +16,7 @@ typedef NS_ENUM(NSUInteger, XCWVideoEncoderMode) { XCWVideoEncoderModeHEVCHardware, XCWVideoEncoderModeH264Hardware, XCWVideoEncoderModeH264Software, + XCWVideoEncoderModeJPEG, }; static XCWVideoEncoderMode XCWVideoEncoderModeFromEnvironment(void) { @@ -25,6 +27,9 @@ static XCWVideoEncoderMode XCWVideoEncoderModeFromEnvironment(void) { if ([value isEqualToString:@"h264-software"] || [value isEqualToString:@"software-h264"]) { return XCWVideoEncoderModeH264Software; } + if ([value isEqualToString:@"jpeg"] || [value isEqualToString:@"jpg"] || [value isEqualToString:@"mjpeg"]) { + return XCWVideoEncoderModeJPEG; + } return XCWVideoEncoderModeHEVCHardware; } @@ -33,6 +38,8 @@ static CMVideoCodecType XCWVideoCodecTypeForMode(XCWVideoEncoderMode mode) { case XCWVideoEncoderModeH264Hardware: case XCWVideoEncoderModeH264Software: return kCMVideoCodecType_H264; + case XCWVideoEncoderModeJPEG: + return kCMVideoCodecType_JPEG; case XCWVideoEncoderModeHEVCHardware: default: return kCMVideoCodecType_HEVC; @@ -45,6 +52,8 @@ static CMVideoCodecType XCWVideoCodecTypeForMode(XCWVideoEncoderMode mode) { return @"h264"; case XCWVideoEncoderModeH264Software: return @"h264-software"; + case XCWVideoEncoderModeJPEG: + return @"jpeg"; case XCWVideoEncoderModeHEVCHardware: default: return @"hevc"; @@ -57,6 +66,8 @@ static CMVideoCodecType XCWVideoCodecTypeForMode(XCWVideoEncoderMode mode) { return nil; case XCWVideoEncoderModeH264Software: return @"com.apple.videotoolbox.videoencoder.h264"; + case XCWVideoEncoderModeJPEG: + return nil; case XCWVideoEncoderModeHEVCHardware: default: return nil; @@ -173,11 +184,106 @@ static uint32_t XCWReverseBits32(uint32_t value) { return @"hevc"; case kCMVideoCodecType_H264: return @"h264"; + case kCMVideoCodecType_JPEG: + return @"jpeg"; default: return [NSString stringWithFormat:@"0x%08x", (unsigned int)codecType]; } } +static CGFloat XCWJPEGQualityFromEnvironment(void) { + NSString *value = [[NSProcessInfo processInfo] environment][@"SIMDECK_JPEG_QUALITY"]; + double quality = value.length > 0 ? value.doubleValue : 1.0; + if (!isfinite(quality) || quality < 0.1 || quality > 1.0) { + return 1.0; + } + return (CGFloat)quality; +} + +static CGColorSpaceRef XCWDeviceRGBColorSpace(void) { + static CGColorSpaceRef colorSpace = NULL; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + colorSpace = CGColorSpaceCreateDeviceRGB(); + }); + return colorSpace; +} + +static NSData *XCWJPEGDataFromPixelBuffer(CVPixelBufferRef pixelBuffer) { + if (pixelBuffer == NULL) { + return nil; + } + + CGImageRef image = NULL; + BOOL didLockPixelBuffer = NO; + OSType pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer); + if (pixelFormat == kCVPixelFormatType_32BGRA && + CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly) == kCVReturnSuccess) { + didLockPixelBuffer = YES; + void *baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer); + size_t width = CVPixelBufferGetWidth(pixelBuffer); + size_t height = CVPixelBufferGetHeight(pixelBuffer); + size_t bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer); + if (baseAddress != NULL && width > 0 && height > 0 && bytesPerRow >= width * 4) { + CGDataProviderRef provider = CGDataProviderCreateWithData(NULL, + baseAddress, + bytesPerRow * height, + NULL); + if (provider != NULL) { + image = CGImageCreate(width, + height, + 8, + 32, + bytesPerRow, + XCWDeviceRGBColorSpace(), + kCGBitmapByteOrder32Little | kCGImageAlphaPremultipliedFirst, + provider, + NULL, + false, + kCGRenderingIntentDefault); + CGDataProviderRelease(provider); + } + } + } + + if (image == NULL) { + if (didLockPixelBuffer) { + CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); + didLockPixelBuffer = NO; + } + OSStatus imageStatus = VTCreateCGImageFromCVPixelBuffer(pixelBuffer, NULL, &image); + if (imageStatus != noErr || image == NULL) { + return nil; + } + } + + NSMutableData *data = [NSMutableData data]; + CGImageDestinationRef destination = + CGImageDestinationCreateWithData((__bridge CFMutableDataRef)data, + CFSTR("public.jpeg"), + 1, + NULL); + if (destination == NULL) { + CGImageRelease(image); + if (didLockPixelBuffer) { + CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); + } + return nil; + } + + NSDictionary *properties = @{ + (__bridge NSString *)kCGImageDestinationLossyCompressionQuality: @(XCWJPEGQualityFromEnvironment()), + }; + CGImageDestinationAddImage(destination, image, (__bridge CFDictionaryRef)properties); + BOOL ok = CGImageDestinationFinalize(destination); + CFRelease(destination); + CGImageRelease(image); + if (didLockPixelBuffer) { + CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); + } + return ok && data.length > 0 ? data : nil; +} + static NSData *XCWCopySampleData(CMSampleBufferRef sampleBuffer) { CMBlockBufferRef blockBuffer = CMSampleBufferGetDataBuffer(sampleBuffer); if (blockBuffer == NULL) { @@ -505,6 +611,12 @@ - (BOOL)encodePixelBufferLocked:(CVPixelBufferRef)pixelBuffer { return NO; } + if (_encoderMode == XCWVideoEncoderModeJPEG) { + return [self encodeJPEGPixelBufferLocked:pixelBuffer + sourceWidth:sourceWidth + sourceHeight:sourceHeight]; + } + if (![self ensureCompressionSessionWithWidth:targetWidth height:targetHeight]) { return NO; } @@ -553,6 +665,40 @@ - (BOOL)encodePixelBufferLocked:(CVPixelBufferRef)pixelBuffer { return YES; } +- (BOOL)encodeJPEGPixelBufferLocked:(CVPixelBufferRef)pixelBuffer + sourceWidth:(int32_t)sourceWidth + sourceHeight:(int32_t)sourceHeight { + uint64_t submittedAtUs = (uint64_t)(CACurrentMediaTime() * 1000000.0); + if (_timestampOriginUs == 0) { + _timestampOriginUs = submittedAtUs; + } + uint64_t relativeTimestampUs = submittedAtUs - _timestampOriginUs; + + NSData *jpegData = XCWJPEGDataFromPixelBuffer(pixelBuffer); + if (jpegData.length == 0) { + _encodeFailureCount += 1; + _lastEncodeStatus = -1; + return NO; + } + + _width = sourceWidth; + _height = sourceHeight; + _submittedFrameCount += 1; + _outputFrameCount += 1; + _keyFrameOutputCount += 1; + _lastEncodeStatus = noErr; + uint64_t nowUs = (uint64_t)(CACurrentMediaTime() * 1000000.0); + _latestEncodeLatencyUs = nowUs >= submittedAtUs ? nowUs - submittedAtUs : 0; + + self.outputHandler(jpegData, + relativeTimestampUs, + YES, + @"jpeg", + nil, + CGSizeMake(sourceWidth, sourceHeight)); + return YES; +} + - (BOOL)ensureCompressionSessionWithWidth:(int32_t)width height:(int32_t)height { if (_compressionSession != NULL && _width == width && _height == height) { return YES; diff --git a/client/src/features/stream/streamWorkerClient.ts b/client/src/features/stream/streamWorkerClient.ts index e7a9679..eec0937 100644 --- a/client/src/features/stream/streamWorkerClient.ts +++ b/client/src/features/stream/streamWorkerClient.ts @@ -8,13 +8,19 @@ import type { const HAVE_CURRENT_DATA = 2; const WEBRTC_CONTROL_CHANNEL_LABEL = "simdeck-control"; +const WEBRTC_JPEG_CHANNEL_LABEL = "simdeck-video-jpeg"; +const JPEG_CHUNK_HEADER_BYTES = 40; +const JPEG_CHUNK_MAGIC = "SDJF"; let activeWebRtcControlChannel: RTCDataChannel | null = null; export type StreamBackend = "webtransport" | "webrtc"; export function isWebRtcStreamMode(): boolean { - return streamTransportMode() === "webrtc"; + return ( + streamTransportMode().startsWith("webrtc") && + Boolean(accessTokenFromLocation()) + ); } export function sendWebRtcControlMessage(encoded: string): boolean { @@ -31,7 +37,7 @@ export function buildStreamTarget(udid: string): StreamConnectTarget { export function initialStreamBackend(): StreamBackend { const mode = streamTransportMode(); - if (mode === "webrtc") { + if (mode.startsWith("webrtc")) { return "webrtc"; } if (mode === "webtransport") { @@ -559,6 +565,458 @@ class WebRtcStreamClient implements StreamClientBackend { } } +class WebRtcJpegDataStreamClient implements StreamClientBackend { + private canvas: HTMLCanvasElement | null = null; + private connectGeneration = 0; + private context: CanvasRenderingContext2D | null = null; + private controlChannel: RTCDataChannel | null = null; + private frameAssembly: JpegFrameAssembly | null = null; + private hasReportedStreaming = false; + private latestAcceptedSequence = 0; + private lastConfigHeight = 0; + private lastConfigWidth = 0; + private lastRenderAt = 0; + private lastStatsReportAt = 0; + private peerConnection: RTCPeerConnection | null = null; + private rendering = false; + private stats: StreamStats = createEmptyStreamStats(); + private videoChannel: RTCDataChannel | null = null; + + constructor( + private readonly onMessage: (message: WorkerToMainMessage) => void, + ) {} + + attachCanvas(canvasElement: HTMLCanvasElement) { + this.canvas = canvasElement; + this.context = canvasElement.getContext("2d", { + alpha: false, + desynchronized: true, + } as CanvasRenderingContext2DSettings & { desynchronized: boolean }); + if (!this.context) { + throw new Error("Unable to create a 2D canvas renderer for WebRTC JPEG."); + } + } + + clear() { + if (!this.canvas || !this.context) { + return; + } + this.context.clearRect(0, 0, this.canvas.width, this.canvas.height); + } + + async connect(target: StreamConnectTarget) { + this.disconnect(); + if (!this.canvas || !this.context) { + return; + } + const generation = ++this.connectGeneration; + this.frameAssembly = null; + this.hasReportedStreaming = false; + this.latestAcceptedSequence = 0; + this.lastConfigHeight = 0; + this.lastConfigWidth = 0; + this.lastRenderAt = 0; + this.lastStatsReportAt = 0; + this.stats = createEmptyStreamStats(); + this.onMessage({ + type: "status", + status: { detail: "Creating WebRTC data channel", state: "connecting" }, + }); + + const peerConnection = new RTCPeerConnection({ + iceServers: iceServers(), + }); + this.peerConnection = peerConnection; + + const controlChannel = peerConnection.createDataChannel( + WEBRTC_CONTROL_CHANNEL_LABEL, + { ordered: true }, + ); + this.controlChannel = controlChannel; + activeWebRtcControlChannel = controlChannel; + controlChannel.addEventListener("close", () => { + if (activeWebRtcControlChannel === controlChannel) { + activeWebRtcControlChannel = null; + } + }); + + const videoChannel = peerConnection.createDataChannel( + WEBRTC_JPEG_CHANNEL_LABEL, + { + maxRetransmits: 0, + ordered: false, + }, + ); + videoChannel.binaryType = "arraybuffer"; + this.videoChannel = videoChannel; + videoChannel.addEventListener("open", () => { + if (generation !== this.connectGeneration) { + return; + } + this.onMessage({ + type: "status", + status: { + detail: "WebRTC JPEG channel connected", + state: "connecting", + }, + }); + }); + videoChannel.addEventListener("message", (event) => { + if (generation !== this.connectGeneration) { + return; + } + const bytes = dataChannelBytes(event.data); + if (!bytes) { + return; + } + void this.consumeJpegChunk(bytes, generation); + }); + videoChannel.addEventListener("close", () => { + if (generation !== this.connectGeneration) { + return; + } + this.onMessage({ + type: "status", + status: { detail: "WebRTC JPEG channel closed", state: "idle" }, + }); + }); + + peerConnection.onconnectionstatechange = () => { + if (peerConnection.connectionState === "failed") { + this.onMessage({ + type: "status", + status: { + error: "WebRTC data-channel connection failed.", + state: "error", + }, + }); + } + }; + + const offer = await peerConnection.createOffer(); + if (generation !== this.connectGeneration) { + return; + } + await peerConnection.setLocalDescription(offer); + await waitForIceGathering(peerConnection); + if (generation !== this.connectGeneration) { + return; + } + const localDescription = peerConnection.localDescription; + if (!localDescription) { + throw new Error("WebRTC data-channel offer was not created."); + } + + const response = await fetch( + `/api/simulators/${encodeURIComponent(target.udid)}/webrtc/offer`, + { + body: JSON.stringify({ + sdp: localDescription.sdp, + transport: "data-channel", + type: localDescription.type, + }), + headers: apiHeaders(), + method: "POST", + }, + ); + if (!response.ok) { + throw new Error(await response.text()); + } + const answer = (await response.json()) as RTCSessionDescriptionInit; + if (generation !== this.connectGeneration) { + return; + } + await peerConnection.setRemoteDescription(answer); + } + + disconnect() { + this.connectGeneration += 1; + this.frameAssembly = null; + this.hasReportedStreaming = false; + this.controlChannel?.close(); + if (activeWebRtcControlChannel === this.controlChannel) { + activeWebRtcControlChannel = null; + } + this.controlChannel = null; + this.videoChannel?.close(); + this.videoChannel = null; + this.peerConnection?.close(); + this.peerConnection = null; + this.onMessage({ type: "status", status: { state: "idle" } }); + } + + destroy() { + this.disconnect(); + } + + private async consumeJpegChunk( + bytes: Uint8Array, + generation: number, + ) { + const chunk = parseJpegChunk(bytes); + if (!chunk || chunk.frameSequence < this.latestAcceptedSequence) { + return; + } + if ( + !this.frameAssembly || + this.frameAssembly.frameSequence !== chunk.frameSequence + ) { + this.frameAssembly = { + chunkCount: chunk.chunkCount, + chunks: new Array | null>( + chunk.chunkCount, + ).fill(null), + frameSequence: chunk.frameSequence, + height: chunk.height, + received: 0, + timestampUs: chunk.timestampUs, + totalLength: chunk.totalLength, + width: chunk.width, + }; + } + const assembly = this.frameAssembly; + if ( + chunk.chunkIndex >= assembly.chunkCount || + chunk.chunkCount !== assembly.chunkCount || + chunk.totalLength !== assembly.totalLength + ) { + return; + } + if (!assembly.chunks[chunk.chunkIndex]) { + assembly.chunks[chunk.chunkIndex] = chunk.payload; + assembly.received += 1; + } + if (assembly.received !== assembly.chunkCount) { + return; + } + + const frame = new Uint8Array(assembly.totalLength); + let offset = 0; + for (const part of assembly.chunks) { + if (!part) { + return; + } + frame.set(part, offset); + offset += part.byteLength; + } + this.frameAssembly = null; + this.latestAcceptedSequence = assembly.frameSequence; + await this.renderJpegFrame(frame, assembly, generation); + } + + private async renderJpegFrame( + payload: Uint8Array, + metadata: JpegFrameAssembly, + generation: number, + ) { + if (!this.canvas || !this.context || this.rendering) { + this.stats.droppedFrames += 1; + return; + } + this.rendering = true; + const startedAt = performance.now(); + try { + const decoded = await decodeJpegFrame(payload); + if (generation !== this.connectGeneration) { + decoded.close(); + return; + } + this.syncCanvasSize(metadata.width, metadata.height); + this.context.drawImage( + decoded.image, + 0, + 0, + this.canvas.width, + this.canvas.height, + ); + decoded.close(); + const now = performance.now(); + const renderMs = performance.now() - startedAt; + this.stats.latestFrameGapMs = + this.lastRenderAt > 0 ? now - this.lastRenderAt : 0; + this.lastRenderAt = now; + this.stats.averageRenderMs = + this.stats.renderedFrames === 0 + ? renderMs + : this.stats.averageRenderMs * 0.85 + renderMs * 0.15; + this.stats.codec = "jpeg/webrtc-data"; + this.stats.decodeQueueSize = this.videoChannel?.bufferedAmount ?? 0; + this.stats.decodedFrames += 1; + this.stats.frameSequence = metadata.frameSequence; + this.stats.height = metadata.height; + this.stats.latestRenderMs = renderMs; + this.stats.maxRenderMs = Math.max(this.stats.maxRenderMs, renderMs); + this.stats.receivedPackets += 1; + this.stats.renderedFrames += 1; + this.stats.waitingForKeyFrame = false; + this.stats.width = metadata.width; + if ( + metadata.width !== this.lastConfigWidth || + metadata.height !== this.lastConfigHeight + ) { + this.lastConfigWidth = metadata.width; + this.lastConfigHeight = metadata.height; + this.onMessage({ + type: "video-config", + size: { height: metadata.height, width: metadata.width }, + }); + } + if (!this.hasReportedStreaming) { + this.hasReportedStreaming = true; + this.onMessage({ + type: "status", + status: { + detail: "WebRTC JPEG stream connected", + state: "streaming", + }, + }); + } + if (now - this.lastStatsReportAt >= 250) { + this.lastStatsReportAt = now; + this.onMessage({ type: "stats", stats: { ...this.stats } }); + } + } finally { + this.rendering = false; + } + } + + private syncCanvasSize(width: number, height: number) { + if (!this.canvas) { + return; + } + const nextWidth = Math.max(1, Math.round(width)); + const nextHeight = Math.max(1, Math.round(height)); + if (this.canvas.width !== nextWidth) { + this.canvas.width = nextWidth; + } + if (this.canvas.height !== nextHeight) { + this.canvas.height = nextHeight; + } + } +} + +interface JpegChunk { + chunkCount: number; + chunkIndex: number; + frameSequence: number; + height: number; + payload: Uint8Array; + timestampUs: number; + totalLength: number; + width: number; +} + +interface JpegFrameAssembly { + chunkCount: number; + chunks: Array | null>; + frameSequence: number; + height: number; + received: number; + timestampUs: number; + totalLength: number; + width: number; +} + +interface DecodedJpegFrame { + image: CanvasImageSource; + close(): void; +} + +interface ImageDecoderConstructor { + new (init: { data: BufferSource; preferAnimation?: boolean; type: string }): { + close(): void; + decode(options?: { frameIndex?: number }): Promise<{ + image: { close(): void }; + }>; + }; +} + +async function decodeJpegFrame( + payload: Uint8Array, +): Promise { + const imageDecoder = ( + globalThis as typeof globalThis & { + ImageDecoder?: ImageDecoderConstructor; + } + ).ImageDecoder; + if (imageDecoder) { + const decoder = new imageDecoder({ + data: payload as BufferSource, + preferAnimation: false, + type: "image/jpeg", + }); + try { + const result = await decoder.decode({ frameIndex: 0 }); + return { + image: result.image as unknown as CanvasImageSource, + close() { + result.image.close(); + decoder.close(); + }, + }; + } catch (error) { + decoder.close(); + throw error; + } + } + + const blob = new Blob([payload as unknown as BlobPart], { + type: "image/jpeg", + }); + const image = await createImageBitmap(blob); + return { + image, + close() { + image.close(); + }, + }; +} + +function parseJpegChunk(bytes: Uint8Array): JpegChunk | null { + if (bytes.byteLength < JPEG_CHUNK_HEADER_BYTES) { + return null; + } + const magic = String.fromCharCode(bytes[0], bytes[1], bytes[2], bytes[3]); + if (magic !== JPEG_CHUNK_MAGIC || bytes[4] !== 1) { + return null; + } + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const chunkIndex = view.getUint16(6, false); + const chunkCount = view.getUint16(8, false); + const frameSequence = Number(view.getBigUint64(12, false)); + const timestampUs = Number(view.getBigUint64(20, false)); + const width = view.getUint32(28, false); + const height = view.getUint32(32, false); + const totalLength = view.getUint32(36, false); + if (chunkCount === 0 || chunkIndex >= chunkCount || totalLength === 0) { + return null; + } + return { + chunkCount, + chunkIndex, + frameSequence, + height, + payload: bytes.subarray(JPEG_CHUNK_HEADER_BYTES), + timestampUs, + totalLength, + width, + }; +} + +function dataChannelBytes(value: unknown): Uint8Array | null { + if (value instanceof ArrayBuffer) { + return new Uint8Array(value); + } + if (value instanceof Blob) { + return null; + } + if (!ArrayBuffer.isView(value)) { + return null; + } + const view = value as ArrayBufferView; + return new Uint8Array(view.buffer, view.byteOffset, view.byteLength); +} + function configureLowLatencyReceiver(receiver: RTCRtpReceiver) { const lowLatencyReceiver = receiver as RTCRtpReceiver & { jitterBufferTarget?: number; @@ -752,7 +1210,12 @@ export class StreamWorkerClient { } private createBackend(canvasElement: HTMLCanvasElement): StreamClientBackend { - if (this.backendMode === "webrtc") { + const mode = streamTransportMode(); + if (mode === "webrtc-data") { + void canvasElement; + return new WebRtcJpegDataStreamClient(this.onMessage); + } + if (this.backendMode === "webrtc" || mode === "webrtc") { return new WebRtcStreamClient(this.onMessage); } void canvasElement; diff --git a/docs/api/health.md b/docs/api/health.md index 8fdf64f..7c2c948 100644 --- a/docs/api/health.md +++ b/docs/api/health.md @@ -12,7 +12,8 @@ Returns the static bootstrap information the browser client needs to open a WebT "httpPort": 4310, "wtPort": 4311, "timestamp": 1714094761.234, - "videoCodec": "hevc", + "videoCodec": "h264-software", + "jpegQuality": 1.0, "webTransport": { "urlTemplate": "https://127.0.0.1:4311/wt/simulators/{udid}?simdeckToken=...", "certificateHash": { @@ -24,15 +25,16 @@ Returns the static bootstrap information the browser client needs to open a WebT } ``` -| Field | Notes | -| ------------------------------------ | --------------------------------------------------------------------------------------------- | -| `ok` | Always `true` if the route is reachable. Network failures are signalled by HTTP errors. | -| `httpPort` / `wtPort` | Numeric ports for the HTTP and WebTransport servers. WebTransport is always `httpPort + 1`. | -| `timestamp` | Server-side `time.now()` as a fractional Unix epoch in seconds. | -| `videoCodec` | Active encoder. One of `hevc`, `h264`, `h264-software`. See [Video Pipeline](/guide/video). | -| `webTransport.urlTemplate` | URL with a `{udid}` placeholder and access token query for the simulator stream. | -| `webTransport.certificateHash.value` | SHA-256 of the server's self-signed cert. Pin via `serverCertificateHashes` in the WT client. | -| `webTransport.packetVersion` | The current binary packet protocol version. Clients should refuse to parse unknown versions. | +| Field | Notes | +| ------------------------------------ | ------------------------------------------------------------------------------------------------------ | +| `ok` | Always `true` if the route is reachable. Network failures are signalled by HTTP errors. | +| `httpPort` / `wtPort` | Numeric ports for the HTTP and WebTransport servers. WebTransport is always `httpPort + 1`. | +| `timestamp` | Server-side `time.now()` as a fractional Unix epoch in seconds. | +| `videoCodec` | Active encoder. One of `hevc`, `h264`, `h264-software`, or `jpeg`. See [Video Pipeline](/guide/video). | +| `jpegQuality` | JPEG encoder quality from `0.1` to `1.0`; only affects `videoCodec: "jpeg"`. | +| `webTransport.urlTemplate` | URL with a `{udid}` placeholder and access token query for the simulator stream. | +| `webTransport.certificateHash.value` | SHA-256 of the server's self-signed cert. Pin via `serverCertificateHashes` in the WT client. | +| `webTransport.packetVersion` | The current binary packet protocol version. Clients should refuse to parse unknown versions. | The certificate and default access token are regenerated every time the server restarts. A client that caches the hash should refetch `/api/health` after any disconnection. diff --git a/docs/api/rest.md b/docs/api/rest.md index 0850929..d29ce3f 100644 --- a/docs/api/rest.md +++ b/docs/api/rest.md @@ -23,7 +23,8 @@ Returns server health, the WebTransport URL template, and the certificate hash t "httpPort": 4310, "wtPort": 4311, "timestamp": 1714094761.234, - "videoCodec": "hevc", + "videoCodec": "h264-software", + "jpegQuality": 1.0, "webTransport": { "urlTemplate": "https://127.0.0.1:4311/wt/simulators/{udid}?simdeckToken=...", "certificateHash": { @@ -150,9 +151,11 @@ video track: } ``` -This endpoint requires the simulator stream to be H.264. For GitHub Actions -provider runs, start the runner host with `--video-codec h264-software` and pass -the one-time provider token as `--access-token`. +By default this endpoint creates an H.264 WebRTC media track, so the simulator +stream must be H.264. For GitHub Actions provider runs that need full +resolution without hardware H.264/HEVC, post the offer with +`"transport": "data-channel"`, start the runner host with `--video-codec jpeg`, +and open the browser UI with `?transport=webrtc-data`. ### `POST /api/simulators/{udid}/open-url` diff --git a/docs/cli/commands.md b/docs/cli/commands.md index 77b1062..23fcf13 100644 --- a/docs/cli/commands.md +++ b/docs/cli/commands.md @@ -31,8 +31,8 @@ Start or reuse the project daemon and serve the browser UI. ```sh simdeck ui [--port 4310] [--bind 127.0.0.1] [--advertise-host ] - [--client-root ] [--video-codec hevc|h264|h264-software] - [--open] + [--client-root ] [--video-codec hevc|h264|h264-software|jpeg] + [--jpeg-quality 1.0] [--open] ``` `--open` opens the authenticated local URL after the daemon is ready. @@ -44,7 +44,8 @@ Start or reuse the project daemon without opening the browser: ```sh simdeck daemon start [--port 4310] [--bind 127.0.0.1] [--advertise-host ] [--client-root ] - [--video-codec hevc|h264|h264-software] + [--video-codec hevc|h264|h264-software|jpeg] + [--jpeg-quality 1.0] ``` Output: @@ -85,8 +86,8 @@ that starts after login and stays available. ```sh simdeck service on [--port 4310] [--bind 127.0.0.1] [--advertise-host ] [--client-root ] - [--video-codec hevc|h264|h264-software] - [--access-token ] + [--video-codec hevc|h264|h264-software|jpeg] + [--jpeg-quality 1.0] [--access-token ] simdeck service restart [same options as service on] simdeck service off ``` diff --git a/docs/cli/flags.md b/docs/cli/flags.md index 335a953..d75e5b7 100644 --- a/docs/cli/flags.md +++ b/docs/cli/flags.md @@ -28,14 +28,15 @@ Targets a specific running SimDeck daemon for commands that support the HTTP fas `ui` and `daemon start` accept the same server options. `ui` also accepts `--open`. -| Flag | Default | Description | -| ------------------ | --------------------- | ---------------------------------------------------------------------------- | -| `--port ` | `4310` | HTTP port. WebTransport listens on `port + 1`. | -| `--bind ` | `127.0.0.1` | Bind address (`0.0.0.0` for [LAN access](/guide/lan-access), `::` for IPv6). | -| `--advertise-host` | matches local host | Hostname or IP advertised in the WebTransport URL template and certificate. | -| `--client-root` | bundled `client/dist` | Override the static browser client directory. | -| `--video-codec` | `h264-software` | One of `hevc`, `h264`, `h264-software`. See [Video Pipeline](/guide/video). | -| `--open` | `false` | `ui` only. Open the browser after the daemon is ready. | +| Flag | Default | Description | +| ------------------ | --------------------- | ----------------------------------------------------------------------------------- | +| `--port ` | `4310` | HTTP port. WebTransport listens on `port + 1`. | +| `--bind ` | `127.0.0.1` | Bind address (`0.0.0.0` for [LAN access](/guide/lan-access), `::` for IPv6). | +| `--advertise-host` | matches local host | Hostname or IP advertised in the WebTransport URL template and certificate. | +| `--client-root` | bundled `client/dist` | Override the static browser client directory. | +| `--video-codec` | `h264-software` | One of `hevc`, `h264`, `h264-software`, `jpeg`. See [Video Pipeline](/guide/video). | +| `--jpeg-quality` | `1.0` | JPEG quality from `0.1` to `1.0` when `--video-codec jpeg` is used. | +| `--open` | `false` | `ui` only. Open the browser after the daemon is ready. | The public commands generate an access token automatically. Use `simdeck daemon status` to read it for direct API callers. diff --git a/docs/guide/daemon.md b/docs/guide/daemon.md index f123521..289f268 100644 --- a/docs/guide/daemon.md +++ b/docs/guide/daemon.md @@ -54,14 +54,15 @@ This starts or reuses the project daemon, serves the bundled browser client, and `daemon start` and `ui` accept the same server options: -| Flag | Default | Notes | -| ------------------ | --------------------- | ------------------------------------------------------------------ | -| `--port ` | `4310` | HTTP port. WebTransport listens on `port + 1`. | -| `--bind ` | `127.0.0.1` | Bind address. Use `0.0.0.0` for [LAN access](/guide/lan-access). | -| `--advertise-host` | matches local host | Hostname or IP advertised to browser clients. | -| `--client-root` | bundled `client/dist` | Override the static browser client directory. | -| `--video-codec` | `h264-software` | One of `hevc`, `h264`, `h264-software`. See [Video](/guide/video). | -| `--open` | `false` | `ui` only. Open the browser after the daemon is ready. | +| Flag | Default | Notes | +| ------------------ | --------------------- | -------------------------------------------------------------------------- | +| `--port ` | `4310` | HTTP port. WebTransport listens on `port + 1`. | +| `--bind ` | `127.0.0.1` | Bind address. Use `0.0.0.0` for [LAN access](/guide/lan-access). | +| `--advertise-host` | matches local host | Hostname or IP advertised to browser clients. | +| `--client-root` | bundled `client/dist` | Override the static browser client directory. | +| `--video-codec` | `h264-software` | One of `hevc`, `h264`, `h264-software`, `jpeg`. See [Video](/guide/video). | +| `--jpeg-quality` | `1.0` | JPEG quality from `0.1` to `1.0` when `--video-codec jpeg` is used. | +| `--open` | `false` | `ui` only. Open the browser after the daemon is ready. | Example: diff --git a/docs/guide/troubleshooting.md b/docs/guide/troubleshooting.md index fd2622e..2fcba2d 100644 --- a/docs/guide/troubleshooting.md +++ b/docs/guide/troubleshooting.md @@ -85,6 +85,15 @@ The encoder did not produce a keyframe within 3 seconds. The most common causes: simdeck daemon start --video-codec h264-software ``` + On virtualized CI Macs where hardware H.264/HEVC is unavailable and full + resolution matters, use the experimental JPEG data-channel stream instead: + + ```sh + simdeck daemon stop + simdeck daemon start --video-codec jpeg + # open the browser with ?transport=webrtc-data + ``` + - **The Simulator window is minimised or off-screen.** The private display bridge captures from a headless context, so this is rare, but if you see it after waking from sleep, shut the simulator down and boot it again. - **The simulator is mid-shutdown.** Wait for `simdeck list` to report `isBooted: true`. diff --git a/docs/guide/video.md b/docs/guide/video.md index d09b4d5..67981cf 100644 --- a/docs/guide/video.md +++ b/docs/guide/video.md @@ -1,16 +1,17 @@ # Video Pipeline -SimDeck streams the iOS Simulator over WebTransport using a binary frame protocol. This page walks through the encoder choices, the keyframe handshake, and the metrics you can use to tune them. +SimDeck streams the iOS Simulator over WebTransport using a binary frame protocol. It also has an experimental WebRTC data-channel path for CI hosts that cannot hardware-encode H.264 or HEVC. This page walks through the encoder choices, the keyframe handshake, and the metrics you can use to tune them. ## Codec selection -The server can encode the simulator display in three modes, picked at startup with `--video-codec`: +The server can encode the simulator display in four modes, picked at startup with `--video-codec`: -| Value | Encoder | When to use it | -| ------------------ | ------------------------------------------- | --------------------------------------------------------------------------- | -| `hevc` _(default)_ | Hardware HEVC via VideoToolbox | Best quality and bandwidth on modern Apple Silicon. The default everywhere. | -| `h264` | Hardware H.264 via VideoToolbox | Falls back if a downstream client cannot decode HEVC. | -| `h264-software` | Software H.264 (libavcodec / openh264 path) | Use when macOS screen recording starves the hardware encoder. | +| Value | Encoder | When to use it | +| --------------------------- | ------------------------------- | ---------------------------------------------------------------------------------------------------- | +| `hevc` | Hardware HEVC via VideoToolbox | Best quality and bandwidth on modern Apple Silicon when hardware encode is available. | +| `h264` | Hardware H.264 via VideoToolbox | Use when a downstream client cannot decode HEVC and hardware H.264 is available. | +| `h264-software` _(default)_ | Software H.264 via VideoToolbox | Compatibility fallback when hardware encode is unavailable, but full-resolution latency may be high. | +| `jpeg` | Software JPEG via ImageIO | Experimental full-resolution CI path. Use with browser query `?transport=webrtc-data`. | You can switch at any time by restarting the server with a different flag: @@ -19,10 +20,18 @@ simdeck daemon stop simdeck daemon start --video-codec h264-software ``` +For GitHub Actions `macos-latest` runners, prefer the experimental JPEG path: + +```sh +simdeck daemon stop +simdeck daemon start --video-codec jpeg +# open the UI with ?transport=webrtc-data +``` + The chosen codec is reported to clients in two places: - The JSON `videoCodec` field on `GET /api/health`. -- The `codec` field of the [`ControlHello`](/api/webtransport#control-hello) message that the WebTransport hub sends as soon as a session attaches. +- The `codec` field of the [`ControlHello`](/api/webtransport#control-hello) message that the WebTransport hub sends as soon as a session attaches. The JPEG WebRTC data-channel path does not use `ControlHello`; its frame chunks carry width, height, timestamp, and frame sequence metadata. ## Keyframe handshake @@ -49,9 +58,23 @@ Discontinuities are signalled to the client through the `FLAG_DISCONTINUITY` bit A few practical guidelines: -- **Start on the default.** HEVC delivers the best quality-per-bit and the lowest CPU on M-series Macs. +- **Start on the default for compatibility.** `h264-software` works without requiring the hardware encoder, but full-resolution latency can be high. +- **Switch to `hevc` on local Apple Silicon when hardware encode is available.** HEVC delivers the best quality-per-bit and the lowest CPU on M-series Macs. - **Switch to `h264` when a remote client cannot decode HEVC.** Some browsers on older Apple devices are H.264-only. -- **Switch to `h264-software` when the hardware encoder stalls.** macOS screen recording can monopolise the VideoToolbox HEVC encoder. If you see "encoder unavailable" errors in the server log while QuickTime or `screencapture` is active, switch to `h264-software`. +- **Switch to `h264-software` when the hardware encoder stalls and you can tolerate extra latency.** macOS screen recording can monopolise the VideoToolbox HEVC encoder. If you see "encoder unavailable" errors in the server log while QuickTime or `screencapture` is active, switch to `h264-software`. +- **Switch to `jpeg` plus `?transport=webrtc-data` on virtualized CI Macs.** GitHub Actions macOS runners commonly fail hardware-required H.264/HEVC session creation, and software H.264 can be too latent at full simulator resolution. The JPEG path is stateless, full resolution, and drops stale frames instead of waiting for inter-frame video dependencies. + +## JPEG tuning + +`--jpeg-quality` accepts values from `0.1` to `1.0` and defaults to full quality, `1.0`. It only affects `--video-codec jpeg`. + +For full quality on a CI runner, start with: + +```sh +simdeck daemon start --video-codec jpeg +``` + +Lower it only if `/api/metrics` shows server drops or the browser diagnostics show non-zero data-channel backlog. ## Tuning with metrics diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs index 307d626..3b6170e 100644 --- a/server/src/api/routes.rs +++ b/server/src/api/routes.rs @@ -476,6 +476,7 @@ async fn health(State(state): State) -> Json { "wtPort": state.config.wt_port, "timestamp": SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or(Duration::ZERO).as_secs_f64(), "videoCodec": state.config.video_codec, + "jpegQuality": state.config.jpeg_quality, "webTransport": { "urlTemplate": auth::tokenized_webtransport_template(&state.config), "certificateHash": { diff --git a/server/src/auth.rs b/server/src/auth.rs index fb53c15..cc7eba4 100644 --- a/server/src/auth.rs +++ b/server/src/auth.rs @@ -262,6 +262,7 @@ mod tests { IpAddr::V4(Ipv4Addr::LOCALHOST), None, "hevc".to_owned(), + 1.0, Some("secret-token".to_owned()), Some("123456".to_owned()), ) @@ -329,6 +330,7 @@ mod tests { IpAddr::V4(Ipv4Addr::UNSPECIFIED), Some("10.0.0.245".to_owned()), "hevc".to_owned(), + 1.0, Some("secret-token".to_owned()), Some("123456".to_owned()), ); diff --git a/server/src/config.rs b/server/src/config.rs index d0fbbf0..1d2c45e 100644 --- a/server/src/config.rs +++ b/server/src/config.rs @@ -11,15 +11,18 @@ pub struct Config { pub wt_port: u16, pub client_root: PathBuf, pub video_codec: String, + pub jpeg_quality: f64, } impl Config { + #[allow(clippy::too_many_arguments)] pub fn new( http_port: u16, client_root: PathBuf, bind_ip: IpAddr, advertise_host: Option, video_codec: String, + jpeg_quality: f64, access_token: Option, pairing_code: Option, ) -> Self { @@ -38,6 +41,7 @@ impl Config { wt_port, client_root, video_codec, + jpeg_quality, } } diff --git a/server/src/main.rs b/server/src/main.rs index 337fc94..cdf586a 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -80,6 +80,8 @@ enum Command { client_root: Option, #[arg(long, value_enum, default_value_t = VideoCodecMode::H264Software)] video_codec: VideoCodecMode, + #[arg(long, default_value_t = 1.0, value_parser = parse_jpeg_quality)] + jpeg_quality: f64, #[arg(long)] open: bool, }, @@ -99,6 +101,8 @@ enum Command { client_root: Option, #[arg(long, value_enum, default_value_t = VideoCodecMode::H264Software)] video_codec: VideoCodecMode, + #[arg(long, default_value_t = 1.0, value_parser = parse_jpeg_quality)] + jpeg_quality: f64, #[arg(long)] access_token: Option, #[arg(long)] @@ -369,6 +373,8 @@ enum DaemonCommand { client_root: Option, #[arg(long, value_enum, default_value_t = VideoCodecMode::H264Software)] video_codec: VideoCodecMode, + #[arg(long, default_value_t = 1.0, value_parser = parse_jpeg_quality)] + jpeg_quality: f64, }, Stop, Status, @@ -388,6 +394,8 @@ enum DaemonCommand { client_root: Option, #[arg(long, value_enum, default_value_t = VideoCodecMode::H264Software)] video_codec: VideoCodecMode, + #[arg(long, default_value_t = 1.0, value_parser = parse_jpeg_quality)] + jpeg_quality: f64, #[arg(long)] access_token: String, #[arg(long)] @@ -408,6 +416,8 @@ enum ServiceCommand { client_root: Option, #[arg(long, value_enum, default_value_t = VideoCodecMode::H264Software)] video_codec: VideoCodecMode, + #[arg(long, default_value_t = 1.0, value_parser = parse_jpeg_quality)] + jpeg_quality: f64, #[arg(long)] access_token: Option, }, @@ -422,6 +432,8 @@ enum ServiceCommand { client_root: Option, #[arg(long, value_enum, default_value_t = VideoCodecMode::H264Software)] video_codec: VideoCodecMode, + #[arg(long, default_value_t = 1.0, value_parser = parse_jpeg_quality)] + jpeg_quality: f64, #[arg(long)] access_token: Option, }, @@ -455,6 +467,7 @@ enum VideoCodecMode { Hevc, H264, H264Software, + Jpeg, } #[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] @@ -493,6 +506,7 @@ struct DaemonLaunchOptions { advertise_host: Option, client_root: Option, video_codec: VideoCodecMode, + jpeg_quality: f64, } impl VideoCodecMode { @@ -501,10 +515,26 @@ impl VideoCodecMode { Self::Hevc => "hevc", Self::H264 => "h264", Self::H264Software => "h264-software", + Self::Jpeg => "jpeg", } } } +fn parse_jpeg_quality(value: &str) -> Result { + let quality = value + .parse::() + .map_err(|error| format!("invalid JPEG quality `{value}`: {error}"))?; + if (0.1..=1.0).contains(&quality) { + Ok(quality) + } else { + Err("JPEG quality must be between 0.1 and 1.0".to_owned()) + } +} + +fn format_jpeg_quality(value: f64) -> String { + format!("{value:.3}") +} + fn command_service_url(explicit: Option) -> anyhow::Result { if let Some(url) = explicit .or_else(|| env::var("SIMDECK_SERVER_URL").ok()) @@ -523,6 +553,7 @@ impl Default for DaemonLaunchOptions { advertise_host: None, client_root: None, video_codec: VideoCodecMode::H264Software, + jpeg_quality: 1.0, } } } @@ -566,6 +597,8 @@ fn start_project_daemon(options: DaemonLaunchOptions) -> anyhow::Result) -> anyhow::Result<()> { Some(advertise_host), None, video_codec, + 1.0, Some(access_token), Some(pairing_code), ); @@ -960,6 +994,7 @@ fn main() -> anyhow::Result<()> { advertise_host, client_root, video_codec, + jpeg_quality, open, } => { let (metadata, started) = ensure_project_daemon_with_status(DaemonLaunchOptions { @@ -968,6 +1003,7 @@ fn main() -> anyhow::Result<()> { advertise_host, client_root, video_codec, + jpeg_quality, })?; if open { open_browser(&metadata.http_url)?; @@ -982,6 +1018,7 @@ fn main() -> anyhow::Result<()> { advertise_host, client_root, video_codec, + jpeg_quality, } => { let (metadata, started) = ensure_project_daemon_with_status(DaemonLaunchOptions { port, @@ -989,6 +1026,7 @@ fn main() -> anyhow::Result<()> { advertise_host, client_root, video_codec, + jpeg_quality, })?; print_daemon_start_result(&metadata, started) } @@ -1002,6 +1040,7 @@ fn main() -> anyhow::Result<()> { advertise_host, client_root, video_codec, + jpeg_quality, access_token, pairing_code, } => { @@ -1023,6 +1062,7 @@ fn main() -> anyhow::Result<()> { advertise_host, client_root, video_codec, + jpeg_quality, Some(access_token), pairing_code, ); @@ -1036,6 +1076,7 @@ fn main() -> anyhow::Result<()> { advertise_host, client_root, video_codec, + jpeg_quality, access_token, pairing_code, } => serve_with_appkit( @@ -1044,6 +1085,7 @@ fn main() -> anyhow::Result<()> { advertise_host, client_root, video_codec, + jpeg_quality, access_token, pairing_code, ), @@ -1054,6 +1096,7 @@ fn main() -> anyhow::Result<()> { advertise_host, client_root, video_codec, + jpeg_quality, access_token, } => service::enable(ServiceOptions { port, @@ -1061,6 +1104,7 @@ fn main() -> anyhow::Result<()> { advertise_host, client_root, video_codec, + jpeg_quality, access_token, pairing_code: None, }), @@ -1070,6 +1114,7 @@ fn main() -> anyhow::Result<()> { advertise_host, client_root, video_codec, + jpeg_quality, access_token, } => service::restart(ServiceOptions { port, @@ -1077,6 +1122,7 @@ fn main() -> anyhow::Result<()> { advertise_host, client_root, video_codec, + jpeg_quality, access_token, pairing_code: None, }), @@ -1772,20 +1818,24 @@ struct ServiceOptions { advertise_host: Option, client_root: Option, video_codec: VideoCodecMode, + jpeg_quality: f64, access_token: Option, pairing_code: Option, } +#[allow(clippy::too_many_arguments)] fn serve_with_appkit( port: u16, bind: IpAddr, advertise_host: Option, client_root: Option, video_codec: VideoCodecMode, + jpeg_quality: f64, access_token: Option, pairing_code: Option, ) -> anyhow::Result<()> { std::env::set_var("SIMDECK_VIDEO_CODEC", video_codec.as_env_value()); + std::env::set_var("SIMDECK_JPEG_QUALITY", format_jpeg_quality(jpeg_quality)); std::env::set_var(RESTART_ON_CORE_SIMULATOR_MISMATCH_ENV, "1"); start_fd_pressure_watchdog(); unsafe { @@ -1805,6 +1855,7 @@ fn serve_with_appkit( advertise_host, client_root, video_codec, + jpeg_quality, access_token, pairing_code, )), @@ -3936,12 +3987,14 @@ fn hid_for_character(character: char) -> Option<(u16, u32)> { Some(mapping) } +#[allow(clippy::too_many_arguments)] async fn serve( port: u16, bind: IpAddr, advertise_host: Option, client_root: Option, video_codec: VideoCodecMode, + jpeg_quality: f64, access_token: Option, pairing_code: Option, ) -> anyhow::Result<()> { @@ -3955,6 +4008,7 @@ async fn serve( bind, advertise_host, video_codec.as_env_value().to_owned(), + jpeg_quality, access_token, pairing_code, ); diff --git a/server/src/service.rs b/server/src/service.rs index 279d10e..8beaa3a 100644 --- a/server/src/service.rs +++ b/server/src/service.rs @@ -179,6 +179,8 @@ fn plist_contents( client_root.to_string_lossy().into_owned(), "--video-codec".to_string(), options.video_codec.as_env_value().to_string(), + "--jpeg-quality".to_string(), + crate::format_jpeg_quality(options.jpeg_quality), ]; if let Some(advertise_host) = options.advertise_host.as_ref() { diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs index 4aa2170..4567f39 100644 --- a/server/src/transport/webrtc.rs +++ b/server/src/transport/webrtc.rs @@ -12,11 +12,13 @@ use webrtc::api::interceptor_registry::register_default_interceptors; use webrtc::api::media_engine::{MediaEngine, MIME_TYPE_H264}; use webrtc::api::APIBuilder; use webrtc::data_channel::data_channel_message::DataChannelMessage; +use webrtc::data_channel::data_channel_state::RTCDataChannelState; use webrtc::data_channel::RTCDataChannel; use webrtc::ice_transport::ice_server::RTCIceServer; use webrtc::interceptor::registry::Registry; use webrtc::media::Sample; use webrtc::peer_connection::configuration::RTCConfiguration; +use webrtc::peer_connection::peer_connection_state::RTCPeerConnectionState; use webrtc::peer_connection::sdp::session_description::RTCSessionDescription; use webrtc::rtp_transceiver::rtp_codec::RTCRtpCodecCapability; use webrtc::track::track_local::track_local_static_sample::TrackLocalStaticSample; @@ -25,11 +27,18 @@ use webrtc::track::track_local::TrackLocal; const ANNEX_B_START_CODE: &[u8] = &[0, 0, 0, 1]; const DEFAULT_STUN_URL: &str = "stun:stun.l.google.com:19302"; const WEBRTC_CONTROL_CHANNEL_LABEL: &str = "simdeck-control"; +const WEBRTC_JPEG_CHANNEL_LABEL: &str = "simdeck-video-jpeg"; const WEBRTC_BOOTSTRAP_KEYFRAME_INTERVAL: Duration = Duration::from_millis(250); const WEBRTC_BOOTSTRAP_KEYFRAME_REPEATS: u8 = 4; const WEBRTC_MIN_REFRESH_INTERVAL: Duration = Duration::from_millis(16); const WEBRTC_MAX_REFRESH_INTERVAL: Duration = Duration::from_millis(100); const WEBRTC_WRITE_TIMEOUT: Duration = Duration::from_millis(120); +const JPEG_CHUNK_MAGIC: &[u8; 4] = b"SDJF"; +const JPEG_CHUNK_VERSION: u8 = 1; +const JPEG_CHUNK_HEADER_BYTES: usize = 40; +const JPEG_CHUNK_PAYLOAD_BYTES: usize = 60 * 1024; +const JPEG_CHANNEL_MIN_BUFFERED_BYTES: usize = 2 * 1024 * 1024; +const JPEG_CHANNEL_BUFFERED_FRAME_MULTIPLIER: usize = 2; #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] @@ -37,6 +46,7 @@ pub struct WebRtcOfferPayload { pub sdp: String, #[serde(rename = "type")] pub kind: String, + pub transport: Option, } #[derive(Debug, Serialize)] @@ -78,6 +88,10 @@ pub async fn create_answer( .wait_for_keyframe(Duration::from_secs(3)) .await .ok_or_else(|| AppError::native("Timed out waiting for a simulator keyframe."))?; + if payload.transport.as_deref() == Some("data-channel") { + return create_data_channel_answer(state, udid, session, first_frame, payload).await; + } + let codec = first_frame .codec .as_deref() @@ -272,6 +286,141 @@ fn redact_candidate_address(address: &str) -> String { "".to_owned() } +async fn create_data_channel_answer( + state: AppState, + udid: String, + session: crate::simulators::session::SimulatorSession, + first_frame: crate::transport::packet::SharedFrame, + payload: WebRtcOfferPayload, +) -> Result { + let codec = first_frame + .codec + .as_deref() + .unwrap_or_default() + .to_lowercase(); + if codec != "jpeg" { + return Err(AppError::bad_request( + "WebRTC data-channel preview requires `--video-codec jpeg`.", + )); + } + + let mut media_engine = MediaEngine::default(); + media_engine + .register_default_codecs() + .map_err(|error| AppError::internal(format!("register WebRTC codecs: {error}")))?; + let mut registry = Registry::new(); + registry = register_default_interceptors(registry, &mut media_engine) + .map_err(|error| AppError::internal(format!("register WebRTC interceptors: {error}")))?; + let api = APIBuilder::new() + .with_media_engine(media_engine) + .with_interceptor_registry(registry) + .build(); + + let peer_connection = Arc::new( + api.new_peer_connection(RTCConfiguration { + ice_servers: ice_servers(), + ..Default::default() + }) + .await + .map_err(|error| AppError::internal(format!("create WebRTC peer connection: {error}")))?, + ); + register_diagnostics(&peer_connection, &udid); + register_data_channel_streams( + &peer_connection, + state.clone(), + session.clone(), + udid.clone(), + first_frame, + ); + + let offer = RTCSessionDescription::offer(payload.sdp) + .map_err(|error| AppError::bad_request(format!("invalid WebRTC offer: {error}")))?; + peer_connection + .set_remote_description(offer) + .await + .map_err(|error| AppError::bad_request(format!("set remote WebRTC offer: {error}")))?; + + let answer = peer_connection + .create_answer(None) + .await + .map_err(|error| AppError::internal(format!("create WebRTC answer: {error}")))?; + let mut gather_complete = peer_connection.gathering_complete_promise().await; + peer_connection + .set_local_description(answer) + .await + .map_err(|error| AppError::internal(format!("set WebRTC answer: {error}")))?; + let _ = gather_complete.recv().await; + let local_description = peer_connection + .local_description() + .await + .ok_or_else(|| AppError::internal("WebRTC local description was not set."))?; + tokio::spawn(hold_peer_connection_until_closed(peer_connection)); + + Ok(WebRtcAnswerPayload { + sdp: local_description.sdp, + kind: "answer".to_owned(), + }) +} + +async fn hold_peer_connection_until_closed( + peer_connection: Arc, +) { + loop { + if matches!( + peer_connection.connection_state(), + RTCPeerConnectionState::Closed | RTCPeerConnectionState::Failed + ) { + break; + } + time::sleep(Duration::from_secs(5)).await; + } +} + +fn register_data_channel_streams( + peer_connection: &Arc, + state: AppState, + session: crate::simulators::session::SimulatorSession, + udid: String, + first_frame: crate::transport::packet::SharedFrame, +) { + peer_connection.on_data_channel(Box::new(move |channel: Arc| { + let state = state.clone(); + let session = session.clone(); + let udid = udid.clone(); + let first_frame = first_frame.clone(); + Box::pin(async move { + match channel.label() { + WEBRTC_CONTROL_CHANNEL_LABEL => { + attach_control_data_channel(channel, session, udid); + } + WEBRTC_JPEG_CHANNEL_LABEL => { + let state_for_stream = state.clone(); + let session_for_stream = session.clone(); + let udid_for_stream = udid.clone(); + let stream_channel = channel.clone(); + channel.on_open(Box::new(move || { + let channel = stream_channel.clone(); + let state = state_for_stream.clone(); + let session = session_for_stream.clone(); + let udid = udid_for_stream.clone(); + let first_frame = first_frame.clone(); + Box::pin(async move { + tokio::spawn(stream_jpeg_data_channel_frames( + state, + udid, + session, + first_frame, + channel, + )); + }) + })); + } + _ => {} + } + }) + })); +} + fn register_control_data_channel( peer_connection: &Arc, session: crate::simulators::session::SimulatorSession, @@ -284,26 +433,34 @@ fn register_control_data_channel( if channel.label() != WEBRTC_CONTROL_CHANNEL_LABEL { return; } - channel.on_message(Box::new(move |message: DataChannelMessage| { - let session = session.clone(); - let udid = udid.clone(); - Box::pin(async move { - let Ok(text) = std::str::from_utf8(&message.data) else { - warn!("Invalid WebRTC control message bytes for {udid}"); - return; - }; - let control_message = match serde_json::from_str::(text) { - Ok(message) => message, - Err(error) => { - warn!("Invalid WebRTC control message for {udid}: {error}"); - return; - } - }; - if let Err(error) = run_control_message(session, control_message).await { - warn!("WebRTC control message failed for {udid}: {error}"); - } - }) - })); + attach_control_data_channel(channel, session, udid); + }) + })); +} + +fn attach_control_data_channel( + channel: Arc, + session: crate::simulators::session::SimulatorSession, + udid: String, +) { + channel.on_message(Box::new(move |message: DataChannelMessage| { + let session = session.clone(); + let udid = udid.clone(); + Box::pin(async move { + let Ok(text) = std::str::from_utf8(&message.data) else { + warn!("Invalid WebRTC control message bytes for {udid}"); + return; + }; + let control_message = match serde_json::from_str::(text) { + Ok(message) => message, + Err(error) => { + warn!("Invalid WebRTC control message for {udid}: {error}"); + return; + } + }; + if let Err(error) = run_control_message(session, control_message).await { + warn!("WebRTC control message failed for {udid}: {error}"); + } }) })); } @@ -459,6 +616,114 @@ fn adaptive_interval_for_write(write_elapsed: Duration) -> Duration { Duration::from_millis(target_ms) } +async fn stream_jpeg_data_channel_frames( + state: AppState, + udid: String, + session: crate::simulators::session::SimulatorSession, + first_frame: crate::transport::packet::SharedFrame, + channel: Arc, +) { + let mut rx = session.subscribe(); + let _guard = WebRtcMetricsGuard::new(state.metrics.clone()); + + if let Err(error) = write_jpeg_frame_chunks(&channel, &first_frame).await { + warn!("WebRTC JPEG first frame write failed for {udid}: {error}"); + return; + } + let mut last_sequence = first_frame.frame_sequence; + state.metrics.frames_sent.fetch_add(1, Ordering::Relaxed); + + loop { + if channel.ready_state() != RTCDataChannelState::Open { + break; + } + + let frame = match rx.recv().await { + Ok(frame) => frame, + Err(broadcast::error::RecvError::Lagged(skipped)) => { + state + .metrics + .frames_dropped_server + .fetch_add(skipped, Ordering::Relaxed); + continue; + } + Err(broadcast::error::RecvError::Closed) => break, + }; + let (frame, skipped) = freshest_available_frame(frame, &mut rx); + if skipped > 0 { + state + .metrics + .frames_dropped_server + .fetch_add(skipped, Ordering::Relaxed); + } + if frame + .codec + .as_deref() + .is_some_and(|codec| !codec.eq_ignore_ascii_case("jpeg")) + { + warn!("WebRTC JPEG channel received non-JPEG frame for {udid}"); + break; + } + + let buffered = channel.buffered_amount().await; + let max_buffered = (frame.data.len() * JPEG_CHANNEL_BUFFERED_FRAME_MULTIPLIER) + .max(JPEG_CHANNEL_MIN_BUFFERED_BYTES); + if buffered > max_buffered { + state + .metrics + .frames_dropped_server + .fetch_add(1 + skipped, Ordering::Relaxed); + continue; + } + + if last_sequence != 0 && frame.frame_sequence > last_sequence + 1 { + state + .metrics + .frames_dropped_server + .fetch_add(frame.frame_sequence - last_sequence - 1, Ordering::Relaxed); + } + + if let Err(error) = write_jpeg_frame_chunks(&channel, &frame).await { + warn!("WebRTC JPEG frame write failed for {udid}: {error}"); + break; + } + last_sequence = frame.frame_sequence; + state.metrics.frames_sent.fetch_add(1, Ordering::Relaxed); + } +} + +async fn write_jpeg_frame_chunks( + channel: &RTCDataChannel, + frame: &crate::transport::packet::FramePacket, +) -> anyhow::Result<()> { + let payload = frame.data.as_ref(); + let chunk_count = payload.len().div_ceil(JPEG_CHUNK_PAYLOAD_BYTES); + if chunk_count == 0 || chunk_count > u16::MAX as usize { + anyhow::bail!("invalid JPEG frame chunk count {chunk_count}"); + } + + for chunk_index in 0..chunk_count { + let start = chunk_index * JPEG_CHUNK_PAYLOAD_BYTES; + let end = (start + JPEG_CHUNK_PAYLOAD_BYTES).min(payload.len()); + let chunk_payload = &payload[start..end]; + let mut message = Vec::with_capacity(JPEG_CHUNK_HEADER_BYTES + chunk_payload.len()); + message.extend_from_slice(JPEG_CHUNK_MAGIC); + message.push(JPEG_CHUNK_VERSION); + message.push(if frame.is_keyframe { 1 } else { 0 }); + message.extend_from_slice(&(chunk_index as u16).to_be_bytes()); + message.extend_from_slice(&(chunk_count as u16).to_be_bytes()); + message.extend_from_slice(&0u16.to_be_bytes()); + message.extend_from_slice(&frame.frame_sequence.to_be_bytes()); + message.extend_from_slice(&frame.timestamp_us.to_be_bytes()); + message.extend_from_slice(&frame.width.to_be_bytes()); + message.extend_from_slice(&frame.height.to_be_bytes()); + message.extend_from_slice(&(payload.len() as u32).to_be_bytes()); + message.extend_from_slice(chunk_payload); + channel.send(&Bytes::from(message)).await?; + } + Ok(()) +} + async fn write_frame_sample( video_track: &TrackLocalStaticSample, frame: &crate::transport::packet::SharedFrame, diff --git a/skills/simdeck/SKILL.md b/skills/simdeck/SKILL.md index d741444..8897cdb 100644 --- a/skills/simdeck/SKILL.md +++ b/skills/simdeck/SKILL.md @@ -21,12 +21,17 @@ simdeck daemon start simdeck ui --open npm run build:cli && ./build/simdeck ui --open simdeck daemon start --video-codec h264-software +simdeck daemon start --video-codec jpeg simdeck ui --bind 0.0.0.0 --advertise-host 192.168.1.50 --open ``` `simdeck` without a subcommand starts a foreground workspace daemon, prints local and LAN HTTP URLs, prints a six-digit pairing code for LAN browsers, and stops on `q` or Ctrl-C. The optional single argument is a simulator name or UDID to select by default. Use `-d` for detached start, `-k` to kill the background daemon, and `-r` to restart it. Viewer: `http://127.0.0.1:4310` or `http://127.0.0.1:4310?device=`. +Use `http://127.0.0.1:4310?transport=webrtc-data` with `--video-codec jpeg` +for the experimental full-resolution stream on CI hosts without hardware H.264 +or HEVC encode. It defaults to full JPEG quality; lower `--jpeg-quality` values +are available later if runner metrics show bandwidth pressure. The local viewer gets the API token automatically. LAN browsers pair with the printed code before receiving the API cookie. Direct HTTP calls need `X-SimDeck-Token` or `Authorization: Bearer `.