From 54f7ee9e8eb971a615ba2959f781298bd2370441 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 29 Apr 2026 16:04:51 -0400 Subject: [PATCH 1/2] Fix chrome rendering and device family gating --- cli/XCWChromeRenderer.h | 2 + cli/XCWChromeRenderer.m | 703 ++++++++++++++---- cli/native/XCWNativeBridge.h | 1 + cli/native/XCWNativeBridge.m | 20 + client/src/api/controls.ts | 6 +- client/src/api/types.ts | 1 + client/src/app/AppShell.tsx | 81 +- .../src/features/stream/streamWorkerClient.ts | 143 +++- client/src/features/stream/useLiveStream.ts | 20 +- client/src/styles/components.css | 47 +- client/src/styles/layout.css | 51 +- server/src/api/routes.rs | 23 +- server/src/auth.rs | 46 +- server/src/native/bridge.rs | 18 + server/src/native/ffi.rs | 4 + server/src/transport/webrtc.rs | 373 +++++++++- 16 files changed, 1339 insertions(+), 200 deletions(-) diff --git a/cli/XCWChromeRenderer.h b/cli/XCWChromeRenderer.h index 8302091f..d6abd048 100644 --- a/cli/XCWChromeRenderer.h +++ b/cli/XCWChromeRenderer.h @@ -6,6 +6,8 @@ NS_ASSUME_NONNULL_BEGIN + (nullable NSData *)PNGDataForDeviceName:(NSString *)deviceName error:(NSError * _Nullable * _Nullable)error; ++ (nullable NSData *)screenMaskPNGDataForDeviceName:(NSString *)deviceName + error:(NSError * _Nullable * _Nullable)error; + (nullable NSDictionary *)profileForDeviceName:(NSString *)deviceName error:(NSError * _Nullable * _Nullable)error; diff --git a/cli/XCWChromeRenderer.m b/cli/XCWChromeRenderer.m index 5150a966..32be6cb5 100644 --- a/cli/XCWChromeRenderer.m +++ b/cli/XCWChromeRenderer.m @@ -4,7 +4,6 @@ #import static NSString * const XCWChromeRendererErrorDomain = @"SimDeck.ChromeRenderer"; - @implementation XCWChromeRenderer + (nullable NSDictionary *)profileForDeviceName:(NSString *)deviceName @@ -13,91 +12,7 @@ @implementation XCWChromeRenderer if (chromeInfo == nil) { return nil; } - - NSDictionary *plist = chromeInfo[@"plist"]; - NSDictionary *json = chromeInfo[@"json"]; - NSDictionary *images = [json[@"images"] isKindOfClass:[NSDictionary class]] ? json[@"images"] : @{}; - NSDictionary *sizing = [images[@"sizing"] isKindOfClass:[NSDictionary class]] ? images[@"sizing"] : @{}; - NSDictionary *stand = [images[@"stand"] isKindOfClass:[NSDictionary class]] ? images[@"stand"] : @{}; - - CGFloat sizingTop = [self numberValue:sizing[@"topHeight"]]; - CGFloat sizingLeft = [self numberValue:sizing[@"leftWidth"]]; - CGFloat sizingBottom = [self numberValue:sizing[@"bottomHeight"]]; - CGFloat sizingRight = [self numberValue:sizing[@"rightWidth"]]; - CGFloat standHeight = [self numberValue:stand[@"height"]]; - - CGSize compositeSize = [self compositeSizeForChromeInfo:chromeInfo error:error]; - if (CGSizeEqualToSize(compositeSize, CGSizeZero)) { - return nil; - } - - NSDictionary *paths = [json[@"paths"] isKindOfClass:[NSDictionary class]] ? json[@"paths"] : @{}; - NSDictionary *border = [paths[@"simpleOutsideBorder"] isKindOfClass:[NSDictionary class]] ? paths[@"simpleOutsideBorder"] : @{}; - NSDictionary *borderInsets = [border[@"insets"] isKindOfClass:[NSDictionary class]] ? border[@"insets"] : @{}; - CGFloat rawCornerRadius = [self numberValue:border[@"cornerRadiusX"]]; - - CGFloat borderTop = [self numberValue:borderInsets[@"top"]]; - CGFloat borderLeft = [self numberValue:borderInsets[@"left"]]; - CGFloat borderBottom = [self numberValue:borderInsets[@"bottom"]]; - CGFloat borderRight = [self numberValue:borderInsets[@"right"]]; - - CGFloat bezelTop = sizingTop + borderTop; - CGFloat bezelLeft = sizingLeft + borderLeft; - CGFloat bezelBottom = sizingBottom + borderBottom; - CGFloat bezelRight = sizingRight + borderRight; - - BOOL watchProfile = [self isWatchProfile:plist]; - BOOL hasComposite = [self compositeAssetPathForChromeInfo:chromeInfo].length > 0; - CGFloat screenScale = MAX([self numberValue:plist[@"mainScreenScale"]], 1.0); - CGFloat profileScreenWidth = [self numberValue:plist[@"mainScreenWidth"]]; - CGFloat profileScreenHeight = [self numberValue:plist[@"mainScreenHeight"]]; - CGFloat pointScreenWidth = watchProfile ? profileScreenWidth : profileScreenWidth / screenScale; - CGFloat pointScreenHeight = watchProfile ? profileScreenHeight : profileScreenHeight / screenScale; - - CGFloat screenWidth; - CGFloat screenHeight; - CGFloat screenX; - CGFloat screenY; - if (hasComposite && pointScreenWidth > 0.0 && pointScreenHeight > 0.0) { - // The composite PDF defines authoritative chrome dimensions; the screen is the - // device's point size centered horizontally inside the chrome with the bezel - // insets pushing it down vertically (and stand area, if any, occupying the - // bottom of the composite). - screenWidth = pointScreenWidth; - screenHeight = pointScreenHeight; - screenX = MAX((compositeSize.width - screenWidth) / 2.0, 0.0); - CGFloat usableHeight = compositeSize.height - standHeight; - screenY = MAX((usableHeight - screenHeight) / 2.0, bezelTop); - } else if (watchProfile) { - screenWidth = profileScreenWidth; - screenHeight = profileScreenHeight; - screenX = MAX((compositeSize.width - screenWidth) / 2.0, 0.0); - screenY = MAX((compositeSize.height - screenHeight) / 2.0, 0.0); - } else { - // 9-slice path: bezel insets (sizing + simpleOutsideBorder) frame the screen. - // The stand, when present, sits below the chrome and outside the screen rect. - screenX = bezelLeft; - screenY = bezelTop; - screenWidth = MAX(compositeSize.width - bezelLeft - bezelRight, 1.0); - screenHeight = MAX(compositeSize.height - standHeight - bezelTop - bezelBottom, 1.0); - } - - // Inner corner radius: when the thickest bezel exceeds the outer radius, the - // screen edge is past the curved region and the inner is effectively rectangular - // (e.g. iPhone 6s Plus's tall top/bottom bezel collapses the screen rounding). - CGFloat innerRadius = MAX(rawCornerRadius - MAX(bezelLeft, bezelTop), 0.0); - CGFloat radiusScale = pointScreenWidth > 0.0 ? screenWidth / pointScreenWidth : 1.0; - CGFloat cornerRadius = watchProfile ? rawCornerRadius : innerRadius * radiusScale; - - return @{ - @"totalWidth": @(compositeSize.width), - @"totalHeight": @(compositeSize.height), - @"screenX": @(screenX), - @"screenY": @(screenY), - @"screenWidth": @(screenWidth), - @"screenHeight": @(screenHeight), - @"cornerRadius": @(cornerRadius), - }; + return [self profileForChromeInfo:chromeInfo error:error]; } + (nullable NSData *)PNGDataForDeviceName:(NSString *)deviceName @@ -108,14 +23,24 @@ + (nullable NSData *)PNGDataForDeviceName:(NSString *)deviceName } NSString *compositePath = [self compositeAssetPathForChromeInfo:chromeInfo]; - CGSize compositeSize = [self compositeSizeForChromeInfo:chromeInfo error:error]; - if (CGSizeEqualToSize(compositeSize, CGSizeZero)) { + CGSize chromeSize = [self compositeSizeForChromeInfo:chromeInfo error:error]; + if (CGSizeEqualToSize(chromeSize, CGSizeZero)) { + return nil; + } + + NSDictionary *profile = [self profileForChromeInfo:chromeInfo error:error]; + if (profile == nil) { return nil; } + CGSize renderSize = CGSizeMake([self numberValue:profile[@"totalWidth"]], + [self numberValue:profile[@"totalHeight"]]); + CGFloat chromeX = [self numberValue:profile[@"chromeX"]]; + CGFloat chromeY = [self numberValue:profile[@"chromeY"]]; + BOOL drawNonTopInputsBeforeBody = YES; CGFloat scale = 3.0; - NSInteger pixelWidth = MAX((NSInteger)ceil(compositeSize.width * scale), 1); - NSInteger pixelHeight = MAX((NSInteger)ceil(compositeSize.height * scale), 1); + NSInteger pixelWidth = MAX((NSInteger)ceil(renderSize.width * scale), 1); + NSInteger pixelHeight = MAX((NSInteger)ceil(renderSize.height * scale), 1); CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); CGContextRef context = CGBitmapContextCreate(NULL, @@ -140,24 +65,27 @@ + (nullable NSData *)PNGDataForDeviceName:(NSString *)deviceName CGContextSaveGState(context); CGContextTranslateCTM(context, 0, pixelHeight); CGContextScaleCTM(context, scale, -scale); - if (![self drawInputImagesForChromeInfo:chromeInfo - inSize:compositeSize - context:context - onlyOnTop:NO - error:error]) { - CGContextRestoreGState(context); - CGContextRelease(context); - return nil; + CGContextTranslateCTM(context, chromeX, chromeY); + if (drawNonTopInputsBeforeBody) { + if (![self drawInputImagesForChromeInfo:chromeInfo + inSize:chromeSize + context:context + onlyOnTop:NO + error:error]) { + CGContextRestoreGState(context); + CGContextRelease(context); + return nil; + } } BOOL rendered = NO; if (compositePath.length > 0) { rendered = [self drawPDFAtPath:compositePath - inRect:CGRectMake(0, 0, compositeSize.width, compositeSize.height) + inRect:CGRectMake(0, 0, chromeSize.width, chromeSize.height) context:context error:error]; } else { rendered = [self drawSlicedChromeInfo:chromeInfo - inSize:compositeSize + inSize:chromeSize context:context error:error]; } @@ -166,10 +94,29 @@ + (nullable NSData *)PNGDataForDeviceName:(NSString *)deviceName CGContextRelease(context); return nil; } + if (!drawNonTopInputsBeforeBody) { + if (![self drawInputImagesForChromeInfo:chromeInfo + inSize:chromeSize + context:context + onlyOnTop:NO + error:error]) { + CGContextRestoreGState(context); + CGContextRelease(context); + return nil; + } + } + CGContextTranslateCTM(context, -chromeX, -chromeY); + [self clearScreenAreaForProfile:profile context:context]; + if (![self drawSensorBarForChromeInfo:chromeInfo profile:profile context:context error:error]) { + CGContextRestoreGState(context); + CGContextRelease(context); + return nil; + } + CGContextTranslateCTM(context, chromeX, chromeY); if (![self drawInputImagesForChromeInfo:chromeInfo - inSize:compositeSize - context:context - onlyOnTop:YES + inSize:chromeSize + context:context + onlyOnTop:YES error:error]) { CGContextRestoreGState(context); CGContextRelease(context); @@ -226,6 +173,127 @@ + (nullable NSData *)PNGDataForDeviceName:(NSString *)deviceName return data; } ++ (nullable NSData *)screenMaskPNGDataForDeviceName:(NSString *)deviceName + error:(NSError * _Nullable __autoreleasing *)error { + NSDictionary *chromeInfo = [self chromeInfoForDeviceName:deviceName error:error]; + if (chromeInfo == nil) { + return nil; + } + + NSString *maskPath = [self screenMaskPathForChromeInfo:chromeInfo]; + if (maskPath.length == 0) { + if (error != NULL) { + *error = [NSError errorWithDomain:XCWChromeRendererErrorDomain + code:13 + userInfo:@{ + NSLocalizedDescriptionKey: [NSString stringWithFormat:@"The device profile for %@ did not specify a framebuffer mask.", deviceName], + }]; + } + return nil; + } + + return [self PNGDataForPDFAtPath:maskPath scale:1.0 error:error]; +} + ++ (nullable NSDictionary *)profileForChromeInfo:(NSDictionary *)chromeInfo + error:(NSError * _Nullable __autoreleasing *)error { + NSDictionary *plist = chromeInfo[@"plist"]; + NSDictionary *json = chromeInfo[@"json"]; + NSDictionary *images = [json[@"images"] isKindOfClass:[NSDictionary class]] ? json[@"images"] : @{}; + NSDictionary *sizing = [images[@"sizing"] isKindOfClass:[NSDictionary class]] ? images[@"sizing"] : @{}; + NSDictionary *stand = [images[@"stand"] isKindOfClass:[NSDictionary class]] ? images[@"stand"] : @{}; + + CGFloat sizingTop = [self numberValue:sizing[@"topHeight"]]; + CGFloat sizingLeft = [self numberValue:sizing[@"leftWidth"]]; + CGFloat sizingBottom = [self numberValue:sizing[@"bottomHeight"]]; + CGFloat sizingRight = [self numberValue:sizing[@"rightWidth"]]; + CGFloat standHeight = [self numberValue:stand[@"height"]]; + + CGSize compositeSize = [self compositeSizeForChromeInfo:chromeInfo error:error]; + if (CGSizeEqualToSize(compositeSize, CGSizeZero)) { + return nil; + } + + NSDictionary *paths = [json[@"paths"] isKindOfClass:[NSDictionary class]] ? json[@"paths"] : @{}; + NSDictionary *border = [paths[@"simpleOutsideBorder"] isKindOfClass:[NSDictionary class]] ? paths[@"simpleOutsideBorder"] : @{}; + NSDictionary *borderInsets = [border[@"insets"] isKindOfClass:[NSDictionary class]] ? border[@"insets"] : @{}; + CGFloat rawCornerRadius = [self numberValue:border[@"cornerRadiusX"]]; + + CGFloat borderTop = [self numberValue:borderInsets[@"top"]]; + CGFloat borderLeft = [self numberValue:borderInsets[@"left"]]; + CGFloat borderBottom = [self numberValue:borderInsets[@"bottom"]]; + CGFloat borderRight = [self numberValue:borderInsets[@"right"]]; + + CGFloat bezelTop = sizingTop + borderTop; + CGFloat bezelLeft = sizingLeft + borderLeft; + CGFloat bezelBottom = sizingBottom + borderBottom; + CGFloat bezelRight = sizingRight + borderRight; + + BOOL watchProfile = [self isWatchProfile:plist]; + BOOL phoneProfile = [self isPhoneProfile:plist]; + NSString *sensorName = [plist[@"sensorBarImage"] isKindOfClass:[NSString class]] ? plist[@"sensorBarImage"] : @""; + BOOL hasModernPhoneSensor = [self shouldRenderPhoneChromeFromSlices:plist sensorName:sensorName]; + BOOL hasComposite = !hasModernPhoneSensor && [self compositeAssetPathForChromeInfo:chromeInfo].length > 0; + CGFloat screenScale = MAX([self numberValue:plist[@"mainScreenScale"]], 1.0); + CGFloat profileScreenWidth = [self numberValue:plist[@"mainScreenWidth"]]; + CGFloat profileScreenHeight = [self numberValue:plist[@"mainScreenHeight"]]; + CGFloat pointScreenWidth = watchProfile ? profileScreenWidth : profileScreenWidth / screenScale; + CGFloat pointScreenHeight = watchProfile ? profileScreenHeight : profileScreenHeight / screenScale; + + CGFloat screenWidth; + CGFloat screenHeight; + CGFloat screenX; + CGFloat screenY; + if (hasComposite && pointScreenWidth > 0.0 && pointScreenHeight > 0.0) { + screenWidth = pointScreenWidth; + screenHeight = pointScreenHeight; + screenX = MAX((compositeSize.width - screenWidth) / 2.0, 0.0); + CGFloat usableHeight = compositeSize.height - standHeight; + screenY = MAX((usableHeight - screenHeight) / 2.0, bezelTop); + } else if (watchProfile) { + screenWidth = profileScreenWidth; + screenHeight = profileScreenHeight; + screenX = MAX((compositeSize.width - screenWidth) / 2.0, 0.0); + screenY = MAX((compositeSize.height - screenHeight) / 2.0, 0.0); + } else { + screenX = bezelLeft; + screenY = bezelTop; + screenWidth = MAX(compositeSize.width - bezelLeft - bezelRight, 1.0); + screenHeight = MAX(compositeSize.height - standHeight - bezelTop - bezelBottom, 1.0); + } + + CGFloat innerRadius = MAX(rawCornerRadius - MAX(bezelLeft, bezelTop), 0.0); + CGFloat radiusScale = pointScreenWidth > 0.0 ? screenWidth / pointScreenWidth : 1.0; + CGFloat chromeCornerRadius = watchProfile ? rawCornerRadius : innerRadius * radiusScale; + CGFloat cornerRadius = chromeCornerRadius; + CGFloat maskCornerRadius = [self framebufferMaskCornerRadiusForChromeInfo:chromeInfo + pointScreenWidth:pointScreenWidth]; + if (maskCornerRadius > 0.0) { + cornerRadius = maskCornerRadius * radiusScale; + } + + CGRect fullFrame = [self fullFrameForChromeInfo:chromeInfo chromeSize:compositeSize]; + CGFloat chromeX = -CGRectGetMinX(fullFrame); + CGFloat chromeY = -CGRectGetMinY(fullFrame); + BOOL hasScreenMask = !phoneProfile && [self screenMaskPathForChromeInfo:chromeInfo].length > 0; + + return @{ + @"totalWidth": @(CGRectGetWidth(fullFrame)), + @"totalHeight": @(CGRectGetHeight(fullFrame)), + @"chromeX": @(chromeX), + @"chromeY": @(chromeY), + @"chromeWidth": @(compositeSize.width), + @"chromeHeight": @(compositeSize.height), + @"screenX": @(screenX + chromeX), + @"screenY": @(screenY + chromeY), + @"screenWidth": @(screenWidth), + @"screenHeight": @(screenHeight), + @"cornerRadius": @(cornerRadius), + @"chromeCornerRadius": @(chromeCornerRadius), + @"hasScreenMask": @(hasScreenMask), + }; +} + + (nullable NSDictionary *)chromeInfoForDeviceName:(NSString *)deviceName error:(NSError * _Nullable __autoreleasing *)error { NSString *profilePath = [NSString stringWithFormat:@"/Library/Developer/CoreSimulator/Profiles/DeviceTypes/%@.simdevicetype/Contents/Resources/profile.plist", deviceName]; @@ -302,14 +370,17 @@ + (nullable NSDictionary *)chromeInfoForDeviceName:(NSString *)deviceName @"plist": plist, @"json": json, @"chromePath": chromePath, + @"profileResourcesPath": profilePath.stringByDeletingLastPathComponent, }; } + (CGSize)compositeSizeForChromeInfo:(NSDictionary *)chromeInfo error:(NSError * _Nullable __autoreleasing *)error { - NSString *compositePath = [self compositeAssetPathForChromeInfo:chromeInfo]; + NSDictionary *plist = chromeInfo[@"plist"]; + NSString *sensorName = [plist[@"sensorBarImage"] isKindOfClass:[NSString class]] ? plist[@"sensorBarImage"] : @""; + BOOL hasModernPhoneSensor = [self shouldRenderPhoneChromeFromSlices:plist sensorName:sensorName]; + NSString *compositePath = hasModernPhoneSensor ? @"" : [self compositeAssetPathForChromeInfo:chromeInfo]; if (compositePath.length == 0) { - NSDictionary *plist = chromeInfo[@"plist"]; NSDictionary *json = chromeInfo[@"json"]; NSDictionary *images = [json[@"images"] isKindOfClass:[NSDictionary class]] ? json[@"images"] : @{}; NSDictionary *sizing = [images[@"sizing"] isKindOfClass:[NSDictionary class]] ? images[@"sizing"] : @{}; @@ -531,31 +602,7 @@ + (BOOL)drawInputImagesForChromeInfo:(NSDictionary *)chromeInfo continue; } - NSDictionary *offsets = [input[@"offsets"] isKindOfClass:[NSDictionary class]] ? input[@"offsets"] : @{}; - NSDictionary *normalOffset = [offsets[@"normal"] isKindOfClass:[NSDictionary class]] ? offsets[@"normal"] : @{}; - CGFloat offsetX = [self numberValue:normalOffset[@"x"]]; - CGFloat offsetY = [self numberValue:normalOffset[@"y"]]; - NSString *anchor = [input[@"anchor"] isKindOfClass:[NSString class]] ? input[@"anchor"] : @""; - NSString *align = [input[@"align"] isKindOfClass:[NSString class]] ? input[@"align"] : @""; - - CGFloat x = offsetX; - CGFloat y = offsetY; - if ([anchor isEqualToString:@"right"]) { - x = size.width + offsetX; - } else if ([anchor isEqualToString:@"bottom"]) { - y = size.height + offsetY; - } else if ([anchor isEqualToString:@"left"]) { - x = offsetX; - } else if ([anchor isEqualToString:@"top"]) { - y = offsetY; - if ([align isEqualToString:@"trailing"]) { - x = size.width + offsetX; - } else if ([align isEqualToString:@"center"]) { - x = (size.width - assetSize.width) / 2.0 + offsetX; - } - } - - CGRect rect = CGRectMake(x, y, assetSize.width, assetSize.height); + CGRect rect = [self inputFrameForInput:input assetSize:assetSize inSize:size]; if (![self drawPDFAtPath:assetPath inRect:rect context:context error:error]) { return NO; } @@ -563,6 +610,366 @@ + (BOOL)drawInputImagesForChromeInfo:(NSDictionary *)chromeInfo return YES; } ++ (CGRect)fullFrameForChromeInfo:(NSDictionary *)chromeInfo + chromeSize:(CGSize)chromeSize { + CGRect bounds = CGRectMake(0.0, 0.0, chromeSize.width, chromeSize.height); + NSDictionary *json = chromeInfo[@"json"]; + NSString *chromePath = chromeInfo[@"chromePath"]; + BOOL hasComposite = [self compositeAssetPathForChromeInfo:chromeInfo].length > 0; + BOOL watchProfile = [self isWatchProfile:chromeInfo[@"plist"]]; + NSArray *inputs = [json[@"inputs"] isKindOfClass:[NSArray class]] ? json[@"inputs"] : @[]; + for (id inputValue in inputs) { + if (![inputValue isKindOfClass:[NSDictionary class]]) { + continue; + } + NSDictionary *input = inputValue; + BOOL onTop = [input[@"onTop"] respondsToSelector:@selector(boolValue)] && [input[@"onTop"] boolValue]; + if (hasComposite && watchProfile && !onTop) { + continue; + } + NSString *assetName = [input[@"image"] isKindOfClass:[NSString class]] ? input[@"image"] : @""; + if (assetName.length == 0) { + continue; + } + NSString *assetPath = [self resolvedChromeAssetPathForName:assetName chromePath:chromePath]; + CGSize assetSize = [self PDFPageSizeAtPath:assetPath]; + if (assetSize.width <= 0.0 || assetSize.height <= 0.0) { + continue; + } + bounds = CGRectUnion(bounds, [self inputFrameForInput:input assetSize:assetSize inSize:chromeSize]); + } + return CGRectIntegral(bounds); +} + ++ (CGRect)inputFrameForInput:(NSDictionary *)input + assetSize:(CGSize)assetSize + inSize:(CGSize)size { + NSDictionary *offsets = [input[@"offsets"] isKindOfClass:[NSDictionary class]] ? input[@"offsets"] : @{}; + NSDictionary *normalOffset = [offsets[@"normal"] isKindOfClass:[NSDictionary class]] ? offsets[@"normal"] : @{}; + CGFloat offsetX = [self numberValue:normalOffset[@"x"]]; + CGFloat offsetY = [self numberValue:normalOffset[@"y"]]; + NSString *anchor = [input[@"anchor"] isKindOfClass:[NSString class]] ? input[@"anchor"] : @""; + NSString *align = [input[@"align"] isKindOfClass:[NSString class]] ? input[@"align"] : @""; + + CGFloat x = offsetX; + CGFloat y = offsetY; + if ([anchor isEqualToString:@"left"]) { + CGFloat visibleWidth = MAX(assetSize.width - MAX(offsetX, 0.0), 0.0) / 2.0; + x = -visibleWidth; + } else if ([anchor isEqualToString:@"right"]) { + CGFloat visibleWidth = MAX(assetSize.width + MIN(offsetX, 0.0), 0.0) / 2.0; + x = size.width - assetSize.width + visibleWidth; + } else if ([anchor isEqualToString:@"top"]) { + CGFloat visibleHeight = MAX(assetSize.height - MAX(offsetY, 0.0), 0.0) / 2.0; + y = -visibleHeight; + } else if ([anchor isEqualToString:@"bottom"]) { + CGFloat visibleHeight = MAX(assetSize.height + MIN(offsetY, 0.0), 0.0) / 2.0; + y = size.height - assetSize.height + visibleHeight; + } + + if ([anchor isEqualToString:@"left"] || [anchor isEqualToString:@"right"]) { + if ([align isEqualToString:@"center"]) { + y = (size.height - assetSize.height) / 2.0 + offsetY; + } else if ([align isEqualToString:@"trailing"]) { + y = size.height - assetSize.height + offsetY; + } + } else if ([anchor isEqualToString:@"top"] || [anchor isEqualToString:@"bottom"]) { + if ([align isEqualToString:@"center"]) { + x = (size.width - assetSize.width) / 2.0 + offsetX; + } else if ([align isEqualToString:@"trailing"]) { + x = size.width - assetSize.width + offsetX; + } + } else if ([align isEqualToString:@"center"]) { + x = (size.width - assetSize.width) / 2.0 + offsetX; + } else if ([align isEqualToString:@"trailing"]) { + x = size.width - assetSize.width + offsetX; + } + + return CGRectMake(x, y, assetSize.width, assetSize.height); +} + ++ (void)clearScreenAreaForProfile:(NSDictionary *)profile + context:(CGContextRef)context { + CGFloat x = [self numberValue:profile[@"screenX"]]; + CGFloat y = [self numberValue:profile[@"screenY"]]; + CGFloat width = [self numberValue:profile[@"screenWidth"]]; + CGFloat height = [self numberValue:profile[@"screenHeight"]]; + CGFloat radius = [self numberValue:profile[@"chromeCornerRadius"]]; + if (radius <= 0.0) { + radius = [self numberValue:profile[@"cornerRadius"]]; + } + if (width <= 0.0 || height <= 0.0) { + return; + } + + CGRect rect = CGRectMake(x, y, width, height); + CGFloat clampedRadius = MIN(MAX(radius, 0.0), MIN(width, height) / 2.0); + + CGContextSaveGState(context); + CGContextSetBlendMode(context, kCGBlendModeClear); + if (clampedRadius <= 0.0) { + CGContextFillRect(context, rect); + } else { + CGMutablePathRef path = CGPathCreateMutable(); + CGPathMoveToPoint(path, NULL, CGRectGetMinX(rect) + clampedRadius, CGRectGetMinY(rect)); + CGPathAddLineToPoint(path, NULL, CGRectGetMaxX(rect) - clampedRadius, CGRectGetMinY(rect)); + CGPathAddArcToPoint(path, NULL, CGRectGetMaxX(rect), CGRectGetMinY(rect), CGRectGetMaxX(rect), CGRectGetMinY(rect) + clampedRadius, clampedRadius); + CGPathAddLineToPoint(path, NULL, CGRectGetMaxX(rect), CGRectGetMaxY(rect) - clampedRadius); + CGPathAddArcToPoint(path, NULL, CGRectGetMaxX(rect), CGRectGetMaxY(rect), CGRectGetMaxX(rect) - clampedRadius, CGRectGetMaxY(rect), clampedRadius); + CGPathAddLineToPoint(path, NULL, CGRectGetMinX(rect) + clampedRadius, CGRectGetMaxY(rect)); + CGPathAddArcToPoint(path, NULL, CGRectGetMinX(rect), CGRectGetMaxY(rect), CGRectGetMinX(rect), CGRectGetMaxY(rect) - clampedRadius, clampedRadius); + CGPathAddLineToPoint(path, NULL, CGRectGetMinX(rect), CGRectGetMinY(rect) + clampedRadius); + CGPathAddArcToPoint(path, NULL, CGRectGetMinX(rect), CGRectGetMinY(rect), CGRectGetMinX(rect) + clampedRadius, CGRectGetMinY(rect), clampedRadius); + CGPathCloseSubpath(path); + CGContextAddPath(context, path); + CGContextFillPath(context); + CGPathRelease(path); + } + CGContextRestoreGState(context); +} + ++ (BOOL)drawSensorBarForChromeInfo:(NSDictionary *)chromeInfo + profile:(NSDictionary *)profile + context:(CGContextRef)context + error:(NSError * _Nullable __autoreleasing *)error { + NSString *sensorPath = [self sensorBarPathForChromeInfo:chromeInfo]; + if (sensorPath.length == 0) { + return YES; + } + + CGSize sensorSize = [self PDFPageSizeAtPath:sensorPath]; + if (sensorSize.width <= 0.0 || sensorSize.height <= 0.0) { + return YES; + } + + CGFloat screenX = [self numberValue:profile[@"screenX"]]; + CGFloat screenY = [self numberValue:profile[@"screenY"]]; + CGFloat screenWidth = [self numberValue:profile[@"screenWidth"]]; + if (screenWidth <= 0.0) { + return YES; + } + + CGRect rect = CGRectMake(screenX + ((screenWidth - sensorSize.width) / 2.0), + screenY, + sensorSize.width, + sensorSize.height); + return [self drawPDFAtPath:sensorPath inRect:rect context:context error:error]; +} + ++ (NSString *)sensorBarPathForChromeInfo:(NSDictionary *)chromeInfo { + NSDictionary *plist = chromeInfo[@"plist"]; + NSString *resourcesPath = [chromeInfo[@"profileResourcesPath"] isKindOfClass:[NSString class]] ? chromeInfo[@"profileResourcesPath"] : @""; + NSString *sensorName = [plist[@"sensorBarImage"] isKindOfClass:[NSString class]] ? plist[@"sensorBarImage"] : @""; + if (resourcesPath.length == 0 || sensorName.length == 0) { + return @""; + } + + NSString *sensorPath = [resourcesPath stringByAppendingPathComponent:[sensorName stringByAppendingPathExtension:@"pdf"]]; + return [[NSFileManager defaultManager] fileExistsAtPath:sensorPath] ? sensorPath : @""; +} + ++ (NSString *)screenMaskPathForChromeInfo:(NSDictionary *)chromeInfo { + NSDictionary *plist = chromeInfo[@"plist"]; + NSString *resourcesPath = [chromeInfo[@"profileResourcesPath"] isKindOfClass:[NSString class]] ? chromeInfo[@"profileResourcesPath"] : @""; + NSString *maskName = [plist[@"framebufferMask"] isKindOfClass:[NSString class]] ? plist[@"framebufferMask"] : @""; + if (resourcesPath.length == 0 || maskName.length == 0) { + return @""; + } + + NSString *maskPath = [resourcesPath stringByAppendingPathComponent:[maskName stringByAppendingPathExtension:@"pdf"]]; + return [[NSFileManager defaultManager] fileExistsAtPath:maskPath] ? maskPath : @""; +} + ++ (CGFloat)framebufferMaskCornerRadiusForChromeInfo:(NSDictionary *)chromeInfo + pointScreenWidth:(CGFloat)pointScreenWidth { + NSString *maskPath = [self screenMaskPathForChromeInfo:chromeInfo]; + if (maskPath.length == 0 || pointScreenWidth <= 0.0) { + return 0.0; + } + + CGPDFDocumentRef document = CGPDFDocumentCreateWithURL((__bridge CFURLRef)[NSURL fileURLWithPath:maskPath]); + if (document == NULL) { + return 0.0; + } + CGPDFPageRef page = CGPDFDocumentGetPage(document, 1); + if (page == NULL) { + CGPDFDocumentRelease(document); + return 0.0; + } + + CGRect mediaBox = CGPDFPageGetBoxRect(page, kCGPDFMediaBox); + NSInteger width = MAX((NSInteger)ceil(mediaBox.size.width), 1); + NSInteger height = MAX((NSInteger)ceil(mediaBox.size.height), 1); + if (width <= 1 || height <= 1 || width > 4096 || height > 4096) { + CGPDFDocumentRelease(document); + return 0.0; + } + + size_t bytesPerRow = (size_t)width * 4; + NSMutableData *pixels = [NSMutableData dataWithLength:(size_t)height * bytesPerRow]; + CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); + CGContextRef context = CGBitmapContextCreate(pixels.mutableBytes, + (size_t)width, + (size_t)height, + 8, + bytesPerRow, + colorSpace, + kCGImageAlphaPremultipliedLast | kCGBitmapByteOrder32Big); + CGColorSpaceRelease(colorSpace); + if (context == NULL) { + CGPDFDocumentRelease(document); + return 0.0; + } + + CGContextClearRect(context, CGRectMake(0, 0, width, height)); + CGContextSaveGState(context); + CGContextTranslateCTM(context, 0, height); + CGContextScaleCTM(context, (CGFloat)width / MAX(mediaBox.size.width, 1.0), -(CGFloat)height / MAX(mediaBox.size.height, 1.0)); + CGContextTranslateCTM(context, -mediaBox.origin.x, -mediaBox.origin.y); + CGContextDrawPDFPage(context, page); + CGContextRestoreGState(context); + CGContextRelease(context); + CGPDFDocumentRelease(document); + + const unsigned char *bytes = pixels.bytes; + NSInteger topInset = -1; + for (NSInteger x = 0; x < width; x++) { + if (bytes[(size_t)x * 4 + 3] > 127) { + topInset = x; + break; + } + } + NSInteger leftInset = -1; + for (NSInteger y = 0; y < height; y++) { + if (bytes[(size_t)y * bytesPerRow + 3] > 127) { + leftInset = y; + break; + } + } + if (topInset < 0 || leftInset < 0) { + return 0.0; + } + + CGFloat maskRadius = MAX((CGFloat)topInset, (CGFloat)leftInset); + CGFloat maskWidth = MAX(mediaBox.size.width, 1.0); + return maskRadius * pointScreenWidth / maskWidth; +} + ++ (nullable NSData *)PNGDataForPDFAtPath:(NSString *)path + scale:(CGFloat)scale + error:(NSError * _Nullable __autoreleasing *)error { + if (path.length == 0) { + return nil; + } + + CGPDFDocumentRef document = CGPDFDocumentCreateWithURL((__bridge CFURLRef)[NSURL fileURLWithPath:path]); + if (document == NULL) { + if (error != NULL) { + *error = [NSError errorWithDomain:XCWChromeRendererErrorDomain + code:7 + userInfo:@{ + NSLocalizedDescriptionKey: [NSString stringWithFormat:@"Unable to open PDF %@.", path.lastPathComponent], + }]; + } + return nil; + } + CGPDFPageRef page = CGPDFDocumentGetPage(document, 1); + if (page == NULL) { + CGPDFDocumentRelease(document); + if (error != NULL) { + *error = [NSError errorWithDomain:XCWChromeRendererErrorDomain + code:8 + userInfo:@{ + NSLocalizedDescriptionKey: [NSString stringWithFormat:@"PDF %@ did not contain a renderable page.", path.lastPathComponent], + }]; + } + return nil; + } + + CGRect mediaBox = CGPDFPageGetBoxRect(page, kCGPDFMediaBox); + CGFloat renderScale = MAX(scale, 1.0); + NSInteger pixelWidth = MAX((NSInteger)ceil(mediaBox.size.width * renderScale), 1); + NSInteger pixelHeight = MAX((NSInteger)ceil(mediaBox.size.height * renderScale), 1); + + CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); + CGContextRef context = CGBitmapContextCreate(NULL, + pixelWidth, + pixelHeight, + 8, + 0, + colorSpace, + kCGImageAlphaPremultipliedLast | kCGBitmapByteOrder32Big); + CGColorSpaceRelease(colorSpace); + if (context == NULL) { + CGPDFDocumentRelease(document); + if (error != NULL) { + *error = [NSError errorWithDomain:XCWChromeRendererErrorDomain + code:9 + userInfo:@{ + NSLocalizedDescriptionKey: @"Unable to create a CoreGraphics bitmap context for PDF rendering.", + }]; + } + return nil; + } + + CGContextClearRect(context, CGRectMake(0, 0, pixelWidth, pixelHeight)); + CGContextSaveGState(context); + CGContextTranslateCTM(context, 0, pixelHeight); + CGContextScaleCTM(context, + (CGFloat)pixelWidth / MAX(mediaBox.size.width, 1.0), + -((CGFloat)pixelHeight / MAX(mediaBox.size.height, 1.0))); + CGContextTranslateCTM(context, -mediaBox.origin.x, -mediaBox.origin.y); + CGContextDrawPDFPage(context, page); + CGContextRestoreGState(context); + CGPDFDocumentRelease(document); + + CGImageRef image = CGBitmapContextCreateImage(context); + CGContextRelease(context); + if (image == NULL) { + if (error != NULL) { + *error = [NSError errorWithDomain:XCWChromeRendererErrorDomain + code:10 + userInfo:@{ + NSLocalizedDescriptionKey: @"Unable to create a CGImage from the PDF bitmap.", + }]; + } + return nil; + } + + NSMutableData *data = [NSMutableData data]; + CGImageDestinationRef destination = CGImageDestinationCreateWithData((__bridge CFMutableDataRef)data, + CFSTR("public.png"), + 1, + NULL); + if (destination == NULL) { + CGImageRelease(image); + if (error != NULL) { + *error = [NSError errorWithDomain:XCWChromeRendererErrorDomain + code:11 + userInfo:@{ + NSLocalizedDescriptionKey: @"Unable to create a PNG encoder for PDF rendering.", + }]; + } + return nil; + } + + CGImageDestinationAddImage(destination, image, NULL); + BOOL finalized = CGImageDestinationFinalize(destination); + CFRelease(destination); + CGImageRelease(image); + if (!finalized) { + if (error != NULL) { + *error = [NSError errorWithDomain:XCWChromeRendererErrorDomain + code:12 + userInfo:@{ + NSLocalizedDescriptionKey: @"Unable to encode PDF render as PNG.", + }]; + } + return nil; + } + return data; +} + + (BOOL)isWatchProfile:(NSDictionary *)plist { NSString *chromeIdentifier = [plist[@"chromeIdentifier"] isKindOfClass:[NSString class]] ? plist[@"chromeIdentifier"] : @""; if ([chromeIdentifier containsString:@".watch"]) { @@ -578,6 +985,21 @@ + (BOOL)isWatchProfile:(NSDictionary *)plist { return NO; } ++ (BOOL)isPhoneProfile:(NSDictionary *)plist { + NSString *chromeIdentifier = [plist[@"chromeIdentifier"] isKindOfClass:[NSString class]] ? plist[@"chromeIdentifier"] : @""; + if ([chromeIdentifier containsString:@".phone"]) { + return YES; + } + + NSArray *families = [plist[@"supportedProductFamilyIDs"] isKindOfClass:[NSArray class]] ? plist[@"supportedProductFamilyIDs"] : @[]; + for (id family in families) { + if ([family respondsToSelector:@selector(integerValue)] && [family integerValue] == 1) { + return YES; + } + } + return NO; +} + + (CGSize)PDFPageSizeAtPath:(NSString *)path { if (path.length == 0) { return CGSizeZero; @@ -621,6 +1043,7 @@ + (BOOL)drawPDFAtPath:(NSString *)path } CGRect mediaBox = CGPDFPageGetBoxRect(page, kCGPDFMediaBox); CGContextSaveGState(context); + CGContextClipToRect(context, rect); CGContextTranslateCTM(context, rect.origin.x, rect.origin.y + rect.size.height); CGContextScaleCTM(context, rect.size.width / MAX(mediaBox.size.width, 1.0), -rect.size.height / MAX(mediaBox.size.height, 1.0)); CGContextTranslateCTM(context, -mediaBox.origin.x, -mediaBox.origin.y); @@ -631,6 +1054,12 @@ + (BOOL)drawPDFAtPath:(NSString *)path } + (NSString *)compositeAssetPathForChromeInfo:(NSDictionary *)chromeInfo { + NSDictionary *plist = chromeInfo[@"plist"]; + NSString *sensorName = [plist[@"sensorBarImage"] isKindOfClass:[NSString class]] ? plist[@"sensorBarImage"] : @""; + if ([self shouldRenderPhoneChromeFromSlices:plist sensorName:sensorName]) { + return @""; + } + NSDictionary *json = chromeInfo[@"json"]; NSString *chromePath = chromeInfo[@"chromePath"]; NSDictionary *images = [json[@"images"] isKindOfClass:[NSDictionary class]] ? json[@"images"] : @{}; @@ -644,6 +1073,20 @@ + (NSString *)compositeAssetPathForChromeInfo:(NSDictionary *)chromeInfo { return [self resolvedChromeAssetPathForName:name chromePath:chromePath]; } ++ (BOOL)shouldRenderPhoneChromeFromSlices:(NSDictionary *)plist sensorName:(NSString *)sensorName { + if (![self isPhoneProfile:plist]) { + return NO; + } + if (sensorName.length > 0) { + return YES; + } + + NSString *chromeIdentifier = [plist[@"chromeIdentifier"] isKindOfClass:[NSString class]] ? plist[@"chromeIdentifier"] : @""; + return [chromeIdentifier hasSuffix:@".phone11"] || + [chromeIdentifier hasSuffix:@".phone12"] || + [chromeIdentifier hasSuffix:@".phone13"]; +} + + (CGFloat)numberValue:(id)value { if ([value respondsToSelector:@selector(doubleValue)]) { return (CGFloat)[value doubleValue]; diff --git a/cli/native/XCWNativeBridge.h b/cli/native/XCWNativeBridge.h index e81aeaf1..715e1290 100644 --- a/cli/native/XCWNativeBridge.h +++ b/cli/native/XCWNativeBridge.h @@ -41,6 +41,7 @@ bool xcw_native_open_url(const char * _Nonnull udid, const char * _Nonnull url, bool xcw_native_launch_bundle(const char * _Nonnull udid, const char * _Nonnull bundle_id, char * _Nullable * _Nullable error_message); char * _Nullable xcw_native_get_chrome_profile(const char * _Nonnull udid, char * _Nullable * _Nullable error_message); xcw_native_owned_bytes xcw_native_render_chrome_png(const char * _Nonnull udid, char * _Nullable * _Nullable error_message); +xcw_native_owned_bytes xcw_native_render_screen_mask_png(const char * _Nonnull udid, char * _Nullable * _Nullable error_message); xcw_native_owned_bytes xcw_native_screenshot_png(const char * _Nonnull udid, char * _Nullable * _Nullable error_message); char * _Nullable xcw_native_recent_logs(const char * _Nonnull udid, double seconds, size_t limit, char * _Nullable * _Nullable error_message); char * _Nullable xcw_native_accessibility_snapshot(const char * _Nonnull udid, bool has_point, double x, double y, size_t max_depth, char * _Nullable * _Nullable error_message); diff --git a/cli/native/XCWNativeBridge.m b/cli/native/XCWNativeBridge.m index c3c09383..5b7e6e31 100644 --- a/cli/native/XCWNativeBridge.m +++ b/cli/native/XCWNativeBridge.m @@ -201,6 +201,26 @@ xcw_native_owned_bytes xcw_native_render_chrome_png(const char *udid, char **err } } +xcw_native_owned_bytes xcw_native_render_screen_mask_png(const char *udid, char **error_message) { + @autoreleasepool { + NSDictionary *simulator = XCWSimulatorRecordForUDID(udid, error_message); + if (simulator == nil) { + return (xcw_native_owned_bytes){0}; + } + + NSError *renderError = nil; + NSString *deviceName = simulator[@"deviceTypeName"] ?: simulator[@"name"] ?: @""; + NSData *pngData = [XCWChromeRenderer screenMaskPNGDataForDeviceName:deviceName + error:&renderError]; + if (pngData == nil) { + XCWSetErrorMessage(error_message, renderError); + return (xcw_native_owned_bytes){0}; + } + + return XCWOwnedBytesFromData(pngData); + } +} + xcw_native_owned_bytes xcw_native_screenshot_png(const char *udid, char **error_message) { @autoreleasepool { XCWSimctl *simctl = [[XCWSimctl alloc] init]; diff --git a/client/src/api/controls.ts b/client/src/api/controls.ts index b107ee1d..c276caf3 100644 --- a/client/src/api/controls.ts +++ b/client/src/api/controls.ts @@ -1,4 +1,4 @@ -import { apiRequest } from "./client"; +import { accessTokenFromLocation, apiRequest } from "./client"; import type { KeyPayload, LaunchPayload, @@ -61,6 +61,10 @@ export function simulatorControlSocketUrl(udid: string) { `/api/simulators/${encodeURIComponent(udid)}/control`, window.location.href, ); + const token = accessTokenFromLocation(); + if (token) { + url.searchParams.set("simdeckToken", token); + } url.protocol = url.protocol === "https:" ? "wss:" : "ws:"; return url.toString(); } diff --git a/client/src/api/types.ts b/client/src/api/types.ts index 01a79ac3..ed2df21b 100644 --- a/client/src/api/types.ts +++ b/client/src/api/types.ts @@ -32,6 +32,7 @@ export interface ChromeProfile { screenWidth: number; screenHeight: number; cornerRadius: number; + hasScreenMask?: boolean; } export interface AccessibilityFrame { diff --git a/client/src/app/AppShell.tsx b/client/src/app/AppShell.tsx index a4540848..f5771b7a 100644 --- a/client/src/app/AppShell.tsx +++ b/client/src/app/AppShell.tsx @@ -1,5 +1,12 @@ -import { useCallback, useEffect, useRef, useState } from "react"; - +import { + useCallback, + useEffect, + useRef, + useState, + type CSSProperties, +} from "react"; + +import { accessTokenFromLocation } from "../api/client"; import { bootSimulator, dismissKeyboard, @@ -21,6 +28,7 @@ import type { AccessibilitySourcePreference, AccessibilityTreeResponse, ChromeProfile, + SimulatorMetadata, TouchPhase, } from "../api/types"; import { AccessibilityInspector } from "../features/accessibility/AccessibilityInspector"; @@ -28,6 +36,10 @@ import { useKeyboardInput } from "../features/input/useKeyboardInput"; import { usePointerInput } from "../features/input/usePointerInput"; import { simulatorRuntimeLabel } from "../features/simulators/simulatorDisplay"; import { useSimulatorList } from "../features/simulators/useSimulatorList"; +import { + isWebRtcStreamMode, + sendWebRtcControlMessage, +} from "../features/stream/streamWorkerClient"; import { useLiveStream } from "../features/stream/useLiveStream"; import { DebugPanel } from "../features/toolbar/DebugPanel"; import { Toolbar } from "../features/toolbar/Toolbar"; @@ -75,7 +87,38 @@ const LOGICAL_INSPECTOR_MAX_DEPTH = 80; clearLegacyVolatileUiState(); function buildChromeUrl(udid: string, stamp: number): string { - return `${STREAM_ORIGIN}/api/simulators/${udid}/chrome.png?stamp=${stamp}`; + return buildAuthenticatedAssetUrl( + `/api/simulators/${udid}/chrome.png`, + stamp, + ); +} + +function buildScreenMaskUrl(udid: string, stamp: number): string { + return buildAuthenticatedAssetUrl( + `/api/simulators/${udid}/screen-mask.png`, + stamp, + ); +} + +function buildAuthenticatedAssetUrl(path: string, stamp: number): string { + const url = new URL(path, `${STREAM_ORIGIN || window.location.origin}/`); + url.searchParams.set("stamp", String(stamp)); + const token = accessTokenFromLocation(); + if (token) { + url.searchParams.set("simdeckToken", token); + } + return url.toString(); +} + +function shouldRenderNativeChrome(simulator: SimulatorMetadata): boolean { + const identifier = simulator.deviceTypeIdentifier ?? ""; + const name = simulator.name ?? ""; + return ( + identifier.includes(".iPhone-") || + identifier.includes(".iPad-") || + name.startsWith("iPhone") || + name.startsWith("iPad") + ); } function mergeAccessibilitySources( @@ -541,6 +584,10 @@ export function AppShell() { setChromeProfile(null); return; } + if (!shouldRenderNativeChrome(selectedSimulator)) { + setChromeProfile(null); + return; + } try { const profile = await fetchChromeProfile(selectedSimulator.udid); @@ -670,13 +717,30 @@ export function AppShell() { ); const chromeScreenStyle = chromeProfile && chromeScreenRect - ? { + ? ({ left: `${(chromeScreenRect.x / chromeProfile.totalWidth) * 100}%`, top: `${(chromeScreenRect.y / chromeProfile.totalHeight) * 100}%`, width: `${(chromeScreenRect.width / chromeProfile.totalWidth) * 100}%`, height: `${(chromeScreenRect.height / chromeProfile.totalHeight) * 100}%`, borderRadius: chromeScreenBorderRadius ?? "0", - } + ...(chromeProfile.hasScreenMask && selectedSimulator + ? { + maskImage: `url("${buildScreenMaskUrl( + selectedSimulator.udid, + streamStamp, + )}")`, + maskMode: "alpha", + maskRepeat: "no-repeat", + maskSize: "100% 100%", + WebkitMaskImage: `url("${buildScreenMaskUrl( + selectedSimulator.udid, + streamStamp, + )}")`, + WebkitMaskRepeat: "no-repeat", + WebkitMaskSize: "100% 100%", + } + : {}), + } satisfies CSSProperties) : null; const shellStyle = chromeProfile ? { @@ -763,8 +827,11 @@ export function AppShell() { function sendControl(udid: string, message: ControlMessage) { setLocalError(""); - const state = ensureControlSocket(udid); const encoded = JSON.stringify(message); + if (sendWebRtcControlMessage(encoded)) { + return; + } + const state = ensureControlSocket(udid); if (state.socket.readyState === WebSocket.OPEN) { state.socket.send(encoded); } else { @@ -773,7 +840,7 @@ export function AppShell() { } useEffect(() => { - if (selectedSimulator?.isBooted) { + if (selectedSimulator?.isBooted && !isWebRtcStreamMode()) { ensureControlSocket(selectedSimulator.udid); } else { closeControlSocket(); diff --git a/client/src/features/stream/streamWorkerClient.ts b/client/src/features/stream/streamWorkerClient.ts index 4327ac48..80010078 100644 --- a/client/src/features/stream/streamWorkerClient.ts +++ b/client/src/features/stream/streamWorkerClient.ts @@ -6,6 +6,23 @@ import type { WorkerToMainMessage, } from "./streamTypes"; +const HAVE_CURRENT_DATA = 2; +const WEBRTC_CONTROL_CHANNEL_LABEL = "simdeck-control"; + +let activeWebRtcControlChannel: RTCDataChannel | null = null; + +export function isWebRtcStreamMode(): boolean { + return streamTransportMode() === "webrtc" && Boolean(accessTokenFromLocation()); +} + +export function sendWebRtcControlMessage(encoded: string): boolean { + if (activeWebRtcControlChannel?.readyState !== "open") { + return false; + } + activeWebRtcControlChannel.send(encoded); + return true; +} + export function buildStreamTarget(udid: string): StreamConnectTarget { return { udid }; } @@ -61,10 +78,13 @@ class WorkerStreamClient implements StreamClientBackend { class WebRtcStreamClient implements StreamClientBackend { private animationFrame = 0; private canvas: HTMLCanvasElement | null = null; + private connectGeneration = 0; private context: CanvasRenderingContext2D | null = null; + private controlChannel: RTCDataChannel | null = null; private peerConnection: RTCPeerConnection | null = null; private stats: StreamStats = createEmptyStreamStats(); private video: HTMLVideoElement | null = null; + private videoFrameCallback = 0; constructor( private readonly onMessage: (message: WorkerToMainMessage) => void, @@ -93,6 +113,7 @@ class WebRtcStreamClient implements StreamClientBackend { if (!this.canvas || !this.context) { return; } + const generation = ++this.connectGeneration; this.stats = createEmptyStreamStats(); this.onMessage({ type: "status", @@ -103,21 +124,46 @@ class WebRtcStreamClient implements StreamClientBackend { iceServers: iceServers(), }); this.peerConnection = peerConnection; - peerConnection.addTransceiver("video", { direction: "recvonly" }); + const transceiver = peerConnection.addTransceiver("video", { + direction: "recvonly", + }); + configureLowLatencyReceiver(transceiver.receiver); + const controlChannel = peerConnection.createDataChannel( + WEBRTC_CONTROL_CHANNEL_LABEL, + { + ordered: true, + }, + ); + this.controlChannel = controlChannel; + activeWebRtcControlChannel = controlChannel; + controlChannel.addEventListener("close", () => { + if (activeWebRtcControlChannel === controlChannel) { + activeWebRtcControlChannel = null; + } + }); peerConnection.ontrack = (event) => { - const [stream] = event.streams; - if (!stream) { + if (generation !== this.connectGeneration) { return; } + for (const receiver of peerConnection.getReceivers()) { + configureLowLatencyReceiver(receiver); + } + const stream = event.streams[0] ?? new MediaStream([event.track]); const video = document.createElement("video"); video.autoplay = true; video.muted = true; video.playsInline = true; + video.preload = "auto"; video.srcObject = stream; this.video = video; video.onloadedmetadata = () => { - void video.play(); + if (generation !== this.connectGeneration) { + return; + } + void video.play().catch(() => { + // The media stream can be detached during reconnect; retry on the next track. + }); this.syncCanvasSize(video.videoWidth, video.videoHeight); this.onMessage({ type: "video-config", @@ -127,7 +173,7 @@ class WebRtcStreamClient implements StreamClientBackend { type: "status", status: { detail: "WebRTC media connected", state: "streaming" }, }); - this.drawVideoFrame(); + this.scheduleVideoFrame(); }; }; @@ -144,8 +190,14 @@ class WebRtcStreamClient implements StreamClientBackend { }; const offer = await peerConnection.createOffer(); + if (generation !== this.connectGeneration) { + return; + } await peerConnection.setLocalDescription(offer); await waitForIceGathering(peerConnection); + if (generation !== this.connectGeneration) { + return; + } const localDescription = peerConnection.localDescription; if (!localDescription) { throw new Error("WebRTC local offer was not created."); @@ -166,14 +218,27 @@ class WebRtcStreamClient implements StreamClientBackend { throw new Error(await response.text()); } const answer = (await response.json()) as RTCSessionDescriptionInit; + if (generation !== this.connectGeneration) { + return; + } await peerConnection.setRemoteDescription(answer); } disconnect() { + this.connectGeneration += 1; window.cancelAnimationFrame(this.animationFrame); this.animationFrame = 0; + this.cancelVideoFrameCallback(); this.video?.pause(); + if (this.video) { + this.video.srcObject = null; + } this.video = null; + this.controlChannel?.close(); + if (activeWebRtcControlChannel === this.controlChannel) { + activeWebRtcControlChannel = null; + } + this.controlChannel = null; this.peerConnection?.close(); this.peerConnection = null; this.onMessage({ type: "status", status: { state: "idle" } }); @@ -184,18 +249,28 @@ class WebRtcStreamClient implements StreamClientBackend { } private drawVideoFrame = () => { + this.videoFrameCallback = 0; if (!this.canvas || !this.context || !this.video) { return; } - if (this.video.videoWidth > 0 && this.video.videoHeight > 0) { + if ( + this.video.readyState >= HAVE_CURRENT_DATA && + this.video.videoWidth > 0 && + this.video.videoHeight > 0 + ) { this.syncCanvasSize(this.video.videoWidth, this.video.videoHeight); - this.context.drawImage( - this.video, - 0, - 0, - this.canvas.width, - this.canvas.height, - ); + try { + this.context.drawImage( + this.video, + 0, + 0, + this.canvas.width, + this.canvas.height, + ); + } catch { + this.scheduleVideoFrame(); + return; + } this.stats.decodedFrames += 1; this.stats.renderedFrames += 1; this.stats.receivedPackets += 1; @@ -204,9 +279,38 @@ class WebRtcStreamClient implements StreamClientBackend { this.stats.codec = "webrtc"; this.onMessage({ type: "stats", stats: { ...this.stats } }); } - this.animationFrame = window.requestAnimationFrame(this.drawVideoFrame); + this.scheduleVideoFrame(); }; + private scheduleVideoFrame() { + this.cancelVideoFrameCallback(); + if (!this.video) { + return; + } + const video = this.video as HTMLVideoElement & { + requestVideoFrameCallback?: (callback: () => void) => number; + }; + if (video.requestVideoFrameCallback) { + this.videoFrameCallback = video.requestVideoFrameCallback( + this.drawVideoFrame, + ); + return; + } + window.cancelAnimationFrame(this.animationFrame); + this.animationFrame = window.requestAnimationFrame(this.drawVideoFrame); + } + + private cancelVideoFrameCallback() { + if (!this.videoFrameCallback || !this.video) { + return; + } + const video = this.video as HTMLVideoElement & { + cancelVideoFrameCallback?: (handle: number) => void; + }; + video.cancelVideoFrameCallback?.(this.videoFrameCallback); + this.videoFrameCallback = 0; + } + private syncCanvasSize(width: number, height: number) { if (!this.canvas) { return; @@ -222,6 +326,15 @@ class WebRtcStreamClient implements StreamClientBackend { } } +function configureLowLatencyReceiver(receiver: RTCRtpReceiver) { + const lowLatencyReceiver = receiver as RTCRtpReceiver & { + jitterBufferTarget?: number; + }; + if ("jitterBufferTarget" in lowLatencyReceiver) { + lowLatencyReceiver.jitterBufferTarget = 0.03; + } +} + function streamTransportMode(): string { if (typeof window === "undefined") { return "webtransport"; @@ -323,7 +436,7 @@ export class StreamWorkerClient { } private createBackend(canvasElement: HTMLCanvasElement): StreamClientBackend { - if (streamTransportMode() === "webrtc" && accessTokenFromLocation()) { + if (isWebRtcStreamMode()) { return new WebRtcStreamClient(this.onMessage); } void canvasElement; diff --git a/client/src/features/stream/useLiveStream.ts b/client/src/features/stream/useLiveStream.ts index 5fe47e0c..9121604e 100644 --- a/client/src/features/stream/useLiveStream.ts +++ b/client/src/features/stream/useLiveStream.ts @@ -1,5 +1,6 @@ import { useEffect, useRef, useState } from "react"; +import { apiHeaders } from "../../api/client"; import type { SimulatorMetadata } from "../../api/types"; import type { Size } from "../viewport/types"; import { createEmptyStreamStats } from "./stats"; @@ -36,13 +37,25 @@ function createDefaultRuntimeInfo(): StreamRuntimeInfo { gpuRenderer: "", gpuVendor: "", renderBackend: "Unavailable", - streamBackend: "Worker / WebTransport", + streamBackend: streamTransportMode() === "webrtc" + ? "Browser WebRTC" + : "Worker / WebTransport", webCodecs: false, webGL2: false, webTransport: false, }; } +function streamTransportMode(): string { + if (typeof window === "undefined") { + return "webtransport"; + } + return ( + new URLSearchParams(window.location.search).get("transport") ?? + "webtransport" + ); +} + function detectRuntimeInfo(): StreamRuntimeInfo { const runtimeInfo = createDefaultRuntimeInfo(); runtimeInfo.webCodecs = typeof VideoDecoder === "function"; @@ -63,7 +76,8 @@ function detectRuntimeInfo(): StreamRuntimeInfo { } runtimeInfo.webGL2 = true; - runtimeInfo.renderBackend = "WebGL2"; + runtimeInfo.renderBackend = + streamTransportMode() === "webrtc" ? "Canvas 2D / Video" : "WebGL2"; const debugInfo = gl.getExtension("WEBGL_debug_renderer_info"); if (debugInfo) { @@ -293,7 +307,7 @@ export function useLiveStream({ visibilityState: document.visibilityState, }), cache: "no-store", - headers: { "content-type": "application/json" }, + headers: apiHeaders(), method: "POST", }).catch(() => { // Diagnostic only; UI state should never depend on telemetry. diff --git a/client/src/styles/components.css b/client/src/styles/components.css index 0068f9a5..b898cc79 100644 --- a/client/src/styles/components.css +++ b/client/src/styles/components.css @@ -1011,7 +1011,7 @@ object-fit: fill; pointer-events: none; -webkit-user-drag: none; - z-index: 0; + z-index: 2; } .pan-enabled .device-bezel, @@ -1196,6 +1196,51 @@ backdrop-filter: blur(10px); } +@media (max-width: 600px) { + .viewport-zoom-dock { + right: 8px; + bottom: 8px; + left: auto; + width: max-content; + max-width: calc(100vw - 16px); + } + + .zoom-controls-floating { + width: max-content; + max-width: 100%; + justify-content: flex-end; + overflow-x: auto; + padding: 5px; + border-radius: 8px; + } + + .zoom-controls-floating .tbtn { + flex: 0 0 auto; + } + + .debug-overlay { + top: 8px; + right: 8px; + left: 8px; + width: auto; + max-height: calc(100% - 72px); + overflow: auto; + } + + .device-bezel { + border-radius: 34px; + padding: 10px; + } + + .device-screen { + width: min(320px, calc(100vw - 44px)); + } + + .accessibility-rect span { + max-width: 120px; + } +} + @keyframes spin { to { transform: rotate(360deg); diff --git a/client/src/styles/layout.css b/client/src/styles/layout.css index 0ea630f7..77b6c53b 100644 --- a/client/src/styles/layout.css +++ b/client/src/styles/layout.css @@ -2,6 +2,7 @@ display: grid; grid-template-rows: auto 1fr; height: 100vh; + height: 100dvh; overflow: hidden; } @@ -78,6 +79,7 @@ display: flex; align-items: center; gap: 2px; + min-width: 0; } .main { @@ -97,6 +99,48 @@ } @media (max-width: 600px) { + .app { + grid-template-rows: auto minmax(0, 1fr); + } + + .toolbar { + height: auto; + min-height: 40px; + flex-wrap: wrap; + align-items: stretch; + gap: 6px; + padding: 6px; + } + + .toolbar-left, + .toolbar-right { + width: 100%; + } + + .toolbar-left { + display: grid; + grid-template-columns: auto auto minmax(0, 1fr); + } + + .toolbar-right { + overflow-x: auto; + padding-bottom: 2px; + scrollbar-width: none; + } + + .toolbar-right::-webkit-scrollbar { + display: none; + } + + .toolbar-actions { + width: max-content; + min-width: 100%; + } + + .main { + flex-direction: column; + } + .debug-grid { grid-template-columns: 1fr; } @@ -116,9 +160,12 @@ } .hierarchy-panel { - width: min(320px, 74vw); + width: 100%; min-width: 0; - max-width: 74vw; + max-width: none; + max-height: min(42dvh, 360px); + border-right: 0; + border-bottom: 1px solid var(--border); } .toolbar-sim-detail { diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs index c974fb1b..664acbcd 100644 --- a/server/src/api/routes.rs +++ b/server/src/api/routes.rs @@ -103,7 +103,7 @@ struct KeyPayload { rename_all = "camelCase", rename_all_fields = "camelCase" )] -enum ControlMessage { +pub(crate) enum ControlMessage { Touch { x: f64, y: f64, @@ -379,6 +379,10 @@ pub fn router(state: AppState) -> Router { .route("/api/simulators/{udid}/rotate-right", post(rotate_right)) .route("/api/simulators/{udid}/chrome-profile", get(chrome_profile)) .route("/api/simulators/{udid}/chrome.png", get(chrome_png)) + .route( + "/api/simulators/{udid}/screen-mask.png", + get(screen_mask_png), + ) .route( "/api/simulators/{udid}/accessibility-tree", get(accessibility_tree), @@ -421,6 +425,7 @@ async fn require_api_auth( request.method(), request.headers(), peer_is_loopback, + request.uri().query(), ) { return auth::unauthorized_response(&state.config, request.headers()); } @@ -952,7 +957,7 @@ async fn handle_control_socket(state: AppState, udid: String, socket: WebSocket) } } -async fn run_control_message( +pub(crate) async fn run_control_message( session: SimulatorSession, message: ControlMessage, ) -> Result<(), AppError> { @@ -1099,6 +1104,20 @@ async fn chrome_png( Ok((StatusCode::OK, headers, png)) } +async fn screen_mask_png( + State(state): State, + Path(udid): Path, +) -> Result<(StatusCode, HeaderMap, Vec), AppError> { + let png = run_bridge_action(state, move |bridge| bridge.screen_mask_png(&udid)).await?; + let mut headers = HeaderMap::new(); + headers.insert(header::CONTENT_TYPE, "image/png".parse().unwrap()); + headers.insert( + header::CACHE_CONTROL, + "no-cache, no-store, must-revalidate".parse().unwrap(), + ); + Ok((StatusCode::OK, headers, png)) +} + async fn accessibility_tree( State(state): State, Path(udid): Path, diff --git a/server/src/auth.rs b/server/src/auth.rs index afcb7cec..4ae8bed0 100644 --- a/server/src/auth.rs +++ b/server/src/auth.rs @@ -46,11 +46,16 @@ pub fn api_request_authorized( method: &Method, headers: &HeaderMap, peer_is_loopback: bool, + uri_query: Option<&str>, ) -> bool { if bearer_or_header_token(headers).is_some_and(|token| token == config.access_token) { return true; } + if query_token_from_query(uri_query).is_some_and(|token| token == config.access_token) { + return true; + } + if cookie_token(headers).is_some_and(|token| token == config.access_token) { return if method == Method::GET || method == Method::HEAD { origin_allowed_or_absent(config, headers) @@ -162,7 +167,11 @@ fn cookie_token(headers: &HeaderMap) -> Option<&str> { } fn query_token(path: &str) -> Option<&str> { - let query = path.split_once('?')?.1; + query_token_from_query(path.split_once('?').map(|(_, query)| query)) +} + +fn query_token_from_query(query: Option<&str>) -> Option<&str> { + let query = query?; query.split('&').find_map(|part| { let (name, value) = part.split_once('=')?; (name == ACCESS_TOKEN_QUERY).then_some(value) @@ -253,7 +262,22 @@ mod tests { &config, &Method::POST, &headers, - false + false, + None + )); + } + + #[test] + fn accepts_explicit_query_token_for_headerless_browser_requests() { + let config = config(); + let headers = HeaderMap::new(); + + assert!(api_request_authorized( + &config, + &Method::GET, + &headers, + false, + Some("stamp=1&simdeckToken=secret-token") )); } @@ -274,7 +298,8 @@ mod tests { &config, &Method::POST, &headers, - false + false, + None )); } @@ -295,7 +320,8 @@ mod tests { &config, &Method::POST, &headers, - false + false, + None )); } @@ -321,7 +347,8 @@ mod tests { &config, &Method::POST, &headers, - true + true, + None )); } @@ -338,7 +365,8 @@ mod tests { &config, &Method::POST, &headers, - true + true, + None )); } @@ -352,7 +380,8 @@ mod tests { &config, &Method::GET, &headers, - true + true, + None )); } @@ -366,7 +395,8 @@ mod tests { &config, &Method::POST, &headers, - true + true, + None )); } } diff --git a/server/src/native/bridge.rs b/server/src/native/bridge.rs index f9545d4f..c453932e 100644 --- a/server/src/native/bridge.rs +++ b/server/src/native/bridge.rs @@ -113,6 +113,8 @@ pub struct ChromeProfile { pub screen_height: f64, #[serde(rename = "cornerRadius")] pub corner_radius: f64, + #[serde(rename = "hasScreenMask", default)] + pub has_screen_mask: bool, } #[derive(Default, Clone)] @@ -213,6 +215,22 @@ impl NativeBridge { } } + pub fn screen_mask_png(&self, udid: &str) -> Result, AppError> { + let udid = CString::new(udid).map_err(|e| AppError::bad_request(e.to_string()))?; + unsafe { + let mut error = ptr::null_mut(); + let bytes = ffi::xcw_native_render_screen_mask_png(udid.as_ptr(), &mut error); + if bytes.data.is_null() { + return Err( + take_error(error).unwrap_or_else(|| AppError::native("Unknown native error.")) + ); + } + let data = std::slice::from_raw_parts(bytes.data, bytes.length).to_vec(); + ffi::xcw_native_free_bytes(bytes); + Ok(data) + } + } + pub fn screenshot_png(&self, udid: &str) -> Result, AppError> { let udid = CString::new(udid).map_err(|e| AppError::bad_request(e.to_string()))?; unsafe { diff --git a/server/src/native/ffi.rs b/server/src/native/ffi.rs index d4b8bd3a..7312a7f3 100644 --- a/server/src/native/ffi.rs +++ b/server/src/native/ffi.rs @@ -62,6 +62,10 @@ unsafe extern "C" { udid: *const c_char, error_message: *mut *mut c_char, ) -> xcw_native_owned_bytes; + pub fn xcw_native_render_screen_mask_png( + udid: *const c_char, + error_message: *mut *mut c_char, + ) -> xcw_native_owned_bytes; pub fn xcw_native_screenshot_png( udid: *const c_char, error_message: *mut *mut c_char, diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs index 0fe84d62..513e7b63 100644 --- a/server/src/transport/webrtc.rs +++ b/server/src/transport/webrtc.rs @@ -1,15 +1,18 @@ -use crate::api::routes::AppState; +use crate::api::routes::{run_control_message, AppState, ControlMessage}; use crate::error::AppError; use bytes::Bytes; use serde::{Deserialize, Serialize}; use std::sync::atomic::Ordering; use std::sync::Arc; use std::time::Duration; -use tokio::sync::broadcast; +use tokio::sync::broadcast::{self, error::TryRecvError}; +use tokio::time; use tracing::warn; use webrtc::api::interceptor_registry::register_default_interceptors; use webrtc::api::media_engine::{MediaEngine, MIME_TYPE_H264}; use webrtc::api::APIBuilder; +use webrtc::data_channel::data_channel_message::DataChannelMessage; +use webrtc::data_channel::RTCDataChannel; use webrtc::ice_transport::ice_server::RTCIceServer; use webrtc::interceptor::registry::Registry; use webrtc::media::Sample; @@ -19,7 +22,11 @@ use webrtc::rtp_transceiver::rtp_codec::RTCRtpCodecCapability; use webrtc::track::track_local::track_local_static_sample::TrackLocalStaticSample; use webrtc::track::track_local::TrackLocal; +const ANNEX_B_START_CODE: &[u8] = &[0, 0, 0, 1]; const DEFAULT_STUN_URL: &str = "stun:stun.l.google.com:19302"; +const WEBRTC_CONTROL_CHANNEL_LABEL: &str = "simdeck-control"; +const WEBRTC_BOOTSTRAP_KEYFRAME_INTERVAL: Duration = Duration::from_millis(250); +const WEBRTC_BOOTSTRAP_KEYFRAME_REPEATS: u8 = 12; #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] @@ -64,7 +71,7 @@ pub async fn create_answer( .as_deref() .unwrap_or_default() .to_lowercase(); - if !codec.contains("h264") { + if !is_h264_codec(&codec) { return Err(AppError::bad_request( "WebRTC preview requires H.264. Restart SimDeck with `--video-codec h264-software`.", )); @@ -90,14 +97,14 @@ pub async fn create_answer( .await .map_err(|error| AppError::internal(format!("create WebRTC peer connection: {error}")))?, ); + register_control_data_channel(&peer_connection, session.clone(), udid.clone()); let video_track = Arc::new(TrackLocalStaticSample::new( RTCRtpCodecCapability { mime_type: MIME_TYPE_H264.to_owned(), clock_rate: 90_000, channels: 0, - sdp_fmtp_line: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f" - .to_owned(), + sdp_fmtp_line: h264_sdp_fmtp_line(&codec), rtcp_feedback: vec![], }, "simdeck-video".to_owned(), @@ -150,6 +157,58 @@ pub async fn create_answer( }) } +fn register_control_data_channel( + peer_connection: &Arc, + session: crate::simulators::session::SimulatorSession, + udid: String, +) { + peer_connection.on_data_channel(Box::new(move |channel: Arc| { + let session = session.clone(); + let udid = udid.clone(); + Box::pin(async move { + if channel.label() != WEBRTC_CONTROL_CHANNEL_LABEL { + return; + } + channel.on_message(Box::new(move |message: DataChannelMessage| { + let session = session.clone(); + let udid = udid.clone(); + Box::pin(async move { + let Ok(text) = std::str::from_utf8(&message.data) else { + warn!("Invalid WebRTC control message bytes for {udid}"); + return; + }; + let control_message = match serde_json::from_str::(text) { + Ok(message) => message, + Err(error) => { + warn!("Invalid WebRTC control message for {udid}: {error}"); + return; + } + }; + if let Err(error) = run_control_message(session, control_message).await { + warn!("WebRTC control message failed for {udid}: {error}"); + } + }) + })); + }) + })); +} + +fn is_h264_codec(codec: &str) -> bool { + let codec = codec.trim().to_ascii_lowercase(); + codec.contains("h264") || codec.starts_with("avc1.") || codec.starts_with("avc3.") +} + +fn h264_sdp_fmtp_line(codec: &str) -> String { + let profile_level_id = codec + .split_once('.') + .map(|(_, value)| value) + .filter(|value| value.len() >= 6) + .map(|value| &value[..6]) + .filter(|value| value.chars().all(|ch| ch.is_ascii_hexdigit())) + .unwrap_or("42e01f"); + format!("level-asymmetry-allowed=1;packetization-mode=1;profile-level-id={profile_level_id}") +} + fn ice_servers() -> Vec { let mut urls = std::env::var("SIMDECK_WEBRTC_ICE_SERVERS") .ok() @@ -178,29 +237,71 @@ async fn stream_h264_frames( video_track: Arc, ) { let mut rx = session.subscribe(); + let mut latest_keyframe = first_frame; + let mut last_sequence = 0u64; + let mut send_timing = WebRtcSendTiming::new(); + let mut bootstrap_interval = time::interval(WEBRTC_BOOTSTRAP_KEYFRAME_INTERVAL); + let mut bootstrap_frames_remaining = WEBRTC_BOOTSTRAP_KEYFRAME_REPEATS; let _guard = WebRtcMetricsGuard::new(state.metrics.clone()); - if let Err(error) = write_frame_sample(&video_track, &first_frame).await { - warn!("failed to write initial WebRTC frame for {udid}: {error}"); - } loop { - let frame = match rx.recv().await { - Ok(frame) => frame, - Err(broadcast::error::RecvError::Lagged(skipped)) => { - state - .metrics - .frames_dropped_server - .fetch_add(skipped, Ordering::Relaxed); - session.request_refresh(); - continue; + tokio::select! { + _ = bootstrap_interval.tick(), if bootstrap_frames_remaining > 0 => { + if let Err(error) = write_frame_sample( + &video_track, + &latest_keyframe, + WEBRTC_BOOTSTRAP_KEYFRAME_INTERVAL, + ).await { + warn!("WebRTC bootstrap keyframe write failed for {udid}: {error}"); + break; + } + state.metrics.frames_sent.fetch_add(1, Ordering::Relaxed); + bootstrap_frames_remaining = bootstrap_frames_remaining.saturating_sub(1); + } + frame = rx.recv() => { + let frame = match frame { + Ok(frame) => frame, + Err(broadcast::error::RecvError::Lagged(skipped)) => { + state + .metrics + .frames_dropped_server + .fetch_add(skipped, Ordering::Relaxed); + session.request_refresh(); + continue; + } + Err(broadcast::error::RecvError::Closed) => break, + }; + let (frame, skipped) = freshest_available_frame(frame, &mut rx); + if skipped > 0 { + state + .metrics + .frames_dropped_server + .fetch_add(skipped, Ordering::Relaxed); + if !frame.is_keyframe { + session.request_refresh(); + continue; + } + } + if last_sequence != 0 && frame.frame_sequence > last_sequence + 1 && !frame.is_keyframe { + state + .metrics + .frames_dropped_server + .fetch_add(frame.frame_sequence - last_sequence - 1, Ordering::Relaxed); + session.request_refresh(); + continue; + } + if frame.is_keyframe { + latest_keyframe = frame.clone(); + } + let duration = send_timing.duration_for(&frame); + if let Err(error) = write_frame_sample(&video_track, &frame, duration).await { + warn!("WebRTC frame write failed for {udid}: {error}"); + break; + } + last_sequence = frame.frame_sequence; + state.metrics.frames_sent.fetch_add(1, Ordering::Relaxed); } - Err(broadcast::error::RecvError::Closed) => break, - }; - if let Err(error) = write_frame_sample(&video_track, &frame).await { - warn!("WebRTC frame write failed for {udid}: {error}"); - break; } - state.metrics.frames_sent.fetch_add(1, Ordering::Relaxed); } let _ = peer_connection.close().await; @@ -209,24 +310,170 @@ async fn stream_h264_frames( async fn write_frame_sample( video_track: &TrackLocalStaticSample, frame: &crate::transport::packet::SharedFrame, + duration: Duration, ) -> anyhow::Result<()> { - let mut data = Vec::new(); - if frame.is_keyframe { - if let Some(description) = frame.description.as_ref() { - data.extend_from_slice(description.as_slice()); - } - } - data.extend_from_slice(frame.data.as_slice()); + let data = h264_annex_b_sample(frame)?; video_track .write_sample(&Sample { data: Bytes::from(data), - duration: Duration::from_millis(16), + duration, ..Default::default() }) .await?; Ok(()) } +fn h264_annex_b_sample(frame: &crate::transport::packet::FramePacket) -> anyhow::Result> { + let data = frame.data.as_slice(); + let description = frame.description.as_ref().map(|bytes| bytes.as_slice()); + let mut sample = Vec::with_capacity(data.len() + description.map_or(0, |bytes| bytes.len())); + + if frame.is_keyframe { + if let Some(avcc) = description { + append_avcc_parameter_sets(avcc, &mut sample)?; + } + } + + if is_annex_b(data) { + sample.extend_from_slice(data); + return Ok(sample); + } + + let nal_length_size = description.and_then(avcc_nal_length_size).unwrap_or(4); + append_length_prefixed_nalus(data, nal_length_size, &mut sample)?; + Ok(sample) +} + +fn is_annex_b(data: &[u8]) -> bool { + data.starts_with(&[0, 0, 1]) || data.starts_with(ANNEX_B_START_CODE) +} + +fn avcc_nal_length_size(avcc: &[u8]) -> Option { + if avcc.len() < 5 { + return None; + } + Some(((avcc[4] & 0x03) + 1) as usize) +} + +fn append_avcc_parameter_sets(avcc: &[u8], output: &mut Vec) -> anyhow::Result<()> { + if avcc.len() < 7 { + return Ok(()); + } + + let sps_count = (avcc[5] & 0x1f) as usize; + let mut offset = 6usize; + for _ in 0..sps_count { + append_avcc_nal(avcc, &mut offset, output)?; + } + + if offset >= avcc.len() { + return Ok(()); + } + + let pps_count = avcc[offset] as usize; + offset += 1; + for _ in 0..pps_count { + append_avcc_nal(avcc, &mut offset, output)?; + } + Ok(()) +} + +fn append_avcc_nal(avcc: &[u8], offset: &mut usize, output: &mut Vec) -> anyhow::Result<()> { + if *offset + 2 > avcc.len() { + anyhow::bail!("truncated H.264 decoder configuration record"); + } + let length = u16::from_be_bytes([avcc[*offset], avcc[*offset + 1]]) as usize; + *offset += 2; + if *offset + length > avcc.len() { + anyhow::bail!("truncated H.264 decoder configuration NAL unit"); + } + if length > 0 { + output.extend_from_slice(ANNEX_B_START_CODE); + output.extend_from_slice(&avcc[*offset..*offset + length]); + } + *offset += length; + Ok(()) +} + +fn append_length_prefixed_nalus( + data: &[u8], + nal_length_size: usize, + output: &mut Vec, +) -> anyhow::Result<()> { + if !(1..=4).contains(&nal_length_size) { + anyhow::bail!("invalid H.264 NAL length size {nal_length_size}"); + } + + let mut offset = 0usize; + while offset < data.len() { + if offset + nal_length_size > data.len() { + anyhow::bail!("truncated H.264 NAL length prefix"); + } + + let mut length = 0usize; + for byte in &data[offset..offset + nal_length_size] { + length = (length << 8) | (*byte as usize); + } + offset += nal_length_size; + if length == 0 { + continue; + } + if offset + length > data.len() { + anyhow::bail!("truncated H.264 NAL unit"); + } + output.extend_from_slice(ANNEX_B_START_CODE); + output.extend_from_slice(&data[offset..offset + length]); + offset += length; + } + Ok(()) +} + +fn freshest_available_frame( + mut frame: crate::transport::packet::SharedFrame, + rx: &mut broadcast::Receiver, +) -> (crate::transport::packet::SharedFrame, u64) { + let mut skipped = 0u64; + loop { + match rx.try_recv() { + Ok(next) => { + skipped += 1; + frame = next; + } + Err(TryRecvError::Lagged(count)) => { + skipped += count; + } + Err(TryRecvError::Empty) | Err(TryRecvError::Closed) => return (frame, skipped), + } + } +} + +struct WebRtcSendTiming { + last_timestamp_us: Option, +} + +impl WebRtcSendTiming { + fn new() -> Self { + Self { + last_timestamp_us: None, + } + } + + fn duration_for(&mut self, frame: &crate::transport::packet::FramePacket) -> Duration { + const MIN_FRAME_DURATION_US: u64 = 1_000; + const DEFAULT_FRAME_DURATION_US: u64 = 16_667; + const MAX_FRAME_DURATION_US: u64 = 100_000; + + let duration_us = self + .last_timestamp_us + .and_then(|previous| frame.timestamp_us.checked_sub(previous)) + .filter(|duration| *duration > 0) + .unwrap_or(DEFAULT_FRAME_DURATION_US) + .clamp(MIN_FRAME_DURATION_US, MAX_FRAME_DURATION_US); + self.last_timestamp_us = Some(frame.timestamp_us); + Duration::from_micros(duration_us) + } +} + struct WebRtcMetricsGuard { metrics: Arc, } @@ -253,3 +500,67 @@ impl Drop for WebRtcMetricsGuard { ); } } + +#[cfg(test)] +mod tests { + use super::{ + append_avcc_parameter_sets, append_length_prefixed_nalus, h264_sdp_fmtp_line, is_annex_b, + is_h264_codec, ANNEX_B_START_CODE, + }; + + #[test] + fn accepts_browser_h264_codec_strings() { + assert!(is_h264_codec("h264")); + assert!(is_h264_codec("avc1.42e01f")); + assert!(is_h264_codec("avc3.640028")); + assert!(!is_h264_codec("hvc1.1.6.L123.B0")); + assert!(!is_h264_codec("")); + } + + #[test] + fn uses_h264_profile_level_id_when_available() { + assert!(h264_sdp_fmtp_line("avc1.42e01f").contains("profile-level-id=42e01f")); + assert!(h264_sdp_fmtp_line("h264").contains("profile-level-id=42e01f")); + } + + #[test] + fn converts_avcc_parameter_sets_to_annex_b() { + let avcc = [ + 1, 0x42, 0xe0, 0x1f, 0xff, 0xe1, 0, 3, 0x67, 0x42, 0x00, 1, 0, 2, 0x68, 0xce, + ]; + let mut output = Vec::new(); + + append_avcc_parameter_sets(&avcc, &mut output).unwrap(); + + assert_eq!( + output, + [ + ANNEX_B_START_CODE, + &[0x67, 0x42, 0x00], + ANNEX_B_START_CODE, + &[0x68, 0xce], + ] + .concat() + ); + } + + #[test] + fn converts_length_prefixed_h264_sample_to_annex_b() { + let sample = [0, 0, 0, 2, 0x65, 0x88, 0, 0, 0, 2, 0x41, 0x9a]; + let mut output = Vec::new(); + + append_length_prefixed_nalus(&sample, 4, &mut output).unwrap(); + + assert_eq!( + output, + [ + ANNEX_B_START_CODE, + &[0x65, 0x88], + ANNEX_B_START_CODE, + &[0x41, 0x9a], + ] + .concat() + ); + assert!(is_annex_b(&output)); + } +} From eee160c3c603adcba19e99eeed973d8351ee6838 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 29 Apr 2026 16:09:42 -0400 Subject: [PATCH 2/2] Fix stream formatting --- client/src/features/stream/streamWorkerClient.ts | 4 +++- client/src/features/stream/useLiveStream.ts | 7 ++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/client/src/features/stream/streamWorkerClient.ts b/client/src/features/stream/streamWorkerClient.ts index 80010078..dfcaf81d 100644 --- a/client/src/features/stream/streamWorkerClient.ts +++ b/client/src/features/stream/streamWorkerClient.ts @@ -12,7 +12,9 @@ const WEBRTC_CONTROL_CHANNEL_LABEL = "simdeck-control"; let activeWebRtcControlChannel: RTCDataChannel | null = null; export function isWebRtcStreamMode(): boolean { - return streamTransportMode() === "webrtc" && Boolean(accessTokenFromLocation()); + return ( + streamTransportMode() === "webrtc" && Boolean(accessTokenFromLocation()) + ); } export function sendWebRtcControlMessage(encoded: string): boolean { diff --git a/client/src/features/stream/useLiveStream.ts b/client/src/features/stream/useLiveStream.ts index 9121604e..38323a33 100644 --- a/client/src/features/stream/useLiveStream.ts +++ b/client/src/features/stream/useLiveStream.ts @@ -37,9 +37,10 @@ function createDefaultRuntimeInfo(): StreamRuntimeInfo { gpuRenderer: "", gpuVendor: "", renderBackend: "Unavailable", - streamBackend: streamTransportMode() === "webrtc" - ? "Browser WebRTC" - : "Worker / WebTransport", + streamBackend: + streamTransportMode() === "webrtc" + ? "Browser WebRTC" + : "Worker / WebTransport", webCodecs: false, webGL2: false, webTransport: false,