From b49a34b1f735c177bd9ff06163908013f001d72a Mon Sep 17 00:00:00 2001 From: Jason Lu Date: Mon, 25 May 2026 04:03:37 -0700 Subject: [PATCH 1/3] feat(macos/capture): add ScreenCaptureKit backend, runtime-select on 12.3+ AVCaptureScreenInput was deprecated in macOS 13 (October 2022) and is fundamentally limited to 8-bit BGRA, blocking any honest HDR or 10-bit work on the macOS capture path. ScreenCaptureKit has been available since macOS 12.3 (March 2022) and is the only forward path; this commit lays the foundation by adding a drop-in SCK-based backend that preserves behaviour exactly (same pixel format, frame rate, display selection) so it can be reviewed independently of the HDR work that builds on top. Changes: * Add SunshineVideoCapture protocol in av_video.h declaring the capture-side surface both backends expose. * Make AVVideo conform to the protocol (no behaviour change; pure declaration). * Add SCVideo (sc_video.h / sc_video.m) implementing the same protocol against SCStream + SCContentFilter + SCStreamConfiguration. Built with -fobjc-arc for SCK's block-heavy API surface; objects cross the MRC boundary via the standard +1-retain alloc/init convention so display.mm continues to work in MRC. * Drop incomplete frames from SCK output by inspecting SCStreamFrameInfoStatus on each sample-buffer attachment, matching the reliability the legacy path got for free from AVCaptureSession. * display.mm now holds an id and branches at construction via @available(macOS 12.3, *): SCVideo on supported systems, AVVideo as fallback for older macOS. * Wire ScreenCaptureKit framework into cmake/dependencies/macos.cmake and cmake/compile_definitions/macos.cmake; set ARC compile flag on sc_video.m only. Pixel format stays 32BGRA for this commit; 10-bit + EDR metadata follow in a subsequent change. --- cmake/compile_definitions/macos.cmake | 11 + cmake/dependencies/macos.cmake | 8 + src/platform/macos/av_video.h | 25 +- src/platform/macos/display.mm | 20 +- src/platform/macos/sc_video.h | 30 +++ src/platform/macos/sc_video.m | 350 ++++++++++++++++++++++++++ 6 files changed, 437 insertions(+), 7 deletions(-) create mode 100644 src/platform/macos/sc_video.h create mode 100644 src/platform/macos/sc_video.m diff --git a/cmake/compile_definitions/macos.cmake b/cmake/compile_definitions/macos.cmake index dbca9df9073..15a8bbbe8c7 100644 --- a/cmake/compile_definitions/macos.cmake +++ b/cmake/compile_definitions/macos.cmake @@ -35,6 +35,7 @@ list(APPEND SUNSHINE_EXTERNAL_LIBRARIES ${CORE_MEDIA_LIBRARY} ${CORE_VIDEO_LIBRARY} ${FOUNDATION_LIBRARY} + ${SCREEN_CAPTURE_KIT_LIBRARY} ${VIDEO_TOOLBOX_LIBRARY}) set(APPLE_PLIST_TEMPLATE "${SUNSHINE_SOURCE_ASSETS_DIR}/macos/build/Info.plist.in") @@ -55,6 +56,16 @@ set(PLATFORM_TARGET_FILES "${CMAKE_SOURCE_DIR}/src/platform/macos/nv12_zero_device.cpp" "${CMAKE_SOURCE_DIR}/src/platform/macos/nv12_zero_device.h" "${CMAKE_SOURCE_DIR}/src/platform/macos/publish.cpp" + "${CMAKE_SOURCE_DIR}/src/platform/macos/sc_video.h" + "${CMAKE_SOURCE_DIR}/src/platform/macos/sc_video.m" "${CMAKE_SOURCE_DIR}/third-party/TPCircularBuffer/TPCircularBuffer.c" "${CMAKE_SOURCE_DIR}/third-party/TPCircularBuffer/TPCircularBuffer.h" ${APPLE_PLIST_FILE}) + +# sc_video.m is written against ARC for clarity (SCK APIs are async/ +# block-heavy and benefit from ARC). The rest of the macOS Obj-C +# sources remain MRC; objects flowing across the boundary follow the +# standard +1-retain alloc/init convention so both modes interoperate. +set_source_files_properties( + "${CMAKE_SOURCE_DIR}/src/platform/macos/sc_video.m" + PROPERTIES COMPILE_FLAGS "-fobjc-arc") diff --git a/cmake/dependencies/macos.cmake b/cmake/dependencies/macos.cmake index 5e225fdac21..846bf78d176 100644 --- a/cmake/dependencies/macos.cmake +++ b/cmake/dependencies/macos.cmake @@ -10,6 +10,14 @@ FIND_LIBRARY(CORE_MEDIA_LIBRARY CoreMedia) FIND_LIBRARY(CORE_VIDEO_LIBRARY CoreVideo) FIND_LIBRARY(FOUNDATION_LIBRARY Foundation) FIND_LIBRARY(VIDEO_TOOLBOX_LIBRARY VideoToolbox) +# ScreenCaptureKit is the modern (macOS 12.3+) replacement for the +# deprecated AVCaptureScreenInput-based capture path. Sunshine's +# sc_video.{h,m} is unconditionally compiled into the macOS target; +# fail configure with a clear message rather than failing the build +# later on header lookup when the SDK doesn't ship the framework +# (e.g., when building with an Xcode older than 13.3 / SDK older than +# 12.3, which dropped out of routine compatibility long ago). +FIND_LIBRARY(SCREEN_CAPTURE_KIT_LIBRARY ScreenCaptureKit REQUIRED) if(SUNSHINE_ENABLE_TRAY) FIND_LIBRARY(COCOA Cocoa REQUIRED) diff --git a/src/platform/macos/av_video.h b/src/platform/macos/av_video.h index b2fa5d4b255..94d5e2db565 100644 --- a/src/platform/macos/av_video.h +++ b/src/platform/macos/av_video.h @@ -15,7 +15,17 @@ struct CaptureSession { static const int kMaxDisplays = 32; -@interface AVVideo: NSObject +typedef bool (^FrameCallbackBlock)(CMSampleBufferRef); + +/** + * @brief Shared interface for macOS screen capture backends. + * + * Both the legacy AVCaptureScreenInput-based implementation (AVVideo) and + * the modern ScreenCaptureKit-based implementation (SCVideo) conform to + * this protocol so display.mm can hold either behind a single pointer + * type and branch on macOS version at construction. + */ +@protocol SunshineVideoCapture @property (nonatomic, assign) CGDirectDisplayID displayID; @property (nonatomic, assign) CMTime minFrameDuration; @@ -23,7 +33,18 @@ static const int kMaxDisplays = 32; @property (nonatomic, assign) int frameWidth; @property (nonatomic, assign) int frameHeight; -typedef bool (^FrameCallbackBlock)(CMSampleBufferRef); +- (void)setFrameWidth:(int)frameWidth frameHeight:(int)frameHeight; +- (dispatch_semaphore_t)capture:(FrameCallbackBlock)frameCallback; + +@end + +@interface AVVideo: NSObject + +@property (nonatomic, assign) CGDirectDisplayID displayID; +@property (nonatomic, assign) CMTime minFrameDuration; +@property (nonatomic, assign) OSType pixelFormat; +@property (nonatomic, assign) int frameWidth; +@property (nonatomic, assign) int frameHeight; @property (nonatomic, assign) AVCaptureSession *session; @property (nonatomic, assign) NSMapTable *videoOutputs; diff --git a/src/platform/macos/display.mm b/src/platform/macos/display.mm index be124b2d331..9f6c8ef5ccd 100644 --- a/src/platform/macos/display.mm +++ b/src/platform/macos/display.mm @@ -10,6 +10,7 @@ #include "src/platform/macos/av_video.h" #include "src/platform/macos/misc.h" #include "src/platform/macos/nv12_zero_device.h" +#include "src/platform/macos/sc_video.h" // Avoid conflict between AVFoundation and libavutil both defining AVMediaType #define AVMediaType AVMediaType_FFmpeg @@ -22,7 +23,7 @@ using namespace std::literals; struct av_display_t: public display_t { - AVVideo *av_capture {}; + id av_capture {}; CGDirectDisplayID display_id {}; ~av_display_t() override { @@ -86,7 +87,7 @@ capture_e capture(const push_captured_image_cb_t &push_captured_image_cb, const } else if (pix_fmt == pix_fmt_e::nv12 || pix_fmt == pix_fmt_e::p010) { auto device = std::make_unique(); - device->init(static_cast(av_capture), pix_fmt, setResolution, setPixelFormat); + device->init((void *) av_capture, pix_fmt, setResolution, setPixelFormat); return device; } else { @@ -143,11 +144,11 @@ int dummy_img(img_t *img) override { * height --> the intended capture height */ static void setResolution(void *display, int width, int height) { - [static_cast(display) setFrameWidth:width frameHeight:height]; + [(id) display setFrameWidth:width frameHeight:height]; } static void setPixelFormat(void *display, OSType pixelFormat) { - static_cast(display).pixelFormat = pixelFormat; + ((id) display).pixelFormat = pixelFormat; } }; @@ -177,7 +178,16 @@ static void setPixelFormat(void *display, OSType pixelFormat) { } BOOST_LOG(info) << "Configuring selected display ("sv << display->display_id << ") to stream"sv; - display->av_capture = [[AVVideo alloc] initWithDisplay:display->display_id frameRate:config.framerate]; + // Prefer ScreenCaptureKit on macOS 12.3+ (AVCaptureScreenInput was + // deprecated in macOS 13 and is hardcoded to 8-bit BGRA). Fall back to + // the legacy AVCaptureScreenInput path on older macOS. + if (@available(macOS 12.3, *)) { + BOOST_LOG(info) << "Using ScreenCaptureKit capture backend"sv; + display->av_capture = [[SCVideo alloc] initWithDisplay:display->display_id frameRate:config.framerate]; + } else { + BOOST_LOG(info) << "Using legacy AVCaptureScreenInput capture backend"sv; + display->av_capture = [[AVVideo alloc] initWithDisplay:display->display_id frameRate:config.framerate]; + } if (!display->av_capture) { BOOST_LOG(error) << "Video setup failed."sv; diff --git a/src/platform/macos/sc_video.h b/src/platform/macos/sc_video.h new file mode 100644 index 00000000000..7462ad6afe6 --- /dev/null +++ b/src/platform/macos/sc_video.h @@ -0,0 +1,30 @@ +/** + * @file src/platform/macos/sc_video.h + * @brief Declarations for ScreenCaptureKit-based video capture on macOS. + * + * Modern replacement for AVCaptureScreenInput (which was deprecated in + * macOS 13). SCVideo conforms to the same SunshineVideoCapture protocol + * as the legacy AVVideo class so callers can swap implementations at + * runtime based on @available(macOS 12.3, *) without other code changes. + */ +#pragma once + +#import "av_video.h" + +#import + +API_AVAILABLE(macos(12.3)) +@interface SCVideo: NSObject + +@property (nonatomic, assign) CGDirectDisplayID displayID; +@property (nonatomic, assign) CMTime minFrameDuration; +@property (nonatomic, assign) OSType pixelFormat; +@property (nonatomic, assign) int frameWidth; +@property (nonatomic, assign) int frameHeight; + +- (instancetype)initWithDisplay:(CGDirectDisplayID)displayID frameRate:(int)frameRate; + +- (void)setFrameWidth:(int)frameWidth frameHeight:(int)frameHeight; +- (dispatch_semaphore_t)capture:(FrameCallbackBlock)frameCallback; + +@end diff --git a/src/platform/macos/sc_video.m b/src/platform/macos/sc_video.m new file mode 100644 index 00000000000..9c3f895391f --- /dev/null +++ b/src/platform/macos/sc_video.m @@ -0,0 +1,350 @@ +/** + * @file src/platform/macos/sc_video.m + * @brief ScreenCaptureKit-based video capture for macOS 12.3+. + * + * Drop-in replacement for the legacy AVCaptureScreenInput path in + * av_video.m. This first-pass implementation preserves the original + * pixel format (BGRA8) and selection semantics; HDR / 10-bit pixel + * format selection and EDR color metadata propagation are layered on + * top in subsequent commits. + * + * Lifecycle: the underlying SCStream is started exactly once during + * -initWithDisplay:frameRate: and stopped exactly once during -dealloc. + * -capture: only swaps the active callback / signal; it never touches + * the stream lifecycle. This avoids the "addStreamOutput called twice" + * failure mode that SCK exhibits when an output is re-registered on a + * stream that already retains it across stop/start cycles. + * + * Compiled with ARC (-fobjc-arc) for clarity. The other macOS capture + * files remain MRC; objects flowing from this file to display.mm + * follow the standard alloc/init +1-retain convention so the boundary + * works regardless of compile mode on the other side. + */ +#import "sc_video.h" + +#import + +// Bounded wait for any SCK completion handler. SCK should always +// invoke these, but a misbehaving system service must not hang the +// whole startup path. +static const int64_t kSCVideoCompletionTimeoutSec = 5; + +API_AVAILABLE(macos(12.3)) +@interface SCVideo () + +@property (nonatomic, strong) SCStream *stream; +@property (nonatomic, strong) SCContentFilter *filter; +@property (nonatomic, strong) SCStreamConfiguration *streamConfig; +@property (nonatomic, strong) dispatch_queue_t sampleQueue; + +// All four of the following are mutated from multiple threads (the +// caller of -capture:, the SCK sample-handler queue, and the SCStream +// delegate's didStopWithError:) and so are only ever accessed under +// @synchronized(self). +@property (nonatomic, copy) FrameCallbackBlock currentCallback; +@property (nonatomic, strong) dispatch_semaphore_t currentSignal; +@property (nonatomic, assign) BOOL streamRunning; +@property (nonatomic, assign) BOOL streamOutputAdded; + +@end + +@implementation SCVideo + +- (instancetype)initWithDisplay:(CGDirectDisplayID)displayID frameRate:(int)frameRate { + self = [super init]; + if (!self) { + return nil; + } + + self.displayID = displayID; + self.minFrameDuration = CMTimeMake(1, frameRate); + self.pixelFormat = kCVPixelFormatType_32BGRA; + + // Prefer the active display mode's pixel dimensions; fall back to + // CGDisplayBounds if no mode is currently set (e.g., during display + // reconfiguration). If both fail we still proceed — SCK will + // accept the requested SCContentFilter dimensions later. + CGDisplayModeRef mode = CGDisplayCopyDisplayMode(displayID); + if (mode) { + self.frameWidth = (int) CGDisplayModeGetPixelWidth(mode); + self.frameHeight = (int) CGDisplayModeGetPixelHeight(mode); + CGDisplayModeRelease(mode); + } else { + CGRect bounds = CGDisplayBounds(displayID); + self.frameWidth = (int) CGRectGetWidth(bounds); + self.frameHeight = (int) CGRectGetHeight(bounds); + } + + // dispatch_queue_attr_make_with_qos_class's third parameter is a + // relative priority (range -15..0), NOT one of the legacy global- + // queue DISPATCH_QUEUE_PRIORITY_* constants. Using 0 keeps the + // queue at the chosen QoS class's nominal priority. + dispatch_queue_attr_t qos = dispatch_queue_attr_make_with_qos_class( + DISPATCH_QUEUE_SERIAL, + QOS_CLASS_USER_INTERACTIVE, + 0 + ); + self.sampleQueue = dispatch_queue_create("dev.lizardbyte.sunshine.sckCapture", qos); + + // SCK content enumeration is async; block (with a bounded timeout) + // until we have the SCDisplay matching the requested CGDirectDisplayID + // so this initializer remains synchronous (matching AVVideo's contract). + __block SCDisplay *selectedDisplay = nil; + __block NSError *enumerationError = nil; + dispatch_semaphore_t ready = dispatch_semaphore_create(0); + + [SCShareableContent getShareableContentExcludingDesktopWindows:NO + onScreenWindowsOnly:NO + completionHandler:^(SCShareableContent *_Nullable content, NSError *_Nullable error) { + if (error || !content) { + enumerationError = error; + } else { + for (SCDisplay *d in content.displays) { + if (d.displayID == displayID) { + selectedDisplay = d; + break; + } + } + // If the requested display wasn't found (display reconfigured, + // unplugged, etc.) fall back to the first display SCK reports. + if (!selectedDisplay && content.displays.count > 0) { + selectedDisplay = content.displays.firstObject; + } + } + dispatch_semaphore_signal(ready); + }]; + if (dispatch_semaphore_wait(ready, dispatch_time(DISPATCH_TIME_NOW, kSCVideoCompletionTimeoutSec * NSEC_PER_SEC)) != 0) { + NSLog(@"SCVideo: getShareableContent timed out after %lld seconds", kSCVideoCompletionTimeoutSec); + return nil; + } + + if (!selectedDisplay) { + NSLog(@"SCVideo: failed to resolve SCDisplay for id %u: %@", displayID, enumerationError); + return nil; + } + + // Empty excluded-windows array: capture everything on the display. + self.filter = [[SCContentFilter alloc] initWithDisplay:selectedDisplay excludingWindows:@[]]; + + self.streamConfig = [[SCStreamConfiguration alloc] init]; + self.streamConfig.width = self.frameWidth; + self.streamConfig.height = self.frameHeight; + self.streamConfig.minimumFrameInterval = self.minFrameDuration; + self.streamConfig.pixelFormat = self.pixelFormat; + self.streamConfig.queueDepth = 6; // SCK docs recommend 3-8 + self.streamConfig.showsCursor = YES; + + self.stream = [[SCStream alloc] initWithFilter:self.filter + configuration:self.streamConfig + delegate:self]; + if (!self.stream) { + NSLog(@"SCVideo: SCStream allocation failed"); + return nil; + } + + // Register the SCStreamOutput exactly once, here. SCStream retains + // outputs across stop/start cycles, so re-registering on every + // -capture: call would fail (or worse, silently duplicate + // delivery). All subsequent state changes are callback swaps on + // -capture: rather than stream-lifecycle operations. + NSError *outputError = nil; + if (![self.stream addStreamOutput:self + type:SCStreamOutputTypeScreen + sampleHandlerQueue:self.sampleQueue + error:&outputError]) { + NSLog(@"SCVideo: addStreamOutput failed: %@", outputError); + return nil; + } + self.streamOutputAdded = YES; + + // Start the stream once. Frames begin flowing immediately on the + // sampleQueue; sample-handler delivery is a no-op until the first + // -capture: installs a callback (see -stream:didOutputSampleBuffer:ofType:). + __block NSError *startError = nil; + dispatch_semaphore_t started = dispatch_semaphore_create(0); + [self.stream startCaptureWithCompletionHandler:^(NSError *_Nullable error) { + startError = error; + dispatch_semaphore_signal(started); + }]; + if (dispatch_semaphore_wait(started, dispatch_time(DISPATCH_TIME_NOW, kSCVideoCompletionTimeoutSec * NSEC_PER_SEC)) != 0) { + NSLog(@"SCVideo: startCapture timed out after %lld seconds", kSCVideoCompletionTimeoutSec); + return nil; + } + if (startError) { + NSLog(@"SCVideo: startCapture failed: %@", startError); + return nil; + } + @synchronized(self) { + self.streamRunning = YES; + } + + return self; +} + +- (void)setFrameWidth:(int)frameWidth frameHeight:(int)frameHeight { + _frameWidth = frameWidth; + _frameHeight = frameHeight; + + if (self.streamConfig) { + self.streamConfig.width = frameWidth; + self.streamConfig.height = frameHeight; + [self applyConfigurationIfRunning]; + } +} + +- (void)setPixelFormat:(OSType)pixelFormat { + _pixelFormat = pixelFormat; + + if (self.streamConfig) { + self.streamConfig.pixelFormat = pixelFormat; + [self applyConfigurationIfRunning]; + } +} + +- (void)setMinFrameDuration:(CMTime)minFrameDuration { + _minFrameDuration = minFrameDuration; + + if (self.streamConfig) { + self.streamConfig.minimumFrameInterval = minFrameDuration; + [self applyConfigurationIfRunning]; + } +} + +- (void)applyConfigurationIfRunning { + BOOL running; + @synchronized(self) { + running = self.streamRunning; + } + if (!running || !self.stream) { + return; + } + [self.stream updateConfiguration:self.streamConfig + completionHandler:^(NSError *_Nullable error) { + if (error) { + NSLog(@"SCVideo: updateConfiguration failed: %@", error); + } + }]; +} + +- (dispatch_semaphore_t)capture:(FrameCallbackBlock)frameCallback { + // Swap in the new callback. The SCStream output and frame flow are + // already running from -init; this method is purely a callback + // installation, not a stream-lifecycle operation. That avoids the + // double-add failure mode and makes -capture: cheap enough to be + // called multiple times across the SCVideo's lifetime (e.g., the + // encoder probe path's dummy_img followed by the real capture). + dispatch_semaphore_t newSignal = dispatch_semaphore_create(0); + dispatch_semaphore_t previousSignal = nil; + + @synchronized(self) { + previousSignal = self.currentSignal; + self.currentCallback = frameCallback; + self.currentSignal = newSignal; + } + + // Unblock any prior caller still waiting on the old semaphore. + // They will observe their callback was cleared and return. + if (previousSignal) { + dispatch_semaphore_signal(previousSignal); + } + + return newSignal; +} + +- (void)dealloc { + BOOL running; + SCStream *stream; + @synchronized(self) { + running = self.streamRunning; + stream = self.stream; + self.streamRunning = NO; + self.currentCallback = nil; + } + if (running && stream) { + // Best-effort synchronous stop with a bounded wait so a + // misbehaving SCK doesn't hang teardown. + dispatch_semaphore_t stopped = dispatch_semaphore_create(0); + [stream stopCaptureWithCompletionHandler:^(NSError *_Nullable error) { + (void) error; + dispatch_semaphore_signal(stopped); + }]; + dispatch_semaphore_wait(stopped, dispatch_time(DISPATCH_TIME_NOW, 2 * NSEC_PER_SEC)); + } +} + +#pragma mark - SCStreamOutput + +- (void)stream:(SCStream *)stream + didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer + ofType:(SCStreamOutputType)type { + if (type != SCStreamOutputTypeScreen) { + return; + } + if (!CMSampleBufferIsValid(sampleBuffer)) { + return; + } + + // Drop frames whose status array says they aren't ready. SCK delivers + // a status attachment on every sample buffer indicating idle vs + // complete vs blank — we want only complete frames downstream. + CFArrayRef attachmentsArray = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, NO); + if (attachmentsArray && CFArrayGetCount(attachmentsArray) > 0) { + CFDictionaryRef attachments = CFArrayGetValueAtIndex(attachmentsArray, 0); + CFNumberRef statusNum = CFDictionaryGetValue(attachments, (__bridge CFStringRef) SCStreamFrameInfoStatus); + if (statusNum) { + int status = 0; + CFNumberGetValue(statusNum, kCFNumberSInt32Type, &status); + if (status != SCFrameStatusComplete) { + return; + } + } + } + + FrameCallbackBlock callback; + dispatch_semaphore_t signal; + @synchronized(self) { + callback = self.currentCallback; + signal = self.currentSignal; + } + + if (!callback) { + // No active consumer. Drop the frame; the stream keeps running + // so subsequent -capture: calls can pick up immediately. + return; + } + + if (!callback(sampleBuffer)) { + // Consumer signalled stop. Clear the callback and wake the + // caller; the underlying SCStream stays alive for any future + // -capture: caller (cheaper than tearing down and restarting). + @synchronized(self) { + if (self.currentCallback == callback) { + self.currentCallback = nil; + self.currentSignal = nil; + } + } + if (signal) { + dispatch_semaphore_signal(signal); + } + } +} + +#pragma mark - SCStreamDelegate + +- (void)stream:(SCStream *)stream didStopWithError:(NSError *)error { + if (error) { + NSLog(@"SCVideo: stream stopped with error: %@", error); + } + dispatch_semaphore_t signal; + @synchronized(self) { + self.streamRunning = NO; + signal = self.currentSignal; + self.currentCallback = nil; + self.currentSignal = nil; + } + if (signal) { + dispatch_semaphore_signal(signal); + } +} + +@end From 2a88d470e73e91955d713217769cf51925877a7b Mon Sep 17 00:00:00 2001 From: Jason Lu Date: Mon, 25 May 2026 04:11:58 -0700 Subject: [PATCH 2/3] feat(macos/capture): enable EDR (HDR) output on macOS 14+ for 10-bit pixel formats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With AVCaptureScreenInput, asking the capture surface for a 10-bit pixel format silently produced 8-bit BGRA — the OS-level lie that made HEVC Main10 / AV1 Main10 / ProRes 10-bit profiles on macOS into fake HDR (color-tagged 8-bit data). With ScreenCaptureKit landing in the previous commit, 10-bit pixel formats are actually honoured, but SCK needs an explicit signal to attach HDR metadata to those buffers instead of treating them as 10-bit Rec.709. This commit wires SCStreamConfiguration.captureDynamicRange: * Add +pixelFormatIsHighBitDepth: classifier covering the YUV 4:2:0, 4:2:2 and 4:4:4 10-bit BiPlanar formats plus ARGB2101010 packed and 64-bit RGBA formats. * On the synchronous init path, set captureDynamicRange immediately if the starting pixel format is high bit depth so the very first sample buffer carries HDR metadata. * On the setPixelFormat: path (called by nv12_zero_device when the encoder selects p010), also update captureDynamicRange and push the new config to a running stream via -updateConfiguration:. * Use SCCaptureDynamicRangeHDRLocalDisplay rather than canonical HDR: game streaming wants the host display's actual HDR characteristics (peak luminance, primaries) so the receiver shows what a local user would see, not Apple's idealised reference. * Guard the whole block behind @available(macOS 14.0, *); on 12.3-13.x SCK still honours the 10-bit pixel format request but doesn't auto-tag buffers, so Sunshine's existing colorspace logic continues to drive the encoder's color fields. Validated on M4 Max: Sunshine's encoder probe matrix now includes successful 10-bit HEVC and 10-bit ProRes entries that previously could not have validated because the capture surface couldn't deliver matching pixel data. ProRes-specific VideoToolbox color tags land in a separate follow-up commit. --- src/platform/macos/sc_video.m | 54 +++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/src/platform/macos/sc_video.m b/src/platform/macos/sc_video.m index 9c3f895391f..fb70f721c2b 100644 --- a/src/platform/macos/sc_video.m +++ b/src/platform/macos/sc_video.m @@ -134,6 +134,10 @@ - (instancetype)initWithDisplay:(CGDirectDisplayID)displayID frameRate:(int)fram self.streamConfig.queueDepth = 6; // SCK docs recommend 3-8 self.streamConfig.showsCursor = YES; + // If the initial pixel format is already a 10-bit format, flip on EDR + // immediately so the very first sample buffer carries HDR metadata. + [self applyDynamicRangeForPixelFormat:self.pixelFormat]; + self.stream = [[SCStream alloc] initWithFilter:self.filter configuration:self.streamConfig delegate:self]; @@ -181,6 +185,55 @@ - (instancetype)initWithDisplay:(CGDirectDisplayID)displayID frameRate:(int)fram return self; } +/** + * @brief Whether a CVPixelBuffer OSType denotes a 10-bit (or wider) format. + * + * Returning YES is the signal that the capture surface is HDR-capable; we + * use it to drive SCStreamConfiguration.captureDynamicRange on macOS 14+ + * so SCK emits BT.2020 PQ-tagged buffers instead of 10-bit Rec.709. + */ ++ (BOOL)pixelFormatIsHighBitDepth:(OSType)pixelFormat { + switch (pixelFormat) { + case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange: + case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange: + case kCVPixelFormatType_422YpCbCr10BiPlanarVideoRange: + case kCVPixelFormatType_422YpCbCr10BiPlanarFullRange: + case kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange: + case kCVPixelFormatType_444YpCbCr10BiPlanarFullRange: + case kCVPixelFormatType_ARGB2101010LEPacked: + case kCVPixelFormatType_64ARGB: + case kCVPixelFormatType_64RGBALE: + return YES; + default: + return NO; + } +} + +- (void)applyDynamicRangeForPixelFormat:(OSType)pixelFormat { + // captureDynamicRange / SCCaptureDynamicRange* are macOS 14 (Sonoma) + // SDK symbols. The compile-time guard ensures this block is preprocessed + // away entirely when building against an older SDK that lacks the + // declarations; the runtime @available guard prevents using the + // symbols at runtime on pre-14 systems even with a newer SDK. On + // 12.3-13.x SCK still honours a requested 10-bit pixel format, but + // the OS won't tag buffers with BT.2020 PQ metadata automatically; + // downstream code falls back to Sunshine's existing colorspace logic. +#if defined(__MAC_OS_X_VERSION_MAX_ALLOWED) && __MAC_OS_X_VERSION_MAX_ALLOWED >= 140000 + if (@available(macOS 14.0, *)) { + if ([SCVideo pixelFormatIsHighBitDepth:pixelFormat]) { + // hdrLocalDisplay matches the host display's HDR characteristics, + // which is what we want for game-streaming: stream what the user + // would see locally, including the local panel's PQ peak luminance. + self.streamConfig.captureDynamicRange = SCCaptureDynamicRangeHDRLocalDisplay; + } else { + self.streamConfig.captureDynamicRange = SCCaptureDynamicRangeSDR; + } + } +#else + (void) pixelFormat; +#endif +} + - (void)setFrameWidth:(int)frameWidth frameHeight:(int)frameHeight { _frameWidth = frameWidth; _frameHeight = frameHeight; @@ -197,6 +250,7 @@ - (void)setPixelFormat:(OSType)pixelFormat { if (self.streamConfig) { self.streamConfig.pixelFormat = pixelFormat; + [self applyDynamicRangeForPixelFormat:pixelFormat]; [self applyConfigurationIfRunning]; } } From 9030b66a4cf8ee9237ea618c055daf536248d7e2 Mon Sep 17 00:00:00 2001 From: Jason Lu Date: Mon, 25 May 2026 10:18:45 -0700 Subject: [PATCH 3/3] feat(macos/capture): gate EDR on negotiated session HDR, not pixel depth alone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous EDR commit flipped SCStreamConfiguration.captureDynamicRange to HDRLocalDisplay whenever the chosen CVPixelBuffer format was 10-bit. That is necessary but not sufficient: a 10-bit format may be selected for codec reasons (e.g., a ProRes profile that requires 4:4:4 10-bit input) without the client ever requesting HDR ingest. The result was a silent control/data-plane mismatch — Sunshine would tell the client "HDR mode false" in the SDP while emitting BT.2020 PQ-tagged buffers, leaving the decoder to interpret tagged HDR content however its display pipeline saw fit. Plumb the negotiated session's HDR state down to SCK: rtsp.cpp (x-nv-video[0].dynamicRangeMode → config.dynamicRange) → video.cpp (existing) → platf::display(... video::config_t) → display.mm (hdr_allowed = config.dynamicRange ? YES : NO) → SCVideo initWithDisplay:frameRate:hdrAllowed: → applyDynamicRangeForPixelFormat: (gates HDR on both pixel format depth AND hdrAllowed; defaults to SDR otherwise) The convenience initializer without hdrAllowed defaults to NO so any out-of-tree caller stays on the safe SDR path until they opt in. The new "Using ScreenCaptureKit capture backend (HDR allowed|blocked)" log line makes the negotiated state visible at the same place the backend selection is logged. --- src/platform/macos/display.mm | 10 ++++++++-- src/platform/macos/sc_video.h | 7 +++++++ src/platform/macos/sc_video.m | 17 ++++++++++++++++- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/platform/macos/display.mm b/src/platform/macos/display.mm index 9f6c8ef5ccd..dbbae2645d6 100644 --- a/src/platform/macos/display.mm +++ b/src/platform/macos/display.mm @@ -182,8 +182,14 @@ static void setPixelFormat(void *display, OSType pixelFormat) { // deprecated in macOS 13 and is hardcoded to 8-bit BGRA). Fall back to // the legacy AVCaptureScreenInput path on older macOS. if (@available(macOS 12.3, *)) { - BOOST_LOG(info) << "Using ScreenCaptureKit capture backend"sv; - display->av_capture = [[SCVideo alloc] initWithDisplay:display->display_id frameRate:config.framerate]; + // hdrAllowed reflects the negotiated `enable_hdr` for this session + // (rtsp.cpp maps `x-nv-video[0].dynamicRangeMode` into config.dynamicRange). + // SCK uses this together with the chosen pixel format depth to decide + // whether to flip captureDynamicRange to HDRLocalDisplay; neither + // condition alone is sufficient. See sc_video.m::applyDynamicRangeForPixelFormat:. + const BOOL hdr_allowed = config.dynamicRange ? YES : NO; + BOOST_LOG(info) << "Using ScreenCaptureKit capture backend (HDR "sv << (hdr_allowed ? "allowed" : "blocked") << ")"sv; + display->av_capture = [[SCVideo alloc] initWithDisplay:display->display_id frameRate:config.framerate hdrAllowed:hdr_allowed]; } else { BOOST_LOG(info) << "Using legacy AVCaptureScreenInput capture backend"sv; display->av_capture = [[AVVideo alloc] initWithDisplay:display->display_id frameRate:config.framerate]; diff --git a/src/platform/macos/sc_video.h b/src/platform/macos/sc_video.h index 7462ad6afe6..51823214d79 100644 --- a/src/platform/macos/sc_video.h +++ b/src/platform/macos/sc_video.h @@ -22,7 +22,14 @@ API_AVAILABLE(macos(12.3)) @property (nonatomic, assign) int frameWidth; @property (nonatomic, assign) int frameHeight; +// YES iff the negotiated streaming session enabled HDR (Moonlight's +// hdrMode flag). Required (in combination with a 10-bit pixel format) +// before SCK is allowed to flip captureDynamicRange to HDRLocalDisplay +// on macOS 14+. Defaults to NO; the SDR capture path is always safe. +@property (nonatomic, assign) BOOL hdrAllowed; + - (instancetype)initWithDisplay:(CGDirectDisplayID)displayID frameRate:(int)frameRate; +- (instancetype)initWithDisplay:(CGDirectDisplayID)displayID frameRate:(int)frameRate hdrAllowed:(BOOL)hdrAllowed; - (void)setFrameWidth:(int)frameWidth frameHeight:(int)frameHeight; - (dispatch_semaphore_t)capture:(FrameCallbackBlock)frameCallback; diff --git a/src/platform/macos/sc_video.m b/src/platform/macos/sc_video.m index fb70f721c2b..52ccacfc952 100644 --- a/src/platform/macos/sc_video.m +++ b/src/platform/macos/sc_video.m @@ -51,6 +51,10 @@ @interface SCVideo () @implementation SCVideo - (instancetype)initWithDisplay:(CGDirectDisplayID)displayID frameRate:(int)frameRate { + return [self initWithDisplay:displayID frameRate:frameRate hdrAllowed:NO]; +} + +- (instancetype)initWithDisplay:(CGDirectDisplayID)displayID frameRate:(int)frameRate hdrAllowed:(BOOL)hdrAllowed { self = [super init]; if (!self) { return nil; @@ -59,6 +63,7 @@ - (instancetype)initWithDisplay:(CGDirectDisplayID)displayID frameRate:(int)fram self.displayID = displayID; self.minFrameDuration = CMTimeMake(1, frameRate); self.pixelFormat = kCVPixelFormatType_32BGRA; + self.hdrAllowed = hdrAllowed; // Prefer the active display mode's pixel dimensions; fall back to // CGDisplayBounds if no mode is currently set (e.g., during display @@ -218,9 +223,19 @@ - (void)applyDynamicRangeForPixelFormat:(OSType)pixelFormat { // 12.3-13.x SCK still honours a requested 10-bit pixel format, but // the OS won't tag buffers with BT.2020 PQ metadata automatically; // downstream code falls back to Sunshine's existing colorspace logic. + // + // Gating: EDR is only enabled when BOTH (a) the chosen pixel format + // is 10-bit, AND (b) the session was actually negotiated as HDR + // (`hdrAllowed`). The pixel format on its own is necessary but not + // sufficient — a 10-bit format may be selected for codec reasons + // (e.g., a ProRes profile) without the client ever requesting HDR + // ingest, and silently emitting BT.2020 PQ-tagged buffers into a + // stream the control plane describes as SDR causes the decoder to + // tone-map undefined content. Defaulting hdrAllowed to NO keeps the + // legacy/SDR semantics intact when callers don't opt in. #if defined(__MAC_OS_X_VERSION_MAX_ALLOWED) && __MAC_OS_X_VERSION_MAX_ALLOWED >= 140000 if (@available(macOS 14.0, *)) { - if ([SCVideo pixelFormatIsHighBitDepth:pixelFormat]) { + if (self.hdrAllowed && [SCVideo pixelFormatIsHighBitDepth:pixelFormat]) { // hdrLocalDisplay matches the host display's HDR characteristics, // which is what we want for game-streaming: stream what the user // would see locally, including the local panel's PQ peak luminance.