Skip to content

Windows playback perf #1616

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
richiemcilroy merged 8 commits into from
Feb 18, 2026
Merged

Windows playback perf #1616

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
e87c7b0
chore: vendor tao dependency
richiemcilroy Feb 18, 2026
d740ddb
fix(desktop): NV12 stride alignment for correct YUV rendering
richiemcilroy Feb 18, 2026
6a6da93
perf(rendering): NV12 stride alignment and shared pipelines
richiemcilroy Feb 18, 2026
79a76ff
perf(editor): NV12 output and non-blocking render
richiemcilroy Feb 18, 2026
00b070b
perf(windows): playback timing and prefetch tuning
richiemcilroy Feb 18, 2026
7bacb60
perf(windows): MediaFoundation decoder batching and zero-copy
richiemcilroy Feb 18, 2026
4a5c780
chore: update generated tauri bindings
richiemcilroy Feb 18, 2026
103feb9
clippy
richiemcilroy Feb 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ cidre = { git = "https://github.com/CapSoftware/cidre", rev = "bf84b67079a8" }
# https://github.com/gfx-rs/wgpu/pull/7550
# wgpu = { git = "https://github.com/gfx-rs/wgpu", rev = "cd41a6e32a6239b65d1cecbeccde6a43a100914a" }
wgpu-hal = { path = "vendor/wgpu-hal" }
tao = { path = "vendor/tao" }

# https://github.com/CapSoftware/posthog-rs/commit/c7e9712be2f9a9122b1df685d5a067afa5415288
posthog-rs = { git = "https://github.com/CapSoftware/posthog-rs", rev = "c7e9712be2f9a9122b1df685d5a067afa5415288" }
Expand Down
54 changes: 34 additions & 20 deletions apps/desktop/src-tauri/src/editor_window.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ use tauri::{AppHandle, Manager, Runtime, Window, ipc::CommandArg};
use tokio::sync::{RwLock, watch};
use tokio_util::sync::CancellationToken;

use cap_rendering::GpuOutputFormat;

use crate::{
create_editor_instance_impl,
frame_ws::{WSFrame, WSFrameFormat, create_watch_frame_ws},
Expand All @@ -29,16 +31,22 @@ async fn do_prewarm(app: AppHandle, path: PathBuf) -> PendingResult {
path,
Box::new(move |output| {
let ws_frame = match output {
cap_editor::EditorFrameOutput::Nv12(frame) => WSFrame {
data: frame.data,
width: frame.width,
height: frame.height,
stride: frame.y_stride,
frame_number: frame.frame_number,
target_time_ns: frame.target_time_ns,
format: WSFrameFormat::Nv12,
created_at: Instant::now(),
},
cap_editor::EditorFrameOutput::Nv12(frame) => {
let ws_format = match frame.format {
GpuOutputFormat::Nv12 => WSFrameFormat::Nv12,
GpuOutputFormat::Rgba => WSFrameFormat::Rgba,
};
WSFrame {
data: frame.data,
width: frame.width,
height: frame.height,
stride: frame.y_stride,
frame_number: frame.frame_number,
target_time_ns: frame.target_time_ns,
format: ws_format,
created_at: Instant::now(),
}
}
cap_editor::EditorFrameOutput::Rgba(frame) => WSFrame {
data: frame.data,
width: frame.width,
Expand Down Expand Up @@ -234,16 +242,22 @@ impl EditorInstances {
path,
Box::new(move |output| {
let ws_frame = match output {
cap_editor::EditorFrameOutput::Nv12(frame) => WSFrame {
data: frame.data,
width: frame.width,
height: frame.height,
stride: frame.y_stride,
frame_number: frame.frame_number,
target_time_ns: frame.target_time_ns,
format: WSFrameFormat::Nv12,
created_at: Instant::now(),
},
cap_editor::EditorFrameOutput::Nv12(frame) => {
let ws_format = match frame.format {
GpuOutputFormat::Nv12 => WSFrameFormat::Nv12,
GpuOutputFormat::Rgba => WSFrameFormat::Rgba,
};
WSFrame {
data: frame.data,
width: frame.width,
height: frame.height,
stride: frame.y_stride,
frame_number: frame.frame_number,
target_time_ns: frame.target_time_ns,
format: ws_format,
created_at: Instant::now(),
}
}
cap_editor::EditorFrameOutput::Rgba(frame) => WSFrame {
data: frame.data,
width: frame.width,
Expand Down
3 changes: 2 additions & 1 deletion apps/desktop/src-tauri/src/frame_ws.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,10 @@ fn pack_nv12_frame_ref(
data: &[u8],
width: u32,
height: u32,
y_stride: u32,
frame_number: u32,
target_time_ns: u64,
) -> Vec<u8> {
let y_stride = width;
let metadata_size = 28;
let mut output = Vec::with_capacity(data.len() + metadata_size);
output.extend_from_slice(data);
Expand Down Expand Up @@ -90,6 +90,7 @@ fn pack_ws_frame_ref(frame: &WSFrame) -> Vec<u8> {
&frame.data,
frame.width,
frame.height,
frame.stride,
frame.frame_number,
frame.target_time_ns,
),
Expand Down
2 changes: 1 addition & 1 deletion apps/desktop/src/routes/editor/context.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ export const getPreviewResolution = (
quality: EditorPreviewQuality,
): XY<number> => {
const scale = previewQualityScale[quality];
const width = (Math.max(2, Math.round(OUTPUT_SIZE.x * scale)) + 1) & ~1;
const width = (Math.max(4, Math.round(OUTPUT_SIZE.x * scale)) + 3) & ~3;
const height = (Math.max(2, Math.round(OUTPUT_SIZE.y * scale)) + 1) & ~1;

return { x: width, y: height };
Expand Down
4 changes: 2 additions & 2 deletions apps/desktop/src/routes/screenshot-editor/context.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ function convertNv12ToRgba(
const ySize = yStride * height;
const yPlane = nv12Data;
const uvPlane = nv12Data.subarray(ySize);
const uvStride = width;
const uvStride = yStride;

for (let row = 0; row < height; row++) {
const yRowOffset = row * yStride;
Expand Down Expand Up @@ -285,7 +285,7 @@ function createScreenshotEditorContext() {
if (!width || !height) return;

const ySize = yStride * height;
const uvSize = width * (height / 2);
const uvSize = yStride * (height / 2);
const totalSize = ySize + uvSize;

const nv12Data = new Uint8ClampedArray(buffer, 0, totalSize);
Expand Down
4 changes: 2 additions & 2 deletions apps/desktop/src/utils/frame-worker.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ function parseFrameMetadata(bytes: Uint8Array): FrameMetadata | null {
if (!width || !height) return null;

const ySize = yStride * height;
const uvSize = width * (height / 2);
const uvSize = yStride * (height / 2);
const totalSize = ySize + uvSize;

if (bytes.byteLength - 28 < totalSize) {
Expand Down Expand Up @@ -303,7 +303,7 @@ function convertNv12ToRgba(
const ySize = yStride * height;
const yPlane = nv12Data;
const uvPlane = nv12Data.subarray(ySize);
const uvStride = width;
const uvStride = yStride;

for (let row = 0; row < height; row++) {
const yRowOffset = row * yStride;
Expand Down
10 changes: 5 additions & 5 deletions apps/desktop/src/utils/socket.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ function convertNv12ToRgbaMainThread(
const ySize = yStride * height;
const yPlane = nv12Data;
const uvPlane = nv12Data.subarray(ySize);
const uvStride = width;
const uvStride = yStride;

for (let row = 0; row < height; row++) {
const yRowOffset = row * yStride;
Expand Down Expand Up @@ -308,7 +308,7 @@ export function createImageDataWS(

if (width > 0 && height > 0) {
const ySize = yStride * height;
const uvSize = width * (height / 2);
const uvSize = yStride * (height / 2);
const totalSize = ySize + uvSize;

const frameData = new Uint8ClampedArray(buffer, 0, totalSize);
Expand Down Expand Up @@ -351,7 +351,7 @@ export function createImageDataWS(

if (width > 0 && height > 0) {
const ySize = yStride * height;
const uvSize = width * (height / 2);
const uvSize = yStride * (height / 2);
const totalSize = ySize + uvSize;

const frameData = new Uint8ClampedArray(buffer, 0, totalSize);
Expand Down Expand Up @@ -669,7 +669,7 @@ export function createImageDataWS(

if (width > 0 && height > 0) {
const ySize = yStride * height;
const uvSize = width * (height / 2);
const uvSize = yStride * (height / 2);
const totalSize = ySize + uvSize;

const frameData = new Uint8ClampedArray(buffer, 0, totalSize);
Expand Down Expand Up @@ -735,7 +735,7 @@ export function createImageDataWS(

if (width > 0 && height > 0) {
const ySize = yStride * height;
const uvSize = width * (height / 2);
const uvSize = yStride * (height / 2);
const totalSize = ySize + uvSize;

const nv12Data = new Uint8ClampedArray(buffer, 0, totalSize);
Expand Down
7 changes: 5 additions & 2 deletions apps/desktop/src/utils/tauri.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -403,6 +403,7 @@ videoImportProgress: "video-import-progress"

/** user-defined types **/

export type AllGpusInfo = { gpus: GpuInfoDiag[]; primaryGpuIndex: number | null; isMultiGpuSystem: boolean; hasDiscreteGpu: boolean }
export type Annotation = { id: string; type: AnnotationType; x: number; y: number; width: number; height: number; strokeColor: string; strokeWidth: number; fillColor: string; opacity: number; rotation: number; text: string | null; maskType?: MaskType | null; maskLevel?: number | null }
export type AnnotationType = "arrow" | "circle" | "rectangle" | "text" | "mask"
export type AppTheme = "system" | "light" | "dark"
Expand Down Expand Up @@ -476,6 +477,7 @@ quality: number | null;
*/
fast: boolean | null }
export type GlideDirection = "none" | "left" | "right" | "up" | "down"
export type GpuInfoDiag = { vendor: string; description: string; dedicatedVideoMemoryMb: number; adapterIndex: number; isSoftwareAdapter: boolean; isBasicRenderDriver: boolean; supportsHardwareEncoding: boolean }
export type HapticPattern = "alignment" | "levelChange" | "generic"
export type HapticPerformanceTime = "default" | "now" | "drawCompleted"
export type Hotkey = { code: string; meta: boolean; ctrl: boolean; alt: boolean; shift: boolean }
Expand All @@ -489,7 +491,6 @@ export type JsonValue<T> = [T]
export type LogicalBounds = { position: LogicalPosition; size: LogicalSize }
export type LogicalPosition = { x: number; y: number }
export type LogicalSize = { width: number; height: number }
export type MacOSVersionInfo = { major: number; minor: number; patch: number; displayName: string; buildNumber: string; isAppleSilicon: boolean }
export type MainWindowRecordingStartBehaviour = "close" | "minimise"
export type MaskKeyframes = { position?: MaskVectorKeyframe[]; size?: MaskVectorKeyframe[]; intensity?: MaskScalarKeyframe[] }
export type MaskKind = "sensitive" | "highlight"
Expand Down Expand Up @@ -533,6 +534,7 @@ export type RecordingStatus = "pending" | "recording"
export type RecordingStopped = null
export type RecordingTargetMode = "display" | "window" | "area" | "camera"
export type RenderFrameEvent = { frame_number: number; fps: number; resolution_base: XY<number> }
export type RenderingStatus = { isUsingSoftwareRendering: boolean; isUsingBasicRenderDriver: boolean; hardwareEncodingAvailable: boolean; warningMessage: string | null }
export type RequestOpenRecordingPicker = { target_mode: RecordingTargetMode | null }
export type RequestOpenSettings = { page: string }
export type RequestScreenCapturePrewarm = { force?: boolean }
Expand All @@ -555,7 +557,7 @@ export type StartRecordingInputs = { capture_target: ScreenCaptureTarget; captur
export type StereoMode = "stereo" | "monoL" | "monoR"
export type StudioRecordingMeta = { segment: SingleSegment } | { inner: MultipleSegments }
export type StudioRecordingStatus = { status: "InProgress" } | { status: "NeedsRemux" } | { status: "Failed"; error: string } | { status: "Complete" }
export type SystemDiagnostics = { macosVersion: MacOSVersionInfo | null; availableEncoders: string[]; screenCaptureSupported: boolean; metalSupported: boolean; gpuName: string | null }
export type SystemDiagnostics = { windowsVersion: WindowsVersionInfo | null; gpuInfo: GpuInfoDiag | null; allGpus: AllGpusInfo | null; renderingStatus: RenderingStatus; availableEncoders: string[]; graphicsCaptureSupported: boolean; d3D11VideoProcessorAvailable: boolean }
export type TargetUnderCursor = { display_id: DisplayId | null; window: WindowUnderCursor | null }
export type TextSegment = { start: number; end: number; enabled?: boolean; content?: string; center?: XY<number>; size?: XY<number>; fontFamily?: string; fontSize?: number; fontWeight?: number; italic?: boolean; color?: string; fadeDuration?: number }
export type TimelineConfiguration = { segments: TimelineSegment[]; zoomSegments: ZoomSegment[]; sceneSegments?: SceneSegment[]; maskSegments?: MaskSegment[]; textSegments?: TextSegment[] }
Expand All @@ -574,6 +576,7 @@ export type WindowExclusion = { bundleIdentifier?: string | null; ownerName?: st
export type WindowId = string
export type WindowPosition = { x: number; y: number; displayId?: DisplayId | null }
export type WindowUnderCursor = { id: WindowId; app_name: string; bounds: LogicalBounds }
export type WindowsVersionInfo = { major: number; minor: number; build: number; displayName: string; meetsRequirements: boolean; isWindows11: boolean }
export type XY<T> = { x: T; y: T }
export type ZoomMode = "auto" | { manual: { x: number; y: number } }
export type ZoomSegment = { start: number; end: number; amount: number; mode: ZoomMode; glideDirection?: GlideDirection; glideSpeed?: number; instantAnimation?: boolean; edgeSnapRatio?: number }
Expand Down
2 changes: 1 addition & 1 deletion apps/desktop/src/utils/webgpu-renderer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -380,7 +380,7 @@ export function renderNv12FrameWebGPU(
const ySize = yStride * height;
const uvWidth = width / 2;
const uvHeight = height / 2;
const uvStride = width;
const uvStride = yStride;
const uvSize = uvStride * uvHeight;

if (data.byteLength < ySize + uvSize) {
Expand Down
3 changes: 3 additions & 0 deletions crates/editor/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,3 +36,6 @@ ringbuf = "0.4.8"
lru = "0.12"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
workspace-hack = { version = "0.1", path = "../workspace-hack" }

[target.'cfg(target_os = "windows")'.dependencies]
windows = { workspace = true, features = ["Win32_Media"] }
15 changes: 5 additions & 10 deletions crates/editor/src/editor.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ impl Renderer {
}
}
match frame_renderer
.render_immediate(
.render_immediate_nv12(
current.segment_frames,
current.uniforms,
&current.cursor,
Expand All @@ -157,7 +157,7 @@ impl Renderer {
.await
{
Ok(frame) => {
(self.frame_cb)(EditorFrameOutput::Rgba(frame));
(self.frame_cb)(EditorFrameOutput::Nv12(frame));
}
Err(e) => {
tracing::error!(error = %e, "Failed to render frame in editor");
Expand All @@ -170,25 +170,20 @@ impl Renderer {
}

impl RendererHandle {
async fn send(&self, msg: RendererMessage) {
let _ = self.tx.send(msg).await;
}

pub async fn render_frame(
pub fn render_frame(
&self,
segment_frames: DecodedSegmentFrames,
uniforms: ProjectUniforms,
cursor: Arc<CursorEvents>,
) {
let (finished_tx, _finished_rx) = oneshot::channel();

self.send(RendererMessage::RenderFrame {
let _ = self.tx.try_send(RendererMessage::RenderFrame {
segment_frames,
uniforms,
finished: finished_tx,
cursor,
})
.await;
});
Comment on lines +181 to +186
Copy link

@tembo tembo bot Feb 18, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

try_send dropping render requests is fine for perf, but swallowing Closed makes failures silent and hard to debug.

Suggested change
let _ = self.tx.try_send(RendererMessage::RenderFrame {
segment_frames,
uniforms,
finished: finished_tx,
cursor,
})
.await;
});
match self.tx.try_send(RendererMessage::RenderFrame {
segment_frames,
uniforms,
finished: finished_tx,
cursor,
}) {
Ok(()) => {}
Err(mpsc::error::TrySendError::Full(_)) => {}
Err(mpsc::error::TrySendError::Closed(_)) => {
tracing::warn!("Failed to send render request to renderer (channel closed)");
}
}

}

pub async fn stop(&self) {
Expand Down
3 changes: 1 addition & 2 deletions crates/editor/src/editor_instance.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -532,8 +532,7 @@ impl EditorInstance {
&zoom_focus_interpolator,
);
self.renderer
.render_frame(segment_frames, uniforms, segment_medias.cursor.clone())
.await;
.render_frame(segment_frames, uniforms, segment_medias.cursor.clone());
} else {
warn!("Preview renderer: no frames returned for frame {}", frame_number);
}
Expand Down
Loading
Loading