Skip to content

Speed up crop preview and reduce timeline overhead #1707

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
richiemcilroy merged 7 commits into from
Apr 1, 2026
Merged

Speed up crop preview and reduce timeline overhead #1707

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
eb538ee
improve: reduce editor work for long recordings
richiemcilroy Apr 1, 2026
be7bbd0
build(desktop): enable window devtools only in debug builds
richiemcilroy Apr 1, 2026
0da6437
refactor(desktop): demote crop frame profile logging to debug
richiemcilroy Apr 1, 2026
600cd01
feat(editor): add optional previewUrl to crop dialog state
richiemcilroy Apr 1, 2026
8ac0875
feat(editor): capture canvas frame when opening crop dialog
richiemcilroy Apr 1, 2026
e82d913
feat(editor): improve crop preview for long recordings and load failures
richiemcilroy Apr 1, 2026
cf27e33
Remove deferred accurate crop preview logic
richiemcilroy Apr 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion apps/desktop/src-tauri/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ tauri = { workspace = true, features = [
"protocol-asset",
"tray-icon",
"image-png",
"devtools",
] }
tauri-specta = { version = "=2.0.0-rc.20", features = ["derive", "typescript"] }
tauri-plugin-dialog = "2.2.0"
Expand Down
27 changes: 27 additions & 0 deletions apps/desktop/src-tauri/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2764,11 +2764,15 @@ async fn get_display_frame_for_cropping(
use cap_rendering::{PixelFormat, cpu_yuv};
use image::{ImageEncoder, codecs::png::PngEncoder};
use std::io::Cursor;
use std::time::Instant;

let total_started_at = Instant::now();

let frame_number = editor_instance.state.lock().await.playhead_position;
let time_secs = frame_number as f64 / fps as f64;

let project = editor_instance.project_config.1.borrow().clone();
let lookup_started_at = Instant::now();

let (segment_time, segment) = project
.get_segment_time(time_secs)
Expand All @@ -2785,19 +2789,23 @@ async fn get_display_frame_for_cropping(
.find(|v| v.index == segment.recording_clip)
.map(|v| v.offsets)
.unwrap_or(ClipOffsets::default());
let lookup_elapsed_ms = lookup_started_at.elapsed().as_secs_f64() * 1000.0;

let decode_started_at = Instant::now();
let segment_frames = segment_medias
.decoders
.get_frames(segment_time as f32, false, true, clip_offsets)
.await
.ok_or_else(|| "Failed to get frame".to_string())?;
let decode_elapsed_ms = decode_started_at.elapsed().as_secs_f64() * 1000.0;

let screen_frame = segment_frames
.screen_frame
.ok_or_else(|| "Failed to get screen frame".to_string())?;
let width = screen_frame.width();
let height = screen_frame.height();

let convert_started_at = Instant::now();
let rgba_data = match screen_frame.format() {
PixelFormat::Rgba => screen_frame.data().to_vec(),
PixelFormat::Nv12 => {
Expand Down Expand Up @@ -2833,12 +2841,31 @@ async fn get_display_frame_for_cropping(
rgba
}
};
let convert_elapsed_ms = convert_started_at.elapsed().as_secs_f64() * 1000.0;

let encode_started_at = Instant::now();
let mut png_data = Cursor::new(Vec::new());
let encoder = PngEncoder::new(&mut png_data);
encoder
.write_image(&rgba_data, width, height, image::ExtendedColorType::Rgba8)
.map_err(|e| format!("Failed to encode PNG: {e}"))?;
let encode_elapsed_ms = encode_started_at.elapsed().as_secs_f64() * 1000.0;
let total_elapsed_ms = total_started_at.elapsed().as_secs_f64() * 1000.0;

debug!(
target: "cap_crop_profile",
frame_number = frame_number,
time_secs = time_secs,
segment_time = segment_time,
width = width,
height = height,
lookup_ms = lookup_elapsed_ms,
decode_ms = decode_elapsed_ms,
convert_ms = convert_elapsed_ms,
encode_ms = encode_elapsed_ms,
total_ms = total_elapsed_ms,
"crop frame profile"
);

Ok(png_data.into_inner())
}
Expand Down
3 changes: 2 additions & 1 deletion apps/desktop/src-tauri/src/windows.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2120,7 +2120,8 @@ impl ShowCapWindow {
.visible(false)
.accept_first_mouse(true)
.shadow(true)
.theme(theme);
.theme(theme)
.devtools(cfg!(debug_assertions));

if !id.is_transparent() {
let is_dark = match theme {
Expand Down
221 changes: 204 additions & 17 deletions apps/desktop/src/routes/editor/Editor.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import {
Match,
on,
onCleanup,
onMount,
Show,
Switch,
} from "solid-js";
Expand Down Expand Up @@ -59,9 +60,16 @@ const MIN_PLAYER_CONTENT_HEIGHT = 320;
const MIN_TIMELINE_HEIGHT = 240;
const RESIZE_HANDLE_HEIGHT = 16;
const MIN_PLAYER_HEIGHT = MIN_PLAYER_CONTENT_HEIGHT + RESIZE_HANDLE_HEIGHT;

const TIMELINE_RESIZE_GRIP_MARKS = [0, 1, 2] as const;

function logCropProfile(
stage: string,
data: Record<string, number | string | boolean | null> = {},
) {
if (!import.meta.env.DEV) return;
console.info("[crop-profile]", stage, data);
}

function getEditorErrorMessage(error: unknown) {
return error instanceof Error ? error.message : String(error);
}
Expand Down Expand Up @@ -772,24 +780,182 @@ function Dialogs() {
const [crop, setCrop] = createSignal(CROP_ZERO);
const [aspect, setAspect] = createSignal<Ratio | null>(null);

const initialPreviewUrl = dialog().previewUrl ?? null;
const [frameBlobUrl, setFrameBlobUrl] = createSignal<
string | null
>(null);
>(initialPreviewUrl);
const [frameSource, setFrameSource] = createSignal<
"captured-preview" | "accurate-frame" | "screenshot"
>(initialPreviewUrl ? "captured-preview" : "screenshot");
const cropOpenedAt = performance.now();
const screenshotSrc = convertFileSrc(
`${editorInstance.path}/screenshots/display.jpg`,
);

let cancelled = false;
let frameLoadDelayTimeoutId:
| ReturnType<typeof globalThis.setTimeout>
| undefined;
let frameLoadTimeoutId:
| ReturnType<typeof globalThis.setTimeout>
| undefined;
let frameLoadIdleId: number | undefined;
let accurateFrameRequested = false;
const idleWindow = globalThis as typeof globalThis & {
requestIdleCallback?: (
callback: () => void,
options?: { timeout?: number },
) => number;
cancelIdleCallback?: (handle: number) => void;
};

const clearScheduledAccurateFrame = () => {
if (frameLoadDelayTimeoutId !== undefined) {
globalThis.clearTimeout(frameLoadDelayTimeoutId);
frameLoadDelayTimeoutId = undefined;
}
if (frameLoadIdleId !== undefined) {
idleWindow.cancelIdleCallback?.(frameLoadIdleId);
frameLoadIdleId = undefined;
}
if (frameLoadTimeoutId !== undefined) {
globalThis.clearTimeout(frameLoadTimeoutId);
frameLoadTimeoutId = undefined;
}
};

const setPreviewBlob = (
blob: Blob,
source: "accurate-frame",
) => {
const nextUrl = URL.createObjectURL(blob);
const previousUrl = frameBlobUrl();
setFrameBlobUrl(nextUrl);
setFrameSource(source);
if (previousUrl) {
URL.revokeObjectURL(previousUrl);
}
};

const requestAccurateFrame = (reason: string) => {
if (accurateFrameRequested || cancelled) return;

commands
.getDisplayFrameForCropping(FPS)
.then((pngBytes) => {
const blob = new Blob([new Uint8Array(pngBytes)], {
type: "image/png",
clearScheduledAccurateFrame();

accurateFrameRequested = true;
const frameRequestStartedAt = performance.now();
logCropProfile("accurate-frame-request-start", {
elapsedMs: Number(
(frameRequestStartedAt - cropOpenedAt).toFixed(2),
),
reason,
});

void commands
.getDisplayFrameForCropping(FPS)
.then((pngBytes) => {
if (cancelled) return;

setPreviewBlob(
new Blob([new Uint8Array(pngBytes)], {
type: "image/png",
}),
"accurate-frame",
);
logCropProfile("accurate-frame-request-finish", {
elapsedMs: Number(
(performance.now() - cropOpenedAt).toFixed(2),
),
requestMs: Number(
(performance.now() - frameRequestStartedAt).toFixed(
2,
),
),
reason,
});
})
.catch((error: unknown) => {
if (cancelled) return;
console.warn("Display frame fetch failed:", error);
logCropProfile("accurate-frame-request-failed", {
elapsedMs: Number(
(performance.now() - cropOpenedAt).toFixed(2),
),
requestMs: Number(
(performance.now() - frameRequestStartedAt).toFixed(
2,
),
),
message:
error instanceof Error
? error.message
: String(error),
reason,
});
});
const url = URL.createObjectURL(blob);
setFrameBlobUrl(url);
})
.catch((error: unknown) => {
console.warn("Display frame fetch failed:", error);
};

const scheduleAccurateFrame = (
reason: string,
options: {
delayMs?: number;
idleTimeoutMs: number;
fallbackDelayMs: number;
},
) => {
const queueIdleFrame = () => {
const loadFrame = () => requestAccurateFrame(reason);

if (idleWindow.requestIdleCallback) {
frameLoadIdleId = idleWindow.requestIdleCallback(
() => {
frameLoadIdleId = undefined;
loadFrame();
},
{
timeout: options.idleTimeoutMs,
},
);
return;
}

frameLoadTimeoutId = globalThis.setTimeout(() => {
frameLoadTimeoutId = undefined;
loadFrame();
}, options.fallbackDelayMs);
};

if (!options.delayMs) {
queueIdleFrame();
return;
}

frameLoadDelayTimeoutId = globalThis.setTimeout(() => {
frameLoadDelayTimeoutId = undefined;
if (cancelled || accurateFrameRequested) return;
queueIdleFrame();
}, options.delayMs);
};

onMount(() => {
logCropProfile("dialog-mounted", {
elapsedMs: Number(
(performance.now() - cropOpenedAt).toFixed(2),
),
recordingDurationSec: Math.round(
editorInstance.recordingDuration,
),
});

scheduleAccurateFrame("immediate", {
Copy link
Copy Markdown

@tembo tembo bot Apr 1, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right now the accurate frame fetch gets queued on every crop-open (even when a fresh previewUrl is available). If the intent is mostly fallback/upgrade, a small delay when we already have a preview can avoid unnecessary work for quick open/close.

Suggested change
scheduleAccurateFrame("immediate", {
scheduleAccurateFrame(initialPreviewUrl ? "preview-available" : "immediate", {
delayMs: initialPreviewUrl ? 250 : undefined,
idleTimeoutMs: 500,
fallbackDelayMs: 16,
});

idleTimeoutMs: 500,
fallbackDelayMs: 16,
});
});

onCleanup(() => {
cancelled = true;
clearScheduledAccurateFrame();
const url = frameBlobUrl();
if (url) {
URL.revokeObjectURL(url);
Expand Down Expand Up @@ -959,12 +1125,33 @@ function Dialogs() {
<img
class="shadow pointer-events-none max-h-[70vh]"
alt="Current frame"
src={
frameBlobUrl() ??
convertFileSrc(
`${editorInstance.path}/screenshots/display.jpg`,
)
onError={() => {
const failedSource = frameSource();
logCropProfile("preview-image-failed", {
elapsedMs: Number(
(performance.now() - cropOpenedAt).toFixed(
2,
),
),
source: failedSource,
});
requestAccurateFrame(
failedSource === "screenshot"
? "screenshot-load-failed"
: "preview-load-failed",
);
}}
onLoad={() =>
logCropProfile("preview-image-loaded", {
elapsedMs: Number(
(performance.now() - cropOpenedAt).toFixed(
2,
),
),
source: frameSource(),
})
}
src={frameBlobUrl() ?? screenshotSrc}
/>
</Cropper>
</div>
Expand Down
Loading
Loading