From eb538eeeb5a9cf735fb0c6208303279239c76448 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 1 Apr 2026 16:47:47 +0100 Subject: [PATCH 1/7] improve: reduce editor work for long recordings --- apps/desktop/src-tauri/src/lib.rs | 27 ++++ apps/desktop/src/routes/editor/Editor.tsx | 142 ++++++++++++++++-- apps/desktop/src/routes/editor/Player.tsx | 25 ++- .../routes/editor/Timeline/CaptionsTrack.tsx | 13 +- .../src/routes/editor/Timeline/ClipTrack.tsx | 45 +++--- .../routes/editor/Timeline/KeyboardTrack.tsx | 13 +- .../src/routes/editor/Timeline/MaskTrack.tsx | 33 +++- .../src/routes/editor/Timeline/SceneTrack.tsx | 18 +-- .../src/routes/editor/Timeline/TextTrack.tsx | 33 +++- .../src/routes/editor/Timeline/ZoomTrack.tsx | 19 +-- .../src/routes/editor/Timeline/index.tsx | 36 +++-- apps/desktop/src/routes/editor/context.ts | 9 -- .../src/routes/editor/cropVideoPreloader.ts | 47 ------ .../src/routes/editor/timelineTracks.ts | 27 +++- 14 files changed, 326 insertions(+), 161 deletions(-) delete mode 100644 apps/desktop/src/routes/editor/cropVideoPreloader.ts diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 8c0239f79f..9ca326820b 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -2764,11 +2764,15 @@ async fn get_display_frame_for_cropping( use cap_rendering::{PixelFormat, cpu_yuv}; use image::{ImageEncoder, codecs::png::PngEncoder}; use std::io::Cursor; + use std::time::Instant; + + let total_started_at = Instant::now(); let frame_number = editor_instance.state.lock().await.playhead_position; let time_secs = frame_number as f64 / fps as f64; let project = editor_instance.project_config.1.borrow().clone(); + let lookup_started_at = Instant::now(); let (segment_time, segment) = project .get_segment_time(time_secs) @@ -2785,12 +2789,15 @@ async fn get_display_frame_for_cropping( .find(|v| v.index == segment.recording_clip) .map(|v| v.offsets) .unwrap_or(ClipOffsets::default()); + let lookup_elapsed_ms = lookup_started_at.elapsed().as_secs_f64() * 1000.0; + let decode_started_at = Instant::now(); let segment_frames = segment_medias .decoders .get_frames(segment_time as f32, false, true, clip_offsets) .await .ok_or_else(|| "Failed to get frame".to_string())?; + let decode_elapsed_ms = decode_started_at.elapsed().as_secs_f64() * 1000.0; let screen_frame = segment_frames .screen_frame @@ -2798,6 +2805,7 @@ async fn get_display_frame_for_cropping( let width = screen_frame.width(); let height = screen_frame.height(); + let convert_started_at = Instant::now(); let rgba_data = match screen_frame.format() { PixelFormat::Rgba => screen_frame.data().to_vec(), PixelFormat::Nv12 => { @@ -2833,12 +2841,31 @@ async fn get_display_frame_for_cropping( rgba } }; + let convert_elapsed_ms = convert_started_at.elapsed().as_secs_f64() * 1000.0; + let encode_started_at = Instant::now(); let mut png_data = Cursor::new(Vec::new()); let encoder = PngEncoder::new(&mut png_data); encoder .write_image(&rgba_data, width, height, image::ExtendedColorType::Rgba8) .map_err(|e| format!("Failed to encode PNG: {e}"))?; + let encode_elapsed_ms = encode_started_at.elapsed().as_secs_f64() * 1000.0; + let total_elapsed_ms = total_started_at.elapsed().as_secs_f64() * 1000.0; + + info!( + target: "cap_crop_profile", + frame_number = frame_number, + time_secs = time_secs, + segment_time = segment_time, + width = width, + height = height, + lookup_ms = lookup_elapsed_ms, + decode_ms = decode_elapsed_ms, + convert_ms = convert_elapsed_ms, + encode_ms = encode_elapsed_ms, + total_ms = total_elapsed_ms, + "crop frame profile" + ); Ok(png_data.into_inner()) } diff --git a/apps/desktop/src/routes/editor/Editor.tsx b/apps/desktop/src/routes/editor/Editor.tsx index 967464273e..9fbc42db84 100644 --- a/apps/desktop/src/routes/editor/Editor.tsx +++ b/apps/desktop/src/routes/editor/Editor.tsx @@ -18,6 +18,7 @@ import { Match, on, onCleanup, + onMount, Show, Switch, } from "solid-js"; @@ -59,9 +60,18 @@ const MIN_PLAYER_CONTENT_HEIGHT = 320; const MIN_TIMELINE_HEIGHT = 240; const RESIZE_HANDLE_HEIGHT = 16; const MIN_PLAYER_HEIGHT = MIN_PLAYER_CONTENT_HEIGHT + RESIZE_HANDLE_HEIGHT; +const ACCURATE_CROP_PREVIEW_MAX_DURATION = 15 * 60; const TIMELINE_RESIZE_GRIP_MARKS = [0, 1, 2] as const; +function logCropProfile( + stage: string, + data: Record = {}, +) { + if (!import.meta.env.DEV) return; + console.info("[crop-profile]", stage, data); +} + function getEditorErrorMessage(error: unknown) { return error instanceof Error ? error.message : String(error); } @@ -775,21 +785,119 @@ function Dialogs() { const [frameBlobUrl, setFrameBlobUrl] = createSignal< string | null >(null); + const cropOpenedAt = performance.now(); + + let cancelled = false; + let frameLoadTimeoutId: + | ReturnType + | undefined; + let frameLoadIdleId: number | undefined; + const idleWindow = globalThis as typeof globalThis & { + requestIdleCallback?: ( + callback: () => void, + options?: { timeout?: number }, + ) => number; + cancelIdleCallback?: (handle: number) => void; + }; - commands - .getDisplayFrameForCropping(FPS) - .then((pngBytes) => { - const blob = new Blob([new Uint8Array(pngBytes)], { - type: "image/png", - }); - const url = URL.createObjectURL(blob); - setFrameBlobUrl(url); - }) - .catch((error: unknown) => { - console.warn("Display frame fetch failed:", error); + onMount(() => { + logCropProfile("dialog-mounted", { + elapsedMs: Number( + (performance.now() - cropOpenedAt).toFixed(2), + ), + recordingDurationSec: Math.round( + editorInstance.recordingDuration, + ), }); + if ( + editorInstance.recordingDuration > + ACCURATE_CROP_PREVIEW_MAX_DURATION + ) { + logCropProfile("accurate-frame-skipped", { + elapsedMs: Number( + (performance.now() - cropOpenedAt).toFixed(2), + ), + recordingDurationSec: Math.round( + editorInstance.recordingDuration, + ), + }); + return; + } + + const loadFrame = () => { + if (cancelled) return; + const frameRequestStartedAt = performance.now(); + logCropProfile("accurate-frame-request-start", { + elapsedMs: Number( + (frameRequestStartedAt - cropOpenedAt).toFixed(2), + ), + }); + + void commands + .getDisplayFrameForCropping(FPS) + .then((pngBytes) => { + if (cancelled) return; + + const blob = new Blob([new Uint8Array(pngBytes)], { + type: "image/png", + }); + const nextUrl = URL.createObjectURL(blob); + const prevUrl = frameBlobUrl(); + setFrameBlobUrl(nextUrl); + if (prevUrl) URL.revokeObjectURL(prevUrl); + logCropProfile("accurate-frame-request-finish", { + elapsedMs: Number( + (performance.now() - cropOpenedAt).toFixed(2), + ), + requestMs: Number( + (performance.now() - frameRequestStartedAt).toFixed( + 2, + ), + ), + }); + }) + .catch((error: unknown) => { + if (cancelled) return; + console.warn("Display frame fetch failed:", error); + logCropProfile("accurate-frame-request-failed", { + elapsedMs: Number( + (performance.now() - cropOpenedAt).toFixed(2), + ), + requestMs: Number( + (performance.now() - frameRequestStartedAt).toFixed( + 2, + ), + ), + message: + error instanceof Error + ? error.message + : String(error), + }); + }); + }; + + if (idleWindow.requestIdleCallback) { + frameLoadIdleId = idleWindow.requestIdleCallback( + loadFrame, + { + timeout: 500, + }, + ); + return; + } + + frameLoadTimeoutId = globalThis.setTimeout(loadFrame, 16); + }); + onCleanup(() => { + cancelled = true; + if (frameLoadIdleId !== undefined) { + idleWindow.cancelIdleCallback?.(frameLoadIdleId); + } + if (frameLoadTimeoutId !== undefined) { + globalThis.clearTimeout(frameLoadTimeoutId); + } const url = frameBlobUrl(); if (url) { URL.revokeObjectURL(url); @@ -959,6 +1067,18 @@ function Dialogs() { Current frame + logCropProfile("preview-image-loaded", { + elapsedMs: Number( + (performance.now() - cropOpenedAt).toFixed( + 2, + ), + ), + source: frameBlobUrl() + ? "accurate-frame" + : "screenshot", + }) + } src={ frameBlobUrl() ?? convertFileSrc( diff --git a/apps/desktop/src/routes/editor/Player.tsx b/apps/desktop/src/routes/editor/Player.tsx index e6feccb354..60e622dfe0 100644 --- a/apps/desktop/src/routes/editor/Player.tsx +++ b/apps/desktop/src/routes/editor/Player.tsx @@ -18,7 +18,6 @@ import { serializeProjectConfiguration, useEditorContext, } from "./context"; -import { preloadCropVideoFull } from "./cropVideoPreloader"; import { MaskOverlay } from "./MaskOverlay"; import { PerformanceOverlay } from "./PerformanceOverlay"; import { TextOverlay } from "./TextOverlay"; @@ -33,6 +32,14 @@ import { import { useEditorShortcuts } from "./useEditorShortcuts"; import { formatTime } from "./utils"; +function logCropProfile( + stage: string, + data: Record = {}, +) { + if (!import.meta.env.DEV) return; + console.info("[crop-profile]", stage, data); +} + export function PlayerContent() { const { project, @@ -149,7 +156,15 @@ export function PlayerContent() { }; const cropDialogHandler = async () => { + const startedAt = performance.now(); const display = editorInstance.recordings.segments[0].display; + logCropProfile("click", { + recordingDurationSec: Math.round(editorInstance.recordingDuration), + playbackTimeSec: Number(editorState.playbackTime.toFixed(3)), + displayWidth: display.width, + displayHeight: display.height, + wasPlaying: editorState.playing, + }); setDialog({ open: true, type: "crop", @@ -163,7 +178,13 @@ export function PlayerContent() { }), }, }); + logCropProfile("dialog-opened", { + elapsedMs: Number((performance.now() - startedAt).toFixed(2)), + }); await commands.stopPlayback(); + logCropProfile("playback-stopped", { + elapsedMs: Number((performance.now() - startedAt).toFixed(2)), + }); setEditorState("playing", false); }; @@ -283,8 +304,6 @@ export function PlayerContent() { } > Crop diff --git a/apps/desktop/src/routes/editor/Timeline/CaptionsTrack.tsx b/apps/desktop/src/routes/editor/Timeline/CaptionsTrack.tsx index caa71805a2..c7f67ca76a 100644 --- a/apps/desktop/src/routes/editor/Timeline/CaptionsTrack.tsx +++ b/apps/desktop/src/routes/editor/Timeline/CaptionsTrack.tsx @@ -29,12 +29,17 @@ export function CaptionsTrack(props: { projectHistory, projectActions, } = useEditorContext(); - const { secsPerPixel, timelineBounds } = useTimelineContext(); + const { secsPerPixel } = useTimelineContext(); const minDuration = () => Math.max(MIN_SEGMENT_SECS, secsPerPixel() * MIN_SEGMENT_PIXELS); const captionSegments = () => project.timeline?.captionSegments ?? []; + const selectedCaptionIndices = createMemo(() => { + const selection = editorState.timeline.selection; + if (!selection || selection.type !== "caption") return null; + return new Set(selection.indices); + }); const neighborBounds = (index: number) => { const segments = captionSegments(); @@ -147,9 +152,9 @@ export function CaptionsTrack(props: { > {(segment, i) => { const isSelected = createMemo(() => { - const selection = editorState.timeline.selection; - if (!selection || selection.type !== "caption") return false; - return selection.indices.includes(i()); + const indices = selectedCaptionIndices(); + if (!indices) return false; + return indices.has(i()); }); const segmentWidth = () => segment.end - segment.start; diff --git a/apps/desktop/src/routes/editor/Timeline/ClipTrack.tsx b/apps/desktop/src/routes/editor/Timeline/ClipTrack.tsx index 03e28c5d21..68c13f09e1 100644 --- a/apps/desktop/src/routes/editor/Timeline/ClipTrack.tsx +++ b/apps/desktop/src/routes/editor/Timeline/ClipTrack.tsx @@ -325,6 +325,19 @@ export function ClipTrack( const segments = (): Array => project.timeline?.segments ?? [{ start: 0, end: duration(), timescale: 1 }]; + const selectedClipIndices = createMemo(() => { + const selection = editorState.timeline.selection; + if (!selection || selection.type !== "clip") return null; + return new Set(selection.indices); + }); + const totalClipTimelineDuration = createMemo(() => { + const segs = segments(); + let total = 0; + for (let i = 0; i < segs.length; i++) { + total += (segs[i].end - segs[i].start) / segs[i].timescale; + } + return total; + }); const segmentOffsets = createMemo(() => { const segs = segments(); @@ -421,17 +434,9 @@ export function ClipTrack( })); const isSelected = createMemo(() => { - const selection = editorState.timeline.selection; - if (!selection || selection.type !== "clip") return false; - const seg = segment(); - - const segmentIndex = project.timeline?.segments?.findIndex( - (s) => s.start === seg.start && s.end === seg.end, - ); - - if (segmentIndex === undefined || segmentIndex === -1) return false; - - return selection.indices.includes(segmentIndex); + const indices = selectedClipIndices(); + if (!indices) return false; + return indices.has(i()); }); const micWaveform = () => { @@ -678,13 +683,8 @@ export function ClipTrack( const availableTimelineDuration = editorInstance.recordingDuration - - segments().reduce( - (acc, s, segmentI) => - segmentI === i() - ? acc - : acc + (s.end - s.start) / s.timescale, - 0, - ); + (totalClipTimelineDuration() - + (seg.end - seg.start) / seg.timescale); const maxDuration = Math.min( maxSegmentDuration, @@ -796,13 +796,8 @@ export function ClipTrack( const availableTimelineDuration = editorInstance.recordingDuration - - segments().reduce( - (acc, s, segmentI) => - segmentI === i() - ? acc - : acc + (s.end - s.start) / s.timescale, - 0, - ); + (totalClipTimelineDuration() - + (seg.end - seg.start) / seg.timescale); const nextSegment = segments()[i() + 1]; const nextSegmentIsSameClip = diff --git a/apps/desktop/src/routes/editor/Timeline/KeyboardTrack.tsx b/apps/desktop/src/routes/editor/Timeline/KeyboardTrack.tsx index 2a470c3823..52c3d6feb2 100644 --- a/apps/desktop/src/routes/editor/Timeline/KeyboardTrack.tsx +++ b/apps/desktop/src/routes/editor/Timeline/KeyboardTrack.tsx @@ -27,12 +27,17 @@ export function KeyboardTrack(props: { projectHistory, projectActions, } = useEditorContext(); - const { secsPerPixel, timelineBounds } = useTimelineContext(); + const { secsPerPixel } = useTimelineContext(); const minDuration = () => Math.max(MIN_SEGMENT_SECS, secsPerPixel() * MIN_SEGMENT_PIXELS); const keyboardSegments = () => project.timeline?.keyboardSegments ?? []; + const selectedKeyboardIndices = createMemo(() => { + const selection = editorState.timeline.selection; + if (!selection || selection.type !== "keyboard") return null; + return new Set(selection.indices); + }); const neighborBounds = (index: number) => { const segments = keyboardSegments(); @@ -139,9 +144,9 @@ export function KeyboardTrack(props: { > {(segment, i) => { const isSelected = createMemo(() => { - const selection = editorState.timeline.selection; - if (!selection || selection.type !== "keyboard") return false; - return selection.indices.includes(i()); + const indices = selectedKeyboardIndices(); + if (!indices) return false; + return indices.has(i()); }); const segmentWidth = () => segment.end - segment.start; diff --git a/apps/desktop/src/routes/editor/Timeline/MaskTrack.tsx b/apps/desktop/src/routes/editor/Timeline/MaskTrack.tsx index 944521f76d..2c9359cade 100644 --- a/apps/desktop/src/routes/editor/Timeline/MaskTrack.tsx +++ b/apps/desktop/src/routes/editor/Timeline/MaskTrack.tsx @@ -51,10 +51,30 @@ export function MaskTrack(props: { .map((segment, index) => ({ segment, index })) .filter(({ segment }) => getSegmentTrack(segment) === props.laneIndex), ); + const laneSegmentPositionByIndex = createMemo(() => { + const positions = new Map(); + const segments = laneSegments(); + for (let i = 0; i < segments.length; i++) { + positions.set(segments[i].index, i); + } + return positions; + }); + const sortedLaneSegments = createMemo(() => { + const sorted = laneSegments() + .map(({ segment }) => segment) + .slice(); + sorted.sort((a, b) => a.start - b.start); + return sorted; + }); + const selectedMaskIndices = createMemo(() => { + const selection = editorState.timeline.selection; + if (!selection || selection.type !== "mask") return null; + return new Set(selection.indices); + }); const neighborBounds = (index: number) => { const segments = laneSegments(); - const laneIndex = segments.findIndex((segment) => segment.index === index); + const laneIndex = laneSegmentPositionByIndex().get(index) ?? -1; return { prevEnd: segments[laneIndex - 1]?.segment.end ?? 0, nextStart: segments[laneIndex + 1]?.segment.start ?? totalDuration(), @@ -63,10 +83,7 @@ export function MaskTrack(props: { const findPlacement = (time: number, length: number) => { const gaps: Array<{ start: number; end: number }> = []; - const sorted = laneSegments() - .map(({ segment }) => segment) - .slice() - .sort((a, b) => a.start - b.start); + const sorted = sortedLaneSegments(); let cursor = 0; for (const segment of sorted) { @@ -301,9 +318,9 @@ export function MaskTrack(props: { > {({ segment, index }) => { const isSelected = createMemo(() => { - const selection = editorState.timeline.selection; - if (!selection || selection.type !== "mask") return false; - return selection.indices.includes(index); + const indices = selectedMaskIndices(); + if (!indices) return false; + return indices.has(index); }); const contentLabel = () => diff --git a/apps/desktop/src/routes/editor/Timeline/SceneTrack.tsx b/apps/desktop/src/routes/editor/Timeline/SceneTrack.tsx index a864e828ea..b0a24f7dba 100644 --- a/apps/desktop/src/routes/editor/Timeline/SceneTrack.tsx +++ b/apps/desktop/src/routes/editor/Timeline/SceneTrack.tsx @@ -50,6 +50,11 @@ export function SceneTrack(props: { const { duration, secsPerPixel } = useTimelineContext(); const setPreviewTime = useSetPreviewTime(); + const selectedSceneIndices = createMemo(() => { + const selection = editorState.timeline.selection; + if (!selection || selection.type !== "scene") return null; + return new Set(selection.indices); + }); const [hoveringSegment, setHoveringSegment] = createSignal(false); const [hoveredTime, setHoveredTime] = createSignal(); @@ -349,16 +354,9 @@ export function SceneTrack(props: { } const isSelected = createMemo(() => { - const selection = editorState.timeline.selection; - if (!selection || selection.type !== "scene") return false; - - const segmentIndex = project.timeline?.sceneSegments?.findIndex( - (s) => s.start === segment.start && s.end === segment.end, - ); - - if (segmentIndex === undefined || segmentIndex === -1) return false; - - return selection.indices.includes(segmentIndex); + const indices = selectedSceneIndices(); + if (!indices) return false; + return indices.has(i()); }); return ( diff --git a/apps/desktop/src/routes/editor/Timeline/TextTrack.tsx b/apps/desktop/src/routes/editor/Timeline/TextTrack.tsx index c148297fc6..46646b7570 100644 --- a/apps/desktop/src/routes/editor/Timeline/TextTrack.tsx +++ b/apps/desktop/src/routes/editor/Timeline/TextTrack.tsx @@ -51,10 +51,30 @@ export function TextTrack(props: { .map((segment, index) => ({ segment, index })) .filter(({ segment }) => getSegmentTrack(segment) === props.laneIndex), ); + const laneSegmentPositionByIndex = createMemo(() => { + const positions = new Map(); + const segments = laneSegments(); + for (let i = 0; i < segments.length; i++) { + positions.set(segments[i].index, i); + } + return positions; + }); + const sortedLaneSegments = createMemo(() => { + const sorted = laneSegments() + .map(({ segment }) => segment) + .slice(); + sorted.sort((a, b) => a.start - b.start); + return sorted; + }); + const selectedTextIndices = createMemo(() => { + const selection = editorState.timeline.selection; + if (!selection || selection.type !== "text") return null; + return new Set(selection.indices); + }); const neighborBounds = (index: number) => { const segments = laneSegments(); - const laneIndex = segments.findIndex((segment) => segment.index === index); + const laneIndex = laneSegmentPositionByIndex().get(index) ?? -1; return { prevEnd: segments[laneIndex - 1]?.segment.end ?? 0, nextStart: segments[laneIndex + 1]?.segment.start ?? totalDuration(), @@ -63,10 +83,7 @@ export function TextTrack(props: { const findPlacement = (time: number, length: number) => { const gaps: Array<{ start: number; end: number }> = []; - const sorted = laneSegments() - .map(({ segment }) => segment) - .slice() - .sort((a, b) => a.start - b.start); + const sorted = sortedLaneSegments(); let cursor = 0; for (const segment of sorted) { @@ -274,9 +291,9 @@ export function TextTrack(props: { > {({ segment, index }) => { const isSelected = createMemo(() => { - const selection = editorState.timeline.selection; - if (!selection || selection.type !== "text") return false; - return selection.indices.includes(index); + const indices = selectedTextIndices(); + if (!indices) return false; + return indices.has(index); }); const segmentWidth = () => segment.end - segment.start; diff --git a/apps/desktop/src/routes/editor/Timeline/ZoomTrack.tsx b/apps/desktop/src/routes/editor/Timeline/ZoomTrack.tsx index b293893543..805f28632c 100644 --- a/apps/desktop/src/routes/editor/Timeline/ZoomTrack.tsx +++ b/apps/desktop/src/routes/editor/Timeline/ZoomTrack.tsx @@ -69,6 +69,11 @@ export function ZoomTrack(props: { const hasZoomSegments = () => (project.timeline?.zoomSegments?.length ?? 0) > 0; + const selectedZoomIndices = createMemo(() => { + const selection = editorState.timeline.selection; + if (!selection || selection.type !== "zoom") return null; + return new Set(selection.indices); + }); const hasRecordedCursorData = () => meta().hasRecordedCursorData; @@ -465,17 +470,9 @@ export function ZoomTrack(props: { } const isSelected = createMemo(() => { - const selection = editorState.timeline.selection; - if (!selection || selection.type !== "zoom") return false; - - const segmentIndex = project.timeline?.zoomSegments?.findIndex( - (s) => s.start === segment().start && s.end === segment().end, - ); - - if (segmentIndex === undefined || segmentIndex === -1) - return false; - - return selection.indices.includes(segmentIndex); + const indices = selectedZoomIndices(); + if (!indices) return false; + return indices.has(i); }); return ( diff --git a/apps/desktop/src/routes/editor/Timeline/index.tsx b/apps/desktop/src/routes/editor/Timeline/index.tsx index 28ae8a3dff..7bd88a5a8a 100644 --- a/apps/desktop/src/routes/editor/Timeline/index.tsx +++ b/apps/desktop/src/routes/editor/Timeline/index.tsx @@ -7,6 +7,7 @@ import { cx } from "cva"; import { batch, createEffect, + createMemo, createRoot, createSignal, For, @@ -161,7 +162,7 @@ export function Timeline(props: { const sceneAvailable = () => meta().hasCamera && !project.camera.hide; const captionTrackVisible = () => trackState().caption; const keyboardTrackVisible = () => trackState().keyboard; - const trackOptions = () => + const trackOptions = createMemo(() => trackDefinitions.map((definition) => ({ ...definition, active: @@ -179,26 +180,33 @@ export function Timeline(props: { available: definition.type === "scene" ? sceneAvailable() : true, supportsMultiple: definition.type === "mask" || definition.type === "text", - })); + })), + ); const sceneTrackVisible = () => trackState().scene && sceneAvailable(); - const textTrackRows = () => + const textTrackRows = createMemo(() => getTrackRowsWithCount( project.timeline?.textSegments ?? [], trackState().text, - ); - const maskTrackRows = () => + ), + ); + const maskTrackRows = createMemo(() => getTrackRowsWithCount( project.timeline?.maskSegments ?? [], trackState().mask, - ); - const visibleTrackCount = () => - 2 + - (captionTrackVisible() ? 1 : 0) + - (keyboardTrackVisible() ? 1 : 0) + - textTrackRows().length + - maskTrackRows().length + - (sceneTrackVisible() ? 1 : 0); - const trackHeight = () => (visibleTrackCount() > 2 ? "3rem" : "3.25rem"); + ), + ); + const visibleTrackCount = createMemo( + () => + 2 + + (captionTrackVisible() ? 1 : 0) + + (keyboardTrackVisible() ? 1 : 0) + + textTrackRows().length + + maskTrackRows().length + + (sceneTrackVisible() ? 1 : 0), + ); + const trackHeight = createMemo(() => + visibleTrackCount() > 2 ? "3rem" : "3.25rem", + ); createEffect(() => { const visibleTracks = visibleTrackCount(); diff --git a/apps/desktop/src/routes/editor/context.ts b/apps/desktop/src/routes/editor/context.ts index 3170b395fd..b3bdf557be 100644 --- a/apps/desktop/src/routes/editor/context.ts +++ b/apps/desktop/src/routes/editor/context.ts @@ -45,10 +45,6 @@ import { type TimelineConfiguration, type XY, } from "~/utils/tauri"; -import { - cleanup as cleanupCropVideoPreloader, - preloadCropVideoMetadata, -} from "./cropVideoPreloader"; import type { MaskSegment } from "./masks"; import type { TextSegment } from "./text"; import { @@ -1043,7 +1039,6 @@ const createEditorInstanceContext = () => { onCleanup(() => { disposeWorkerReadyEffect?.(); canvasControls()?.dispose(); - cleanupCropVideoPreloader(); }); const [editorInstance, { refetch: refetchEditorInstance }] = createResource( @@ -1079,10 +1074,6 @@ const createEditorInstanceContext = () => { console.log("[Editor] Editor instance created, setting up WebSocket"); - preloadCropVideoMetadata( - `${instance.path}/content/segments/segment-0/display.mp4`, - ); - const requestFrame = () => { events.renderFrameEvent.emit({ frame_number: 0, diff --git a/apps/desktop/src/routes/editor/cropVideoPreloader.ts b/apps/desktop/src/routes/editor/cropVideoPreloader.ts deleted file mode 100644 index 4f1ee17d08..0000000000 --- a/apps/desktop/src/routes/editor/cropVideoPreloader.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { convertFileSrc } from "@tauri-apps/api/core"; - -let preloadedVideo: HTMLVideoElement | null = null; -let preloadState: "idle" | "metadata" | "full" | "ready" = "idle"; -let currentVideoPath: string | null = null; - -export function preloadCropVideoMetadata(videoPath: string) { - if (preloadState !== "idle") return; - - currentVideoPath = videoPath; - preloadedVideo = document.createElement("video"); - preloadedVideo.preload = "metadata"; - preloadedVideo.src = convertFileSrc(videoPath); - preloadedVideo.muted = true; - preloadedVideo.load(); - preloadState = "metadata"; -} - -export function preloadCropVideoFull() { - if (!preloadedVideo || preloadState === "full" || preloadState === "ready") - return; - - preloadedVideo.preload = "auto"; - preloadedVideo.load(); - preloadState = "full"; - - preloadedVideo.oncanplaythrough = () => { - preloadState = "ready"; - }; -} - -export function getPreloadState() { - return preloadState; -} - -export function getPreloadedVideoPath() { - return currentVideoPath; -} - -export function cleanup() { - if (preloadedVideo) { - preloadedVideo.src = ""; - preloadedVideo = null; - } - currentVideoPath = null; - preloadState = "idle"; -} diff --git a/apps/desktop/src/routes/editor/timelineTracks.ts b/apps/desktop/src/routes/editor/timelineTracks.ts index 205e3bc48b..d2cb805d54 100644 --- a/apps/desktop/src/routes/editor/timelineTracks.ts +++ b/apps/desktop/src/routes/editor/timelineTracks.ts @@ -37,18 +37,31 @@ export function getTrackRows(segments: T[]) { } export function getUsedTrackCount(segments: T[]) { - if (segments.length === 0) return 0; - return Math.max(...segments.map((segment) => getSegmentTrack(segment))) + 1; + let maxTrack = -1; + for (let i = 0; i < segments.length; i++) { + const track = getSegmentTrack(segments[i]); + if (track > maxTrack) { + maxTrack = track; + } + } + return maxTrack + 1; } export function getTrackRowsWithCount( segments: T[], count: number, ) { - const maxRow = Math.max( - count - 1, - ...segments.map((segment) => getSegmentTrack(segment)), - ); + let maxRow = count - 1; + for (let i = 0; i < segments.length; i++) { + const track = getSegmentTrack(segments[i]); + if (track > maxRow) { + maxRow = track; + } + } if (maxRow < 0) return []; - return Array.from({ length: maxRow + 1 }, (_, index) => index); + const rows = new Array(maxRow + 1); + for (let i = 0; i <= maxRow; i++) { + rows[i] = i; + } + return rows; } From be7bbd0cc3a3c5b65c9782b74fe24928fb85a4af Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 1 Apr 2026 19:01:02 +0100 Subject: [PATCH 2/7] build(desktop): enable window devtools only in debug builds Made-with: Cursor --- apps/desktop/src-tauri/Cargo.toml | 1 - apps/desktop/src-tauri/src/windows.rs | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index 301d01e5b7..9443e2802a 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -24,7 +24,6 @@ tauri = { workspace = true, features = [ "protocol-asset", "tray-icon", "image-png", - "devtools", ] } tauri-specta = { version = "=2.0.0-rc.20", features = ["derive", "typescript"] } tauri-plugin-dialog = "2.2.0" diff --git a/apps/desktop/src-tauri/src/windows.rs b/apps/desktop/src-tauri/src/windows.rs index 89eacf6d0b..2199e66914 100644 --- a/apps/desktop/src-tauri/src/windows.rs +++ b/apps/desktop/src-tauri/src/windows.rs @@ -2120,7 +2120,8 @@ impl ShowCapWindow { .visible(false) .accept_first_mouse(true) .shadow(true) - .theme(theme); + .theme(theme) + .devtools(cfg!(debug_assertions)); if !id.is_transparent() { let is_dark = match theme { From 0da6437b63e320c5c5371408c470e9ab94ee3aa6 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 1 Apr 2026 19:01:04 +0100 Subject: [PATCH 3/7] refactor(desktop): demote crop frame profile logging to debug Made-with: Cursor --- apps/desktop/src-tauri/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 9ca326820b..e40200ec40 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -2852,7 +2852,7 @@ async fn get_display_frame_for_cropping( let encode_elapsed_ms = encode_started_at.elapsed().as_secs_f64() * 1000.0; let total_elapsed_ms = total_started_at.elapsed().as_secs_f64() * 1000.0; - info!( + debug!( target: "cap_crop_profile", frame_number = frame_number, time_secs = time_secs, From 600cd012e073c579575a8cc821c1838c7b56dca9 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 1 Apr 2026 19:01:04 +0100 Subject: [PATCH 4/7] feat(editor): add optional previewUrl to crop dialog state Made-with: Cursor --- apps/desktop/src/routes/editor/context.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/desktop/src/routes/editor/context.ts b/apps/desktop/src/routes/editor/context.ts index b3bdf557be..4fec1fd8ce 100644 --- a/apps/desktop/src/routes/editor/context.ts +++ b/apps/desktop/src/routes/editor/context.ts @@ -58,7 +58,12 @@ export type ModalDialog = | { type: "createPreset" } | { type: "renamePreset"; presetIndex: number } | { type: "deletePreset"; presetIndex: number } - | { type: "crop"; position: XY; size: XY }; + | { + type: "crop"; + position: XY; + size: XY; + previewUrl?: string | null; + }; export type LayoutMode = { type: "export" } | { type: "transcript" }; From 8ac0875dc49dfbe538826fddd7b6a5cc197b5373 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 1 Apr 2026 19:01:05 +0100 Subject: [PATCH 5/7] feat(editor): capture canvas frame when opening crop dialog Made-with: Cursor --- apps/desktop/src/routes/editor/Player.tsx | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/apps/desktop/src/routes/editor/Player.tsx b/apps/desktop/src/routes/editor/Player.tsx index 60e622dfe0..a35b07b865 100644 --- a/apps/desktop/src/routes/editor/Player.tsx +++ b/apps/desktop/src/routes/editor/Player.tsx @@ -50,6 +50,7 @@ export function PlayerContent() { setEditorState, zoomOutLimit, setProject, + canvasControls, previewResolutionBase, previewQuality, setPreviewQuality, @@ -158,6 +159,8 @@ export function PlayerContent() { const cropDialogHandler = async () => { const startedAt = performance.now(); const display = editorInstance.recordings.segments[0].display; + const controls = canvasControls(); + let previewUrl: string | null = null; logCropProfile("click", { recordingDurationSec: Math.round(editorInstance.recordingDuration), playbackTimeSec: Number(editorState.playbackTime.toFixed(3)), @@ -165,6 +168,20 @@ export function PlayerContent() { displayHeight: display.height, wasPlaying: editorState.playing, }); + if (controls?.hasRenderedFrame()) { + try { + const previewFrame = await controls.captureFrame(); + if (previewFrame) { + previewUrl = URL.createObjectURL(previewFrame); + } + } catch (error) { + console.warn("Preview frame capture failed:", error); + } + } + logCropProfile("preview-frame-captured", { + elapsedMs: Number((performance.now() - startedAt).toFixed(2)), + available: previewUrl !== null, + }); setDialog({ open: true, type: "crop", @@ -177,6 +194,7 @@ export function PlayerContent() { y: display.height, }), }, + previewUrl, }); logCropProfile("dialog-opened", { elapsedMs: Number((performance.now() - startedAt).toFixed(2)), From e82d913a8878bcbd81dcb1ccdec0dbbe39a0bd30 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 1 Apr 2026 19:01:05 +0100 Subject: [PATCH 6/7] feat(editor): improve crop preview for long recordings and load failures Made-with: Cursor --- apps/desktop/src/routes/editor/Editor.tsx | 251 +++++++++++++++------- 1 file changed, 171 insertions(+), 80 deletions(-) diff --git a/apps/desktop/src/routes/editor/Editor.tsx b/apps/desktop/src/routes/editor/Editor.tsx index 9fbc42db84..d318acd764 100644 --- a/apps/desktop/src/routes/editor/Editor.tsx +++ b/apps/desktop/src/routes/editor/Editor.tsx @@ -782,16 +782,27 @@ function Dialogs() { const [crop, setCrop] = createSignal(CROP_ZERO); const [aspect, setAspect] = createSignal(null); + const initialPreviewUrl = dialog().previewUrl ?? null; const [frameBlobUrl, setFrameBlobUrl] = createSignal< string | null - >(null); + >(initialPreviewUrl); + const [frameSource, setFrameSource] = createSignal< + "captured-preview" | "accurate-frame" | "screenshot" + >(initialPreviewUrl ? "captured-preview" : "screenshot"); const cropOpenedAt = performance.now(); + const screenshotSrc = convertFileSrc( + `${editorInstance.path}/screenshots/display.jpg`, + ); let cancelled = false; + let frameLoadDelayTimeoutId: + | ReturnType + | undefined; let frameLoadTimeoutId: | ReturnType | undefined; let frameLoadIdleId: number | undefined; + let accurateFrameRequested = false; const idleWindow = globalThis as typeof globalThis & { requestIdleCallback?: ( callback: () => void, @@ -800,6 +811,134 @@ function Dialogs() { cancelIdleCallback?: (handle: number) => void; }; + const clearScheduledAccurateFrame = () => { + if (frameLoadDelayTimeoutId !== undefined) { + globalThis.clearTimeout(frameLoadDelayTimeoutId); + frameLoadDelayTimeoutId = undefined; + } + if (frameLoadIdleId !== undefined) { + idleWindow.cancelIdleCallback?.(frameLoadIdleId); + frameLoadIdleId = undefined; + } + if (frameLoadTimeoutId !== undefined) { + globalThis.clearTimeout(frameLoadTimeoutId); + frameLoadTimeoutId = undefined; + } + }; + + const setPreviewBlob = ( + blob: Blob, + source: "accurate-frame", + ) => { + const nextUrl = URL.createObjectURL(blob); + const previousUrl = frameBlobUrl(); + setFrameBlobUrl(nextUrl); + setFrameSource(source); + if (previousUrl) { + URL.revokeObjectURL(previousUrl); + } + }; + + const requestAccurateFrame = (reason: string) => { + if (accurateFrameRequested || cancelled) return; + + clearScheduledAccurateFrame(); + + accurateFrameRequested = true; + const frameRequestStartedAt = performance.now(); + logCropProfile("accurate-frame-request-start", { + elapsedMs: Number( + (frameRequestStartedAt - cropOpenedAt).toFixed(2), + ), + reason, + }); + + void commands + .getDisplayFrameForCropping(FPS) + .then((pngBytes) => { + if (cancelled) return; + + setPreviewBlob( + new Blob([new Uint8Array(pngBytes)], { + type: "image/png", + }), + "accurate-frame", + ); + logCropProfile("accurate-frame-request-finish", { + elapsedMs: Number( + (performance.now() - cropOpenedAt).toFixed(2), + ), + requestMs: Number( + (performance.now() - frameRequestStartedAt).toFixed( + 2, + ), + ), + reason, + }); + }) + .catch((error: unknown) => { + if (cancelled) return; + console.warn("Display frame fetch failed:", error); + logCropProfile("accurate-frame-request-failed", { + elapsedMs: Number( + (performance.now() - cropOpenedAt).toFixed(2), + ), + requestMs: Number( + (performance.now() - frameRequestStartedAt).toFixed( + 2, + ), + ), + message: + error instanceof Error + ? error.message + : String(error), + reason, + }); + }); + }; + + const scheduleAccurateFrame = ( + reason: string, + options: { + delayMs?: number; + idleTimeoutMs: number; + fallbackDelayMs: number; + }, + ) => { + const queueIdleFrame = () => { + const loadFrame = () => requestAccurateFrame(reason); + + if (idleWindow.requestIdleCallback) { + frameLoadIdleId = idleWindow.requestIdleCallback( + () => { + frameLoadIdleId = undefined; + loadFrame(); + }, + { + timeout: options.idleTimeoutMs, + }, + ); + return; + } + + frameLoadTimeoutId = globalThis.setTimeout(() => { + frameLoadTimeoutId = undefined; + loadFrame(); + }, options.fallbackDelayMs); + }; + + if (!options.delayMs) { + queueIdleFrame(); + return; + } + + frameLoadDelayTimeoutId = globalThis.setTimeout(() => { + frameLoadDelayTimeoutId = undefined; + if (cancelled || accurateFrameRequested) return; + queueIdleFrame(); + }, options.delayMs); + }; + onMount(() => { logCropProfile("dialog-mounted", { elapsedMs: Number( @@ -812,92 +951,35 @@ function Dialogs() { if ( editorInstance.recordingDuration > - ACCURATE_CROP_PREVIEW_MAX_DURATION + ACCURATE_CROP_PREVIEW_MAX_DURATION && + initialPreviewUrl ) { - logCropProfile("accurate-frame-skipped", { + logCropProfile("accurate-frame-deferred", { elapsedMs: Number( (performance.now() - cropOpenedAt).toFixed(2), ), recordingDurationSec: Math.round( editorInstance.recordingDuration, ), + reason: "current-preview-available", }); - return; - } - - const loadFrame = () => { - if (cancelled) return; - const frameRequestStartedAt = performance.now(); - logCropProfile("accurate-frame-request-start", { - elapsedMs: Number( - (frameRequestStartedAt - cropOpenedAt).toFixed(2), - ), + scheduleAccurateFrame("deferred-long-recording", { + delayMs: 2000, + idleTimeoutMs: 500, + fallbackDelayMs: 0, }); - - void commands - .getDisplayFrameForCropping(FPS) - .then((pngBytes) => { - if (cancelled) return; - - const blob = new Blob([new Uint8Array(pngBytes)], { - type: "image/png", - }); - const nextUrl = URL.createObjectURL(blob); - const prevUrl = frameBlobUrl(); - setFrameBlobUrl(nextUrl); - if (prevUrl) URL.revokeObjectURL(prevUrl); - logCropProfile("accurate-frame-request-finish", { - elapsedMs: Number( - (performance.now() - cropOpenedAt).toFixed(2), - ), - requestMs: Number( - (performance.now() - frameRequestStartedAt).toFixed( - 2, - ), - ), - }); - }) - .catch((error: unknown) => { - if (cancelled) return; - console.warn("Display frame fetch failed:", error); - logCropProfile("accurate-frame-request-failed", { - elapsedMs: Number( - (performance.now() - cropOpenedAt).toFixed(2), - ), - requestMs: Number( - (performance.now() - frameRequestStartedAt).toFixed( - 2, - ), - ), - message: - error instanceof Error - ? error.message - : String(error), - }); - }); - }; - - if (idleWindow.requestIdleCallback) { - frameLoadIdleId = idleWindow.requestIdleCallback( - loadFrame, - { - timeout: 500, - }, - ); return; } - frameLoadTimeoutId = globalThis.setTimeout(loadFrame, 16); + scheduleAccurateFrame("immediate", { + idleTimeoutMs: 500, + fallbackDelayMs: 16, + }); }); onCleanup(() => { cancelled = true; - if (frameLoadIdleId !== undefined) { - idleWindow.cancelIdleCallback?.(frameLoadIdleId); - } - if (frameLoadTimeoutId !== undefined) { - globalThis.clearTimeout(frameLoadTimeoutId); - } + clearScheduledAccurateFrame(); const url = frameBlobUrl(); if (url) { URL.revokeObjectURL(url); @@ -1067,6 +1149,22 @@ function Dialogs() { Current frame { + const failedSource = frameSource(); + logCropProfile("preview-image-failed", { + elapsedMs: Number( + (performance.now() - cropOpenedAt).toFixed( + 2, + ), + ), + source: failedSource, + }); + requestAccurateFrame( + failedSource === "screenshot" + ? "screenshot-load-failed" + : "preview-load-failed", + ); + }} onLoad={() => logCropProfile("preview-image-loaded", { elapsedMs: Number( @@ -1074,17 +1172,10 @@ function Dialogs() { 2, ), ), - source: frameBlobUrl() - ? "accurate-frame" - : "screenshot", + source: frameSource(), }) } - src={ - frameBlobUrl() ?? - convertFileSrc( - `${editorInstance.path}/screenshots/display.jpg`, - ) - } + src={frameBlobUrl() ?? screenshotSrc} /> From cf27e330d738375f17b9f0e037422b7a0b3d0795 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Wed, 1 Apr 2026 19:09:09 +0100 Subject: [PATCH 7/7] Remove deferred accurate crop preview logic --- apps/desktop/src/routes/editor/Editor.tsx | 24 ----------------------- 1 file changed, 24 deletions(-) diff --git a/apps/desktop/src/routes/editor/Editor.tsx b/apps/desktop/src/routes/editor/Editor.tsx index d318acd764..4090048ada 100644 --- a/apps/desktop/src/routes/editor/Editor.tsx +++ b/apps/desktop/src/routes/editor/Editor.tsx @@ -60,8 +60,6 @@ const MIN_PLAYER_CONTENT_HEIGHT = 320; const MIN_TIMELINE_HEIGHT = 240; const RESIZE_HANDLE_HEIGHT = 16; const MIN_PLAYER_HEIGHT = MIN_PLAYER_CONTENT_HEIGHT + RESIZE_HANDLE_HEIGHT; -const ACCURATE_CROP_PREVIEW_MAX_DURATION = 15 * 60; - const TIMELINE_RESIZE_GRIP_MARKS = [0, 1, 2] as const; function logCropProfile( @@ -949,28 +947,6 @@ function Dialogs() { ), }); - if ( - editorInstance.recordingDuration > - ACCURATE_CROP_PREVIEW_MAX_DURATION && - initialPreviewUrl - ) { - logCropProfile("accurate-frame-deferred", { - elapsedMs: Number( - (performance.now() - cropOpenedAt).toFixed(2), - ), - recordingDurationSec: Math.round( - editorInstance.recordingDuration, - ), - reason: "current-preview-available", - }); - scheduleAccurateFrame("deferred-long-recording", { - delayMs: 2000, - idleTimeoutMs: 500, - fallbackDelayMs: 0, - }); - return; - } - scheduleAccurateFrame("immediate", { idleTimeoutMs: 500, fallbackDelayMs: 16,