From d9abe8f8f966d3fe90c13a5b44890782a45fb3f5 Mon Sep 17 00:00:00 2001 From: Sportinger Date: Sun, 3 May 2026 17:40:30 +0200 Subject: [PATCH 1/3] Improve media board performance --- docs/Features/3D-Layers.md | 1 + docs/Features/Export.md | 4 +- docs/Features/GPU-Engine.md | 9 + docs/Features/Keyframes.md | 21 +- docs/Features/Motion-Design.md | 27 + docs/Features/Project-Persistence.md | 1 + docs/Features/README.md | 1 + docs/Features/Timeline.md | 13 +- src/App.css | 25 +- src/components/panels/MediaPanel.tsx | 159 ++- .../panels/properties/MotionShapeTab.tsx | 313 ++++++ .../panels/properties/TransformTab.tsx | 17 +- src/components/panels/properties/index.tsx | 24 +- src/components/preview/Preview.tsx | 20 + src/engine/WebGPUEngine.ts | 10 +- src/engine/export/ExportLayerBuilder.ts | 40 + src/engine/featureFlags.ts | 2 + src/engine/motion/MotionBuffers.ts | 74 ++ src/engine/motion/MotionPipeline.ts | 59 ++ src/engine/motion/MotionRenderer.ts | 108 ++ src/engine/motion/MotionTypes.ts | 51 + src/engine/motion/shaders/motionShapes.wgsl | 97 ++ src/engine/render/LayerCollector.ts | 15 + src/engine/render/NestedCompRenderer.ts | 23 +- src/engine/render/RenderDispatcher.ts | 41 +- .../layerBuilder/LayerBuilderService.ts | 189 +++- src/services/project/projectLoad.ts | 33 + src/services/project/projectSave.ts | 2 + .../project/types/composition.types.ts | 6 +- src/services/properties/PropertyRegistry.ts | 132 +++ src/services/properties/index.ts | 7 + .../properties/registerCoreProperties.ts | 973 ++++++++++++++++++ src/services/proxyGenerator.ts | 35 +- src/stores/engineStore.ts | 10 + .../mediaStore/helpers/proxyCompleteness.ts | 11 +- .../mediaStore/helpers/thumbnailHelpers.ts | 104 +- .../mediaStore/slices/fileManageSlice.ts | 2 +- src/stores/timeline/clipboardSlice.ts | 21 +- src/stores/timeline/helpers/idGenerator.ts | 7 + src/stores/timeline/index.ts | 3 + src/stores/timeline/keyframeSlice.ts | 22 + src/stores/timeline/motionClipSlice.ts | 214 ++++ src/stores/timeline/serializationUtils.ts | 55 + src/stores/timeline/types.ts | 20 +- src/types/index.ts | 13 +- src/types/motionDesign.ts | 324 ++++++ src/types/propertyRegistry.ts | 46 + src/utils/motionInterpolation.ts | 47 + tests/helpers/storeFactory.ts | 3 + .../stores/mediaStore/fileManageSlice.test.ts | 15 +- tests/unit/PreviewSourceMonitor.test.tsx | 1 + tests/unit/TransformTab.test.tsx | 20 + tests/unit/layerBuilderService.test.ts | 128 +++ tests/unit/mediaPanelSourceMonitor.test.tsx | 12 + tests/unit/motionDesignRendering.test.ts | 129 +++ tests/unit/propertyRegistry.test.ts | 139 +++ tests/unit/proxyCompleteness.test.ts | 52 + tests/unit/renderDispatcher.test.ts | 73 ++ 58 files changed, 3941 insertions(+), 62 deletions(-) create mode 100644 docs/Features/Motion-Design.md create mode 100644 src/components/panels/properties/MotionShapeTab.tsx create mode 100644 src/engine/motion/MotionBuffers.ts create mode 100644 src/engine/motion/MotionPipeline.ts create mode 100644 src/engine/motion/MotionRenderer.ts create mode 100644 src/engine/motion/MotionTypes.ts create mode 100644 src/engine/motion/shaders/motionShapes.wgsl create mode 100644 src/services/properties/PropertyRegistry.ts create mode 100644 src/services/properties/index.ts create mode 100644 src/services/properties/registerCoreProperties.ts create mode 100644 src/stores/timeline/motionClipSlice.ts create mode 100644 src/types/motionDesign.ts create mode 100644 src/types/propertyRegistry.ts create mode 100644 src/utils/motionInterpolation.ts create mode 100644 tests/unit/motionDesignRendering.test.ts create mode 100644 tests/unit/propertyRegistry.test.ts create mode 100644 tests/unit/proxyCompleteness.test.ts diff --git a/docs/Features/3D-Layers.md b/docs/Features/3D-Layers.md index 791eb3d2..2eb2a182 100644 --- a/docs/Features/3D-Layers.md +++ b/docs/Features/3D-Layers.md @@ -82,6 +82,7 @@ Create mesh clips from the Media Panel via `+ Add > Mesh` or the context menu: ### Preview Scene Gizmo - Selected scene objects expose a viewport transform gizmo for Move, Rotate, and Scale. +- The Preview scene-handle toggle hides both the React hit handles and the native WebGPU gizmo pass, so disabling it removes the visible axis gizmo from the preview. - Move, Rotate, and Scale visuals are drawn by the native WebGPU scene gizmo pass from the selected object's local transform basis. The React overlay only supplies hit targets and the mode toolbar. - Move and Scale use larger Unreal-style colored local-axis handles with dark outlines and a white center hub. Hovering an axis brightens and thickens that native gizmo handle. Dragging the center hub moves freely in the preview plane for Move and applies proportional uniform scale for Scale. - Rotate mode draws larger screen-space stable colored rings from the X, Y, and Z local rotation planes, so the ring orientation changes with object rotation and the scene view without the rings visually growing or shrinking. The invisible SVG hit targets are generated from the same projected 3D ring points, and hover/drag chooses the nearest ring instead of whichever SVG stroke is visually on top. diff --git a/docs/Features/Export.md b/docs/Features/Export.md index 7b1f5e15..639eb4df 100644 --- a/docs/Features/Export.md +++ b/docs/Features/Export.md @@ -43,7 +43,7 @@ FCPXML is exposed as a selectable export container for NLE interchange. `FrameExporter` is used for both the WebCodecs and HTMLVideo export buttons. -Canvas-backed sources such as text, solids, and Lottie are re-rendered for every export frame before capture, so the exported frame matches the current timeline time instead of reusing a stale first-frame texture. +Canvas-backed sources such as text, solids, and Lottie are re-rendered for every export frame before capture, so the exported frame matches the current timeline time instead of reusing a stale first-frame texture. Motion shape clips are built as `motion` layer sources and rendered by the WebGPU motion renderer at export frame time before compositing. ### Fast Mode @@ -272,7 +272,7 @@ The PNG frame export action reads the current composited frame from the GPU. 1. Prepare clips and runtimes for the selected export mode. 2. Seek all clips to each export time. 3. Build layers for that frame. -4. Render through the GPU engine. +4. Render procedural motion shape textures and nested compositions, then composite through the GPU engine. 5. Capture a `VideoFrame` from the export canvas when possible, otherwise fall back to pixel readback. 6. Encode and mux the file. diff --git a/docs/Features/GPU-Engine.md b/docs/Features/GPU-Engine.md index f9475363..d25950fa 100644 --- a/docs/Features/GPU-Engine.md +++ b/docs/Features/GPU-Engine.md @@ -55,6 +55,12 @@ The engine currently supports: - Images and canvases are copied into `rgba8unorm` GPU textures. - Cached image views are reused when possible. +### Motion Shapes + +- `motion-shape` clips render through `src/engine/motion/MotionRenderer.ts`. +- Rectangle and ellipse primitives are drawn with analytic WGSL SDFs into transparent `rgba8unorm` textures. +- The resulting texture view is composited through the normal `CompositorPipeline`, so masks, effects, blend mode, nested comps, preview targets, and export share the same downstream path. + ### Masks - Mask textures are uploaded per layer. @@ -155,6 +161,7 @@ Runtime flags are exposed on `window.__ENGINE_FLAGS__`. - `useLiveSlotTrigger` swaps slot-grid clicks from editor-open behavior to direct live triggering. - `useWarmSlotDecks` prepares reusable slot-owned live decks for faster layer adoption. - `use3DLayers` and `useGaussianSplat` are enabled in this branch. +- `useMotionDesignSystem` exists for the motion-design rollout; current rectangle/ellipse render plumbing is additive for `motion-shape` clips. --- @@ -192,6 +199,8 @@ Key implementation files: - `src/engine/render/RenderDispatcher.ts` - `src/engine/render/RenderLoop.ts` - `src/engine/render/htmlVideoPreviewFallback.ts` +- `src/engine/motion/MotionRenderer.ts` +- `src/engine/motion/shaders/motionShapes.wgsl` - `src/engine/pipeline/OutputPipeline.ts` - `src/engine/managers/ExportCanvasManager.ts` - `src/engine/texture/ScrubbingCache.ts` diff --git a/docs/Features/Keyframes.md b/docs/Features/Keyframes.md index ad5bdc8e..54e788d6 100644 --- a/docs/Features/Keyframes.md +++ b/docs/Features/Keyframes.md @@ -2,7 +2,7 @@ [<- Back to Index](./README.md) -The keyframe system animates clip properties over time using per-clip keyframe maps, curve editors, and Bezier handles. It supports transform properties, speed, numeric effect parameters, mask properties, and vector-animation state/input properties. +The keyframe system animates clip properties over time using per-clip keyframe maps, curve editors, and Bezier handles. It supports transform properties, speed, numeric effect parameters, mask properties, vector-animation state/input properties, and numeric motion-shape properties. --- @@ -82,6 +82,24 @@ lottieInput.{stateMachine}.{input} `lottieState.*` keyframes are discrete named states. They render as blue diamonds and stepped curves because a state change should hold until the next state keyframe, not ease between values. Boolean and numeric `lottieInput.*` properties use the normal stopwatch/keyframe workflow. +### Motion Shape Properties + +Motion shape clips use flat property paths from the property registry: + +```text +shape.size.w +shape.size.h +shape.cornerRadius +appearance.{appearanceId}.opacity +appearance.{appearanceId}.color.r +appearance.{appearanceId}.color.g +appearance.{appearanceId}.color.b +appearance.{appearanceId}.color.a +appearance.{appearanceId}.stroke.width +``` + +Numeric motion properties are interpolated before `MotionRenderer` draws the shape texture, so preview, nested compositions, and export evaluate the same frame state. Enum-like fields such as primitive and stroke alignment are currently static controls. + ### Visibility Rules - 2D clips hide `rotation.x`, `rotation.y`, `position.z`, and `scale.z` in the timeline UI. @@ -110,6 +128,7 @@ The diamond button writes a keyframe at the playhead. If a keyframe already exis - `scale.all` does not overwrite `scale.x`, `scale.y`, or `scale.z`; render, export, and scene-gizmo paths multiply it into the final visible scale only at evaluation time. - Camera stopwatch buttons are per camera value. FOV and mm both write `camera.fov`; Near, Far, Resolution X, and Resolution Y write their own camera properties. - Mask panel stopwatch buttons are available for Feather, Feather Quality, Position X/Y, and the whole Mask Path. +- Motion shape stopwatch buttons are available for size, corner radius, fill opacity, and stroke width in the Motion tab. ### Recording Mode diff --git a/docs/Features/Motion-Design.md b/docs/Features/Motion-Design.md new file mode 100644 index 00000000..6bad3e08 --- /dev/null +++ b/docs/Features/Motion-Design.md @@ -0,0 +1,27 @@ +[Back to Docs](./README.md) + +# Motion Design + +Status: shape MVP in progress. The data model, property registry, shape editing tab, GPU rectangle/ellipse renderer, persistence, nested composition path, and export layer path are wired. + +The motion design system follows `docs/plans/motion-design-system-plan.md`. It is native MasterSelects timeline content, not an embedded external editor. + +## Current Scope + +- `src/types/motionDesign.ts` defines versioned motion layer data for shape, null, adjustment, and group layers. +- `TimelineSourceType`, `TimelineClip`, `SerializableClip`, and project clip persistence accept `motion-shape`, `motion-null`, and `motion-adjustment`. +- Motion definitions are plain JSON and survive timeline/project serialization. +- `src/services/properties/PropertyRegistry.ts` describes transform, effect, color, mask, vector-animation, and motion properties without owning Zustand state. +- `src/stores/timeline/motionClipSlice.ts` can create rectangle/ellipse shape clips, null clips, adjustment clips, update motion definitions, and convert solid clips to motion rectangle clips. +- `src/components/panels/properties/MotionShapeTab.tsx` exposes primitive, size, corner radius, fill, and stroke controls for motion shape clips. +- `src/engine/motion/MotionRenderer.ts` renders rectangle and ellipse primitives into transparent `rgba8unorm` textures using analytic WGSL SDFs. +- `LayerBuilderService`, `NestedCompRenderer`, `RenderDispatcher`, and `ExportLayerBuilder` pass motion shape layers through the same compositor path as image/text/video textures. +- Numeric motion properties are evaluated through the keyframe store via the property registry before rendering. + +## Not Yet Implemented + +- Replicators are represented in the schema and registry, but no GPU instancing pipeline is wired. +- Texture fills, gradients, appearance blend modes, polygon/star rendering, viewport motion paths, and graph mode are not implemented yet. +- Adjustment layers remain blocked on the render graph work. + +The next implementation slice should add user-facing creation affordances, pinned motion property lanes, and the first replicator controls while keeping adjustment layers deferred until the render graph work is ready. diff --git a/docs/Features/Project-Persistence.md b/docs/Features/Project-Persistence.md index 1eaa84f7..b7b78753 100644 --- a/docs/Features/Project-Persistence.md +++ b/docs/Features/Project-Persistence.md @@ -342,6 +342,7 @@ interface ProjectFile { - Text clip properties - Solid clip color - Vector animation settings (loop, end behavior, fit, animation selection, background) +- Motion design definitions for shape/null/adjustment clips - Transcript and analysis data per clip - Scene description data diff --git a/docs/Features/README.md b/docs/Features/README.md index 42635bc4..8f18dd29 100644 --- a/docs/Features/README.md +++ b/docs/Features/README.md @@ -55,6 +55,7 @@ The docs in this folder were re-audited against the current codebase and now tra | [Professional Color Correction Plan](./Color-Correction-Professional-Plan.md) | Tactical roadmap for wheels, curves, LUTs, secondaries, float precision, scopes, compare, and presets | | [Masks](./Masks.md) | Overlay mask editing, whole-path keyframes, feathering, and stored modes | | [Text Clips](./Text-Clips.md) | Canvas-backed text rendering, typography controls, and timeline text items | +| [Motion Design](./Motion-Design.md) | Motion layer schema, property registry, rectangle/ellipse shape editing, GPU renderer, and persistence/export plumbing | | [3D Layers](./3D-Layers.md) | Shared-scene path, native Gaussian splats, cameras, and splat effectors | | [Vector Animation](./Vector-Animation.md) | Lottie import, runtime playback, bounce modes, state-machine keyframes, and export behavior | | [Audio](./Audio.md) | Playback sync, EQ, waveform extraction, audio clip behavior, and export | diff --git a/docs/Features/Timeline.md b/docs/Features/Timeline.md index aeaeaab5..c98ddc58 100644 --- a/docs/Features/Timeline.md +++ b/docs/Features/Timeline.md @@ -2,14 +2,14 @@ [<- Back to Index](./README.md) -The Timeline is the core editing interface for multi-track editing. It now covers video, audio, image, Lottie, text, solid, mesh, composition, camera, and splat-effector clips, with keyframe lanes, transitions, multicam grouping, pick-whip parenting, and slot-grid playback. +The Timeline is the core editing interface for multi-track editing. It now covers video, audio, image, Lottie, text, solid, motion shape, mesh, composition, camera, and splat-effector clips, with keyframe lanes, transitions, multicam grouping, pick-whip parenting, and slot-grid playback. --- ## Track Types ### Video Tracks -- Hold video, image, Lottie, text, solid, mesh, composition, camera, and splat-effector clips. +- Hold video, image, Lottie, text, solid, motion shape, mesh, composition, camera, and splat-effector clips. - Higher tracks render on top of lower tracks. - Expanded tracks can show keyframe property rows and curve editors. - Default layout starts with `Video 2` above `Video 1`. @@ -63,6 +63,12 @@ getTrackChildren() // Query child tracks ### Solid - Flat color clips used for mattes and backgrounds. +### Motion Shape +- Rectangle and ellipse shape clips are timeline clips with JSON motion definitions. +- The Motion tab exposes primitive, size, radius, fill, and stroke controls. +- Solid clips can be converted in the store to motion rectangle clips while preserving timeline identity, timing, transform, effects, and keyframes. +- Motion shape rendering uses WebGPU SDF textures, then the normal compositor stack. + ### Mesh - Primitive 3D meshes such as cube, sphere, plane, cylinder, torus, and cone. - Rendered as 3D clips with full transform and keyframe support. @@ -97,6 +103,7 @@ getTrackChildren() // Query child tracks ### Copy and Paste - Copying clips includes linked audio automatically when the video clip is selected. - Copy/paste preserves Lottie clip type and vector animation settings. +- Copy/paste preserves motion shape definitions. - Copying keyframes stores them relative to the earliest copied keyframe. - Pasting keyframes targets the selected clip when exactly one clip is selected; otherwise it falls back to the original clip from the clipboard data. @@ -132,6 +139,7 @@ getTrackChildren() // Query child tracks - Camera clips and native-render gaussian splats keep the camera-style property model visible. - Numeric effect parameters appear as `effect.{effectId}.{paramName}` lanes. - Lottie state changes appear as `lottieState.{stateMachine}` lanes; state-machine inputs appear as `lottieInput.{stateMachine}.{input}` lanes. +- Motion shape numeric lanes use registry paths such as `shape.size.w` and `appearance.{id}.stroke.width`. - Audio EQ lanes sort `volume` and the band parameters first. ### Curve Editor @@ -228,6 +236,7 @@ The timeline store in `src/stores/timeline/index.ts` combines 20 slices plus 2 u - `clipSlice` - `textClipSlice` - `solidClipSlice` +- `motionClipSlice` - `meshClipSlice` - `cameraClipSlice` - `splatEffectorClipSlice` diff --git a/src/App.css b/src/App.css index adef675f..420e6efc 100644 --- a/src/App.css +++ b/src/App.css @@ -9674,11 +9674,17 @@ input[type="checkbox"] { } .media-board-wrapper.board-interacting .media-board-node { - will-change: left, top, transform; + will-change: transform; } .media-board-wrapper.board-interacting .media-board-group { - will-change: left, top, width, height, transform; + will-change: transform; +} + +.media-board-wrapper.board-interacting .media-board-node, +.media-board-wrapper.board-interacting .media-board-group, +.media-board-wrapper.board-interacting .media-board-insert-gap { + transition: none; } .media-board-wrapper.board-interacting .media-board-node-thumb img { @@ -9696,6 +9702,7 @@ input[type="checkbox"] { .media-board-group { position: absolute; + contain: layout paint style; border: 1px solid rgba(255, 255, 255, 0.09); border-radius: 6px; background: rgba(255, 255, 255, 0.035); @@ -9788,6 +9795,7 @@ input[type="checkbox"] { .media-board-insert-gap { position: absolute; + contain: layout paint style; border: 1px dashed rgba(45, 140, 235, 0.68); border-radius: 6px; background: rgba(45, 140, 235, 0.12); @@ -9800,6 +9808,7 @@ input[type="checkbox"] { .media-board-node { position: absolute; display: block; + contain: layout paint style; overflow: hidden; border: 1px solid var(--border-color); border-top: 3px solid var(--border-color); @@ -9835,6 +9844,18 @@ input[type="checkbox"] { opacity: 0.55; } +.media-board-node.lod-compact { + box-shadow: 0 3px 8px rgba(0, 0, 0, 0.18); +} + +.media-board-node.lod-compact .media-board-node-timeline-drag { + display: none; +} + +.media-board-node.lod-thumbnail-paused .media-board-node-placeholder { + opacity: 0.54; +} + .media-board-node-thumb { position: relative; width: 100%; diff --git a/src/components/panels/MediaPanel.tsx b/src/components/panels/MediaPanel.tsx index 1f844831..302a9649 100644 --- a/src/components/panels/MediaPanel.tsx +++ b/src/components/panels/MediaPanel.tsx @@ -38,6 +38,18 @@ interface MediaBoardViewport { panY: number; } +interface MediaBoardViewportSize { + width: number; + height: number; +} + +interface MediaBoardVisibleRect { + left: number; + top: number; + right: number; + bottom: number; +} + interface MediaBoardGroupOffset { x: number; y: number; @@ -143,6 +155,9 @@ const MEDIA_BOARD_GRID_PARALLAX = 0.18; const MEDIA_BOARD_AUTOPAN_EDGE_PX = 72; const MEDIA_BOARD_AUTOPAN_MAX_SPEED = 620; const MEDIA_BOARD_TIMELINE_HANDOFF_DISTANCE_PX = 96; +const MEDIA_BOARD_RENDER_BUFFER_PX = 760; +const MEDIA_BOARD_COMPACT_LOD_ZOOM = 0.34; +const MEDIA_BOARD_THUMBNAIL_LOD_MIN_ZOOM = 0.42; const MEDIA_PANEL_VIEW_TRANSITION_MS = 500; interface MediaPanelTransitionBox { @@ -447,6 +462,45 @@ function getMediaBoardNodeSize(item: MediaBoardItem): { width: number; height: n }; } +function getMediaBoardVisibleRect( + viewport: MediaBoardViewport, + viewportSize: MediaBoardViewportSize, +): MediaBoardVisibleRect { + const zoom = Math.max(viewport.zoom, MEDIA_BOARD_PAN_ZOOM_MIN); + const buffer = MEDIA_BOARD_RENDER_BUFFER_PX; + + return { + left: (-viewport.panX - buffer) / zoom, + top: (-viewport.panY - buffer) / zoom, + right: (viewportSize.width - viewport.panX + buffer) / zoom, + bottom: (viewportSize.height - viewport.panY + buffer) / zoom, + }; +} + +function mediaBoardNodeIntersectsVisibleRect( + layout: MediaBoardNodeLayout, + visibleRect: MediaBoardVisibleRect, +): boolean { + return ( + layout.x < visibleRect.right + && layout.x + layout.width > visibleRect.left + && layout.y < visibleRect.bottom + && layout.y + layout.height > visibleRect.top + ); +} + +function mediaBoardGroupIntersectsVisibleRect( + group: MediaBoardGroupLayout, + visibleRect: MediaBoardVisibleRect, +): boolean { + return ( + group.x < visibleRect.right + && group.x + group.width > visibleRect.left + && group.y < visibleRect.bottom + && group.y + group.height > visibleRect.top + ); +} + function rectToTransitionBox(rect: DOMRect): MediaPanelTransitionBox { return { left: rect.left, @@ -561,6 +615,10 @@ export function MediaPanel() { const [mediaBoardViewport, setMediaBoardViewport] = useState(loadMediaBoardViewport); const [mediaBoardOrder, setMediaBoardOrder] = useState>(loadMediaBoardOrder); const [mediaBoardGroupOffsets, setMediaBoardGroupOffsets] = useState>(loadMediaBoardGroupOffsets); + const [mediaBoardCanvasSize, setMediaBoardCanvasSize] = useState(() => ({ + width: typeof window === 'undefined' ? 1280 : Math.max(1, window.innerWidth), + height: typeof window === 'undefined' ? 720 : Math.max(1, window.innerHeight), + })); const [mediaBoardMarquee, setMediaBoardMarquee] = useState(null); const [mediaBoardInsertionPreview, setMediaBoardInsertionPreview] = useState(null); const suppressMediaBoardContextMenuRef = useRef(false); @@ -596,6 +654,35 @@ export function MediaPanel() { localStorage.setItem(BOARD_GROUP_OFFSETS_STORAGE_KEY, JSON.stringify(mediaBoardGroupOffsets)); }, [mediaBoardGroupOffsets]); + useLayoutEffect(() => { + if (viewMode !== 'board') return; + + const canvas = boardCanvasRef.current; + if (!canvas) return; + + const updateCanvasSize = () => { + const rect = canvas.getBoundingClientRect(); + const width = Math.max(1, Math.round(rect.width)); + const height = Math.max(1, Math.round(rect.height)); + setMediaBoardCanvasSize((current) => ( + current.width === width && current.height === height + ? current + : { width, height } + )); + }; + + updateCanvasSize(); + + if (typeof ResizeObserver === 'undefined') { + window.addEventListener('resize', updateCanvasSize); + return () => window.removeEventListener('resize', updateCanvasSize); + } + + const resizeObserver = new ResizeObserver(updateCanvasSize); + resizeObserver.observe(canvas); + return () => resizeObserver.disconnect(); + }, [viewMode]); + useEffect(() => () => { if (boardInteractionFrameRef.current !== null) { window.cancelAnimationFrame(boardInteractionFrameRef.current); @@ -2140,6 +2227,7 @@ export function MediaPanel() { ]), [files, compositions, textItems, solidItems, meshItems, cameraItems, splatEffectorItems]); const mediaBoardItemIds = useMemo(() => new Set(mediaBoardItems.map((item) => item.id)), [mediaBoardItems]); + const selectedIdSet = useMemo(() => new Set(selectedIds), [selectedIds]); useEffect(() => { setMediaBoardOrder((current) => { @@ -2544,6 +2632,32 @@ export function MediaPanel() { return new Map(mediaBoardLayout.placements.map((placement) => [placement.item.id, placement])); }, [mediaBoardLayout.placements]); + const mediaBoardVisibleRect = useMemo(() => getMediaBoardVisibleRect( + mediaBoardViewport, + mediaBoardCanvasSize, + ), [mediaBoardCanvasSize, mediaBoardViewport]); + + const mediaBoardRenderLod = useMemo(() => ({ + compact: mediaBoardViewport.zoom <= MEDIA_BOARD_COMPACT_LOD_ZOOM, + showImages: mediaBoardViewport.zoom >= MEDIA_BOARD_THUMBNAIL_LOD_MIN_ZOOM, + }), [mediaBoardViewport.zoom]); + + const visibleMediaBoardGroups = useMemo(() => ( + mediaBoardLayout.groups.filter((group) => mediaBoardGroupIntersectsVisibleRect(group, mediaBoardVisibleRect)) + ), [mediaBoardLayout.groups, mediaBoardVisibleRect]); + + const visibleMediaBoardInsertGaps = useMemo(() => ( + mediaBoardLayout.insertGaps.filter((gap) => mediaBoardNodeIntersectsVisibleRect(gap.layout, mediaBoardVisibleRect)) + ), [mediaBoardLayout.insertGaps, mediaBoardVisibleRect]); + + const visibleMediaBoardPlacements = useMemo(() => ( + mediaBoardLayout.placements.filter((placement) => ( + placement.isDraggingPreview + || selectedIdSet.has(placement.item.id) + || mediaBoardNodeIntersectsVisibleRect(placement.layout, mediaBoardVisibleRect) + )) + ), [mediaBoardLayout.placements, mediaBoardVisibleRect, selectedIdSet]); + const screenToMediaBoard = useCallback((clientX: number, clientY: number) => { const rect = boardCanvasRef.current?.getBoundingClientRect(); if (!rect) return { x: 0, y: 0 }; @@ -3537,7 +3651,7 @@ export function MediaPanel() { const renderMediaBoardNode = (placement: MediaBoardNodePlacement) => { const { item, layout } = placement; - const isSelected = selectedIds.includes(item.id); + const isSelected = selectedIdSet.has(item.id); const isMediaFile = isImportedMediaFileItem(item); const mediaFile = isMediaFile ? item : null; const isComp = item.type === 'composition'; @@ -3563,6 +3677,8 @@ export function MediaPanel() { const boardCodecLabel = mediaFile?.type === 'gaussian-splat' ? getMediaFileContainerLabel(mediaFile) : getMediaFileCodecLabel(mediaFile); + const isCompactNode = mediaBoardRenderLod.compact; + const shouldRenderThumb = Boolean(thumbUrl && mediaBoardRenderLod.showImages); return (
{textItem.text}
- ) : thumbUrl ? ( + ) : shouldRenderThumb ? ( { void refreshFileUrls(mediaFile.id); } : undefined} /> ) : ( @@ -3608,8 +3735,8 @@ export function MediaPanel() { )} - {duration ? {formatDuration(duration)} : null} - {importProgress !== null ? {importProgress}% : null} + {!isCompactNode && duration ? {formatDuration(duration)} : null} + {!isCompactNode && importProgress !== null ? {importProgress}% : null} -
-
{item.name}
-
- {getMediaBoardTypeLabel(item)} - {resolutionLabel ? {resolutionLabel} : null} - {boardCodecLabel ? {boardCodecLabel} : null} + {!isCompactNode ? ( +
+
{item.name}
+
+ {getMediaBoardTypeLabel(item)} + {resolutionLabel ? {resolutionLabel} : null} + {boardCodecLabel ? {boardCodecLabel} : null} +
-
+ ) : null}
); }; @@ -3693,7 +3822,7 @@ export function MediaPanel() { transform: `translate(${mediaBoardViewport.panX}px, ${mediaBoardViewport.panY}px) scale(${mediaBoardViewport.zoom})`, }} > - {mediaBoardLayout.groups.map((group) => { + {visibleMediaBoardGroups.map((group) => { const folder = group.id ? folders.find((candidate) => candidate.id === group.id) : null; const isRenamingGroup = group.id !== null && renamingId === group.id; return ( @@ -3757,7 +3886,7 @@ export function MediaPanel() { ); })} - {mediaBoardLayout.insertGaps.map((gap) => ( + {visibleMediaBoardInsertGaps.map((gap) => (
))} - {mediaBoardLayout.placements.map(renderMediaBoardNode)} + {visibleMediaBoardPlacements.map(renderMediaBoardNode)} {mediaBoardMarquee && (() => { const left = Math.min(mediaBoardMarquee.startX, mediaBoardMarquee.currentX); const top = Math.min(mediaBoardMarquee.startY, mediaBoardMarquee.currentY); diff --git a/src/components/panels/properties/MotionShapeTab.tsx b/src/components/panels/properties/MotionShapeTab.tsx new file mode 100644 index 00000000..126b258a --- /dev/null +++ b/src/components/panels/properties/MotionShapeTab.tsx @@ -0,0 +1,313 @@ +import { useCallback } from 'react'; +import { useTimelineStore } from '../../../stores/timeline'; +import type { AnimatableProperty } from '../../../types'; +import type { + AppearanceItem, + ColorFillAppearance, + MotionColor, + ShapePrimitive, + StrokeAppearance, +} from '../../../types/motionDesign'; +import { createColorFillAppearance, createStrokeAppearance } from '../../../types/motionDesign'; +import { DraggableNumber, KeyframeToggle } from './shared'; + +interface MotionShapeTabProps { + clipId: string; +} + +function clamp01(value: number): number { + return Math.max(0, Math.min(1, value)); +} + +function componentToHex(value: number): string { + return Math.round(clamp01(value) * 255).toString(16).padStart(2, '0'); +} + +function colorToHex(color: MotionColor | undefined, fallback = '#ffffff'): string { + if (!color) return fallback; + return `#${componentToHex(color.r)}${componentToHex(color.g)}${componentToHex(color.b)}`; +} + +function hexToColor(hex: string, alpha: number): MotionColor { + const normalized = hex.replace('#', ''); + const value = normalized.length === 3 + ? normalized.split('').map((part) => part + part).join('') + : normalized.padEnd(6, '0').slice(0, 6); + + return { + r: parseInt(value.slice(0, 2), 16) / 255, + g: parseInt(value.slice(2, 4), 16) / 255, + b: parseInt(value.slice(4, 6), 16) / 255, + a: alpha, + }; +} + +function NumberRow({ + clipId, + label, + property, + value, + min, + max, + suffix, + defaultValue, +}: { + clipId: string; + label: string; + property: AnimatableProperty; + value: number; + min?: number; + max?: number; + suffix?: string; + defaultValue?: number; +}) { + const setPropertyValue = useTimelineStore(state => state.setPropertyValue); + + return ( +
+ + {label} + setPropertyValue(clipId, property, nextValue)} + min={min} + max={max} + suffix={suffix} + defaultValue={defaultValue} + /> +
+ ); +} + +function updateAppearanceItem( + items: AppearanceItem[], + itemId: string, + updater: (item: T) => T, +): AppearanceItem[] { + return items.map((item) => item.id === itemId ? updater(item as T) : item); +} + +export function MotionShapeTab({ clipId }: MotionShapeTabProps) { + const clip = useTimelineStore(state => state.clips.find(candidate => candidate.id === clipId)); + const updateMotionLayer = useTimelineStore(state => state.updateMotionLayer); + const setPropertyValue = useTimelineStore(state => state.setPropertyValue); + + const motion = clip?.motion; + const shape = motion?.shape; + const appearanceItems = motion?.appearance?.items ?? []; + const fill = appearanceItems.find((item): item is ColorFillAppearance => item.kind === 'color-fill'); + const stroke = appearanceItems.find((item): item is StrokeAppearance => item.kind === 'stroke'); + + const updatePrimitive = useCallback((primitive: ShapePrimitive) => { + updateMotionLayer(clipId, (current) => ({ + ...current, + shape: current.shape + ? { + ...current.shape, + primitive, + cornerRadius: primitive === 'rectangle' ? current.shape.cornerRadius ?? 0 : undefined, + } + : current.shape, + })); + }, [clipId, updateMotionLayer]); + + const updateFillColor = useCallback((hex: string) => { + if (!fill) { + updateMotionLayer(clipId, (current) => { + const currentAppearance = current.appearance ?? { version: 1 as const, items: [] }; + return { + ...current, + appearance: { + version: currentAppearance.version, + items: [ + ...currentAppearance.items, + createColorFillAppearance(hexToColor(hex, 1)), + ], + }, + }; + }); + return; + } + + const nextColor = hexToColor(hex, fill.color.a); + setPropertyValue(clipId, `appearance.${fill.id}.color.r` as AnimatableProperty, nextColor.r); + setPropertyValue(clipId, `appearance.${fill.id}.color.g` as AnimatableProperty, nextColor.g); + setPropertyValue(clipId, `appearance.${fill.id}.color.b` as AnimatableProperty, nextColor.b); + }, [clipId, fill, setPropertyValue, updateMotionLayer]); + + const updateStrokeColor = useCallback((hex: string) => { + if (!stroke) return; + const nextColor = hexToColor(hex, stroke.color.a); + setPropertyValue(clipId, `appearance.${stroke.id}.color.r` as AnimatableProperty, nextColor.r); + setPropertyValue(clipId, `appearance.${stroke.id}.color.g` as AnimatableProperty, nextColor.g); + setPropertyValue(clipId, `appearance.${stroke.id}.color.b` as AnimatableProperty, nextColor.b); + }, [clipId, setPropertyValue, stroke]); + + const setStrokeVisible = useCallback((visible: boolean) => { + updateMotionLayer(clipId, (current) => { + const appearance = current.appearance ?? { version: 1 as const, items: [] }; + const existingStroke = appearance.items.find((item): item is StrokeAppearance => item.kind === 'stroke'); + if (!existingStroke) { + return { + ...current, + appearance: { + ...appearance, + items: [ + ...appearance.items, + { ...createStrokeAppearance(), visible }, + ], + }, + }; + } + + return { + ...current, + appearance: { + ...appearance, + items: updateAppearanceItem( + appearance.items, + existingStroke.id, + (item) => ({ ...item, visible }), + ), + }, + }; + }); + }, [clipId, updateMotionLayer]); + + const updateStrokeAlignment = useCallback((alignment: StrokeAppearance['alignment']) => { + if (!stroke) return; + updateMotionLayer(clipId, (current) => current.appearance + ? { + ...current, + appearance: { + ...current.appearance, + items: updateAppearanceItem( + current.appearance.items, + stroke.id, + (item) => ({ ...item, alignment }), + ), + }, + } + : current); + }, [clipId, stroke, updateMotionLayer]); + + if (!clip || !motion || !shape) { + return

Select a motion shape clip

; + } + + return ( +
+
+
+ + +
+ + + + {shape.primitive === 'rectangle' && ( + + )} +
+ +
+
+ {fill && ( + + )} + + updateFillColor(event.target.value)} + /> + {fill && ( + setPropertyValue(clipId, `appearance.${fill.id}.opacity` as AnimatableProperty, clamp01(value / 100))} + min={0} + max={100} + suffix="%" + defaultValue={100} + /> + )} +
+
+ +
+
+ + setStrokeVisible(event.target.checked)} + /> + {stroke && ( + <> + updateStrokeColor(event.target.value)} + disabled={!stroke.visible} + /> + + + )} +
+ {stroke && ( + + )} +
+
+ ); +} diff --git a/src/components/panels/properties/TransformTab.tsx b/src/components/panels/properties/TransformTab.tsx index 19a6cbf5..b4ae4e26 100644 --- a/src/components/panels/properties/TransformTab.tsx +++ b/src/components/panels/properties/TransformTab.tsx @@ -57,6 +57,11 @@ const CAMERA_RESET_KEYFRAME_PROPERTIES: AnimatableProperty[] = [ 'scale.z', ]; +const CLIP_SPEED_MIN_PERCENT = -10000; +const CLIP_SPEED_MAX_PERCENT = 10000; +const CLIP_SPEED_MIN_MULTIPLIER = CLIP_SPEED_MIN_PERCENT / 100; +const CLIP_SPEED_MAX_MULTIPLIER = CLIP_SPEED_MAX_PERCENT / 100; + interface TransformTabProps { clipId: string; transform: { @@ -604,7 +609,13 @@ export function TransformTab({ Speed WIP @@ -614,8 +625,8 @@ export function TransformTab({ defaultValue={100} decimals={0} suffix="%" - min={-400} - max={400} + min={CLIP_SPEED_MIN_PERCENT} + max={CLIP_SPEED_MAX_PERCENT} sensitivity={1} onDragStart={handleBatchStart} onDragEnd={handleBatchEnd} diff --git a/src/components/panels/properties/index.tsx b/src/components/panels/properties/index.tsx index 3c17318e..b3e9e8bb 100644 --- a/src/components/panels/properties/index.tsx +++ b/src/components/panels/properties/index.tsx @@ -7,7 +7,7 @@ import { DEFAULT_TEXT_3D_PROPERTIES } from '../../../stores/timeline/constants'; import { TextTab } from '../TextTab'; // Tab type -type PropertiesTab = 'transform' | 'color' | 'effects' | 'masks' | 'transcript' | 'analysis' | 'text' | '3d-text' | 'math' | 'blendshapes' | 'gaussian-splat' | 'camera' | 'splat-effector' | 'lottie' | 'slot-clip'; +type PropertiesTab = 'transform' | 'color' | 'effects' | 'masks' | 'transcript' | 'analysis' | 'text' | '3d-text' | 'math' | 'motion' | 'blendshapes' | 'gaussian-splat' | 'camera' | 'splat-effector' | 'lottie' | 'slot-clip'; // Lazy load tab components for code splitting const TransformTab = lazy(() => import('./TransformTab').then(m => ({ default: m.TransformTab }))); @@ -23,6 +23,7 @@ const ThreeDTextTab = lazy(() => import('./ThreeDTextTab').then(m => ({ default: const LottieTab = lazy(() => import('./LottieTab').then(m => ({ default: m.LottieTab }))); const SlotClipTab = lazy(() => import('./SlotClipTab').then(m => ({ default: m.SlotClipTab }))); const MathSceneTab = lazy(() => import('./MathSceneTab').then(m => ({ default: m.MathSceneTab }))); +const MotionShapeTab = lazy(() => import('./MotionShapeTab').then(m => ({ default: m.MotionShapeTab }))); // Tab loading fallback function TabLoading() { @@ -70,6 +71,7 @@ export function PropertiesPanel() { // Check if it's a solid clip const isSolidClip = selectedClip?.source?.type === 'solid'; const isMathSceneClip = selectedClip?.source?.type === 'math-scene'; + const isMotionShapeClip = selectedClip?.source?.type === 'motion-shape'; const isLottieClip = selectedClip?.source?.type === 'lottie'; const selectedMeshType = selectedClip?.meshType ?? selectedClip?.source?.meshType; const is3DTextClip = selectedClip?.source?.type === 'model' && selectedMeshType === 'text3d'; @@ -146,6 +148,8 @@ export function PropertiesPanel() { setActiveTab('transform'); } else if (isGaussianSplat) { setActiveTab('transform'); + } else if (isMotionShapeClip) { + setActiveTab('motion'); } else if (isMathSceneClip) { setActiveTab('math'); } else if (isSolidClip) { @@ -164,6 +168,7 @@ export function PropertiesPanel() { activeTab === 'text' || activeTab === '3d-text' || (!isMathSceneClip && activeTab === 'math') || + (!isMotionShapeClip && activeTab === 'motion') || (!isGaussianAvatar && activeTab === 'blendshapes') || (!isGaussianSplat && activeTab === 'gaussian-splat') || (!isCameraClip && activeTab === 'camera') || @@ -174,7 +179,7 @@ export function PropertiesPanel() { setActiveTab('transform'); } } - }, [selectedClipId, isAudioClip, isTextClip, is3DTextClip, isMathSceneClip, isSolidClip, isLottieClip, isGaussianAvatar, isGaussianSplat, isCameraClip, isSplatEffectorClip, isSlotMode, lastClipId, activeTab]); + }, [selectedClipId, isAudioClip, isTextClip, is3DTextClip, isMathSceneClip, isMotionShapeClip, isSolidClip, isLottieClip, isGaussianAvatar, isGaussianSplat, isCameraClip, isSplatEffectorClip, isSlotMode, lastClipId, activeTab]); // Listen for external tab navigation requests (e.g. badge clicks in MediaPanel) useEffect(() => { @@ -292,6 +297,18 @@ export function PropertiesPanel() { Masks {selectedClip.masks && selectedClip.masks.length > 0 && {selectedClip.masks.length}} + ) : isMotionShapeClip ? ( + <> + + + + + + ) : isTextClip ? ( <> @@ -376,6 +393,9 @@ export function PropertiesPanel() { {activeTab === 'math' && isMathSceneClip && selectedClip.mathScene && ( )} + {activeTab === 'motion' && isMotionShapeClip && ( + + )} {activeTab === 'transform' && !isAudioClip && } {activeTab === 'color' && !isAudioClip && !isCameraClip && !isSplatEffectorClip && } {activeTab === 'blendshapes' && isGaussianAvatar && } diff --git a/src/components/preview/Preview.tsx b/src/components/preview/Preview.tsx index 7742abf4..f034501b 100644 --- a/src/components/preview/Preview.tsx +++ b/src/components/preview/Preview.tsx @@ -457,6 +457,7 @@ export function Preview({ panelId, source, showTransparencyGrid }: PreviewProps) const sceneNavNoKeyframes = useEngineStore(selectSceneNavNoKeyframes); const previewCameraOverride = useEngineStore((s) => s.previewCameraOverride); const setPreviewCameraOverride = useEngineStore((s) => s.setPreviewCameraOverride); + const setSceneGizmoVisible = useEngineStore((s) => s.setSceneGizmoVisible); const setSceneGizmoClipIdOverride = useEngineStore((s) => s.setSceneGizmoClipIdOverride); const activeSplatLoadProgress = useEngineStore(selectActiveGaussianSplatLoadProgress); const setSceneNavFpsMoveSpeed = useEngineStore((s) => s.setSceneNavFpsMoveSpeed); @@ -760,6 +761,25 @@ export function Preview({ panelId, source, showTransparencyGrid }: PreviewProps) const editCameraViewTransitionRef = useRef(false); const editCameraModeActiveRef = useRef(false); + useEffect(() => { + return () => { + setSceneGizmoVisible(true); + }; + }, [setSceneGizmoVisible]); + + useEffect(() => { + setSceneGizmoVisible(sceneObjectOverlayEnabled && isEditableSource && !sourceMonitorActive); + if (isEngineReady) { + engine.requestRender(); + } + }, [ + isEditableSource, + isEngineReady, + sceneObjectOverlayEnabled, + setSceneGizmoVisible, + sourceMonitorActive, + ]); + useEffect(() => { if (!isEditableSource) { setEditMode(false); diff --git a/src/engine/WebGPUEngine.ts b/src/engine/WebGPUEngine.ts index 72e78752..9cd46c6d 100644 --- a/src/engine/WebGPUEngine.ts +++ b/src/engine/WebGPUEngine.ts @@ -32,6 +32,7 @@ import { LayerCollector } from './render/LayerCollector'; import { Compositor } from './render/Compositor'; import { NestedCompRenderer } from './render/NestedCompRenderer'; import { RenderDispatcher, type RenderDeps, type RenderDispatcherDebugSnapshot } from './render/RenderDispatcher'; +import { MotionRenderer } from './motion/MotionRenderer'; export class WebGPUEngine { // Core context @@ -46,6 +47,7 @@ export class WebGPUEngine { private compositor: Compositor | null = null; private nestedCompRenderer: NestedCompRenderer | null = null; private renderDispatcher: RenderDispatcher | null = null; + private motionRenderer: MotionRenderer | null = null; // Existing managers (unchanged) private textureManager: TextureManager | null = null; @@ -153,6 +155,7 @@ export class WebGPUEngine { this.outputWindowManager = new OutputWindowManager(width, height); this.layerCollector = new LayerCollector(); + this.motionRenderer = new MotionRenderer(device); this.compositor = new Compositor( this.compositorPipeline, @@ -168,7 +171,8 @@ export class WebGPUEngine { this.textureManager, this.maskTextureManager, this.cacheManager.getScrubbingCache(), - this.colorPipeline + this.colorPipeline, + this.motionRenderer ); this.renderLoop = new RenderLoop(this.performanceStats, { @@ -195,6 +199,7 @@ export class WebGPUEngine { layerCollector: { get: () => this.layerCollector }, compositor: { get: () => this.compositor }, nestedCompRenderer: { get: () => this.nestedCompRenderer }, + motionRenderer: { get: () => this.motionRenderer }, cacheManager: { get: () => this.cacheManager }, exportCanvasManager: { get: () => this.exportCanvasManager }, performanceStats: { get: () => this.performanceStats }, @@ -219,6 +224,8 @@ export class WebGPUEngine { // Clear managers this.textureManager = null; this.maskTextureManager = null; + this.motionRenderer?.destroy(); + this.motionRenderer = null; this.compositorPipeline = null; this.effectsPipeline = null; this.colorPipeline = null; @@ -1033,6 +1040,7 @@ export class WebGPUEngine { this.outputWindowManager?.destroy(); this.renderTargetManager?.destroy(); this.nestedCompRenderer?.destroy(); + this.motionRenderer?.destroy(); this.textureManager?.destroy(); this.maskTextureManager?.destroy(); this.cacheManager.destroy(); diff --git a/src/engine/export/ExportLayerBuilder.ts b/src/engine/export/ExportLayerBuilder.ts index 91aa8d8b..6cbede1e 100644 --- a/src/engine/export/ExportLayerBuilder.ts +++ b/src/engine/export/ExportLayerBuilder.ts @@ -12,6 +12,7 @@ import { useTimelineStore } from '../../stores/timeline'; import { ParallelDecodeManager } from '../ParallelDecodeManager'; import { getInterpolatedClipTransform } from '../../utils/keyframeInterpolation'; import { getEffectiveScale } from '../../utils/transformScale'; +import { getInterpolatedMotionLayer } from '../../utils/motionInterpolation'; import { DEFAULT_TEXT_3D_PROPERTIES, DEFAULT_TRANSFORM } from '../../stores/timeline/constants'; import { DEFAULT_GAUSSIAN_SPLAT_SETTINGS, type GaussianSplatSettings } from '../gaussian/types'; import { lottieRuntimeManager } from '../../services/vectorAnimation/LottieRuntimeManager'; @@ -128,6 +129,18 @@ function buildGaussianSplatSource(clip: TimelineClip, clipLocalTime: number): La }; } +function buildMotionSource(clip: TimelineClip, clipLocalTime: number): Layer['source'] | null { + if (clip.source?.type !== 'motion-shape' || clip.motion?.kind !== 'shape') { + return null; + } + + const keyframes = useTimelineStore.getState().clipKeyframes.get(clip.id) ?? []; + return { + type: 'motion', + motion: getInterpolatedMotionLayer(clip, keyframes, clipLocalTime) ?? clip.motion, + }; +} + function getExportVideoElement( clip: TimelineClip, clipStates: Map @@ -234,6 +247,16 @@ export function buildLayersAtTime( source: { type: 'image', imageElement: clip.source.imageElement }, }); } + // Handle motion shape clips + else if (clip.source?.type === 'motion-shape') { + const source = buildMotionSource(clip, clipLocalTime); + if (source) { + layers.push({ + ...baseLayerProps, + source, + }); + } + } // Handle 3D model clips else if (clip.source?.type === 'model') { const modelSourceTime = getClipSourceWindowTime(clip, clipLocalTime, ctx); @@ -600,6 +623,23 @@ function buildNestedLayerForExport( } as Layer; } + if (nestedClip.source?.type === 'motion-shape') { + const source = buildMotionSource(nestedClip, nestedClipLocalTime); + return source + ? { + ...baseLayer, + source, + } as Layer + : null; + } + + if ( + nestedClip.source?.type === 'motion-null' || + nestedClip.source?.type === 'motion-adjustment' + ) { + return null; + } + if (nestedClip.source?.type === 'model') { const nestedSourceTime = nestedClip.reversed ? nestedClip.outPoint - nestedClipLocalTime diff --git a/src/engine/featureFlags.ts b/src/engine/featureFlags.ts index 5d5282ef..fcd56687 100644 --- a/src/engine/featureFlags.ts +++ b/src/engine/featureFlags.ts @@ -11,6 +11,8 @@ export const flags = { useWarmSlotDecks: false, // Prepare reusable slot-owned live decks for low-latency triggering use3DLayers: true, // Shared 3D scene support useGaussianSplat: true, // Gaussian Splat avatar rendering (old WebGL path) + useMotionDesignSystem: false, // Motion shape/null/adjustment system foundation + useMotionReplicators: false, // GPU-instanced motion replicators }; // Expose for runtime toggling from devtools diff --git a/src/engine/motion/MotionBuffers.ts b/src/engine/motion/MotionBuffers.ts new file mode 100644 index 00000000..100d6b6f --- /dev/null +++ b/src/engine/motion/MotionBuffers.ts @@ -0,0 +1,74 @@ +import type { + ColorFillAppearance, + MotionColor, + MotionLayerDefinition, + ShapePrimitive, + StrokeAppearance, +} from '../../types/motionDesign'; +import type { MotionRenderSize } from './MotionTypes'; + +const TRANSPARENT: MotionColor = { r: 0, g: 0, b: 0, a: 0 }; + +function primitiveCode(primitive: ShapePrimitive | undefined): number { + return primitive === 'ellipse' ? 1 : 0; +} + +function strokeAlignmentCode(alignment: StrokeAppearance['alignment'] | undefined): number { + if (alignment === 'inside') return 1; + if (alignment === 'outside') return 2; + return 0; +} + +function findFill(motion: MotionLayerDefinition): ColorFillAppearance | undefined { + return motion.appearance?.items.find((item): item is ColorFillAppearance => ( + item.kind === 'color-fill' && + item.visible !== false && + item.opacity > 0 + )); +} + +function findStroke(motion: MotionLayerDefinition): StrokeAppearance | undefined { + return motion.appearance?.items.find((item): item is StrokeAppearance => ( + item.kind === 'stroke' && + item.visible !== false && + item.opacity > 0 && + item.width > 0 + )); +} + +function writeColor(target: Float32Array, offset: number, color: MotionColor): void { + target[offset] = color.r; + target[offset + 1] = color.g; + target[offset + 2] = color.b; + target[offset + 3] = color.a; +} + +export function createMotionUniformArray( + motion: MotionLayerDefinition, + size: MotionRenderSize, +): Float32Array { + const shape = motion.shape; + const fill = findFill(motion); + const stroke = findStroke(motion); + const data = new Float32Array(20); + + data[0] = Math.max(1, shape?.size.w ?? 1); + data[1] = Math.max(1, shape?.size.h ?? 1); + data[2] = size.width; + data[3] = size.height; + + data[4] = Math.max(0, shape?.cornerRadius ?? 0); + data[5] = primitiveCode(shape?.primitive); + data[6] = fill?.opacity ?? 0; + data[7] = stroke?.opacity ?? 0; + + writeColor(data, 8, fill?.color ?? TRANSPARENT); + writeColor(data, 12, stroke?.color ?? TRANSPARENT); + + data[16] = stroke?.width ?? 0; + data[17] = stroke ? 1 : 0; + data[18] = strokeAlignmentCode(stroke?.alignment); + data[19] = 0; + + return data; +} diff --git a/src/engine/motion/MotionPipeline.ts b/src/engine/motion/MotionPipeline.ts new file mode 100644 index 00000000..7f05e3fc --- /dev/null +++ b/src/engine/motion/MotionPipeline.ts @@ -0,0 +1,59 @@ +import shaderSource from './shaders/motionShapes.wgsl?raw'; +import { MOTION_RENDER_TEXTURE_FORMAT } from './MotionTypes'; + +export class MotionPipeline { + private device: GPUDevice; + private bindGroupLayout: GPUBindGroupLayout | null = null; + private pipeline: GPURenderPipeline | null = null; + + constructor(device: GPUDevice) { + this.device = device; + } + + getBindGroupLayout(): GPUBindGroupLayout { + if (!this.bindGroupLayout) { + this.bindGroupLayout = this.device.createBindGroupLayout({ + label: 'motion-shape-bind-group-layout', + entries: [{ + binding: 0, + visibility: GPUShaderStage.FRAGMENT, + buffer: { type: 'uniform' }, + }], + }); + } + + return this.bindGroupLayout; + } + + getPipeline(): GPURenderPipeline { + if (!this.pipeline) { + const module = this.device.createShaderModule({ + label: 'motion-shape-shader', + code: shaderSource, + }); + const layout = this.device.createPipelineLayout({ + label: 'motion-shape-pipeline-layout', + bindGroupLayouts: [this.getBindGroupLayout()], + }); + + this.pipeline = this.device.createRenderPipeline({ + label: 'motion-shape-pipeline', + layout, + vertex: { + module, + entryPoint: 'vertexMain', + }, + fragment: { + module, + entryPoint: 'fragmentMain', + targets: [{ format: MOTION_RENDER_TEXTURE_FORMAT }], + }, + primitive: { + topology: 'triangle-list', + }, + }); + } + + return this.pipeline; + } +} diff --git a/src/engine/motion/MotionRenderer.ts b/src/engine/motion/MotionRenderer.ts new file mode 100644 index 00000000..3966dc88 --- /dev/null +++ b/src/engine/motion/MotionRenderer.ts @@ -0,0 +1,108 @@ +import type { Layer } from '../core/types'; +import type { MotionLayerDefinition } from '../../types/motionDesign'; +import { createMotionUniformArray } from './MotionBuffers'; +import { + getMotionRenderSize, + MOTION_RENDER_TEXTURE_FORMAT, + type MotionClipGpuCache, + type MotionRenderResult, +} from './MotionTypes'; +import { MotionPipeline } from './MotionPipeline'; + +function isRenderableMotionShape(motion: MotionLayerDefinition | undefined): motion is MotionLayerDefinition { + const primitive = motion?.shape?.primitive; + return motion?.kind === 'shape' && (primitive === 'rectangle' || primitive === 'ellipse'); +} + +export class MotionRenderer { + private device: GPUDevice; + private pipeline: MotionPipeline; + private caches = new Map(); + + constructor(device: GPUDevice) { + this.device = device; + this.pipeline = new MotionPipeline(device); + } + + renderLayer(layer: Layer, commandEncoder: GPUCommandEncoder): MotionRenderResult | null { + const motion = layer.source?.motion; + if (!isRenderableMotionShape(motion)) { + return null; + } + + const size = getMotionRenderSize(motion); + const cache = this.getOrCreateCache(layer, size.width, size.height); + const uniforms = createMotionUniformArray(motion, size); + this.device.queue.writeBuffer(cache.uniformBuffer, 0, uniforms as GPUAllowSharedBufferSource); + + const pass = commandEncoder.beginRenderPass({ + label: 'motion-shape-render-pass', + colorAttachments: [{ + view: cache.view, + clearValue: { r: 0, g: 0, b: 0, a: 0 }, + loadOp: 'clear', + storeOp: 'store', + }], + }); + pass.setPipeline(this.pipeline.getPipeline()); + pass.setBindGroup(0, cache.bindGroup); + pass.draw(6); + pass.end(); + + return { + ...size, + textureView: cache.view, + }; + } + + destroy(): void { + for (const cache of this.caches.values()) { + cache.texture.destroy(); + cache.uniformBuffer.destroy(); + } + this.caches.clear(); + } + + private getCacheKey(layer: Layer): string { + return layer.sourceClipId ? `${layer.id}:${layer.sourceClipId}` : layer.id; + } + + private getOrCreateCache(layer: Layer, width: number, height: number): MotionClipGpuCache { + const key = this.getCacheKey(layer); + const existing = this.caches.get(key); + if (existing && existing.width === width && existing.height === height) { + return existing; + } + + if (existing) { + existing.texture.destroy(); + existing.uniformBuffer.destroy(); + this.caches.delete(key); + } + + const texture = this.device.createTexture({ + label: `motion-shape-texture-${key}`, + size: { width, height }, + format: MOTION_RENDER_TEXTURE_FORMAT, + usage: GPUTextureUsage.RENDER_ATTACHMENT | GPUTextureUsage.TEXTURE_BINDING | GPUTextureUsage.COPY_SRC, + }); + const view = texture.createView(); + const uniformBuffer = this.device.createBuffer({ + label: `motion-shape-uniforms-${key}`, + size: 20 * 4, + usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, + }); + const bindGroup = this.device.createBindGroup({ + label: `motion-shape-bind-group-${key}`, + layout: this.pipeline.getBindGroupLayout(), + entries: [{ + binding: 0, + resource: { buffer: uniformBuffer }, + }], + }); + + const cache = { texture, view, uniformBuffer, bindGroup, width, height }; + this.caches.set(key, cache); + return cache; + } +} diff --git a/src/engine/motion/MotionTypes.ts b/src/engine/motion/MotionTypes.ts new file mode 100644 index 00000000..475f1889 --- /dev/null +++ b/src/engine/motion/MotionTypes.ts @@ -0,0 +1,51 @@ +import type { MotionLayerDefinition, StrokeAppearance } from '../../types/motionDesign'; + +export const MOTION_RENDER_TEXTURE_FORMAT: GPUTextureFormat = 'rgba8unorm'; + +export interface MotionRenderSize { + width: number; + height: number; + strokePadding: number; +} + +export interface MotionRenderResult extends MotionRenderSize { + textureView: GPUTextureView; +} + +export interface MotionClipGpuCache { + texture: GPUTexture; + view: GPUTextureView; + uniformBuffer: GPUBuffer; + bindGroup: GPUBindGroup; + width: number; + height: number; +} + +function getVisibleStroke(motion: MotionLayerDefinition): StrokeAppearance | undefined { + return motion.appearance?.items.find((item): item is StrokeAppearance => ( + item.kind === 'stroke' && + item.visible !== false && + item.opacity > 0 && + item.width > 0 + )); +} + +function getStrokePadding(stroke: StrokeAppearance | undefined): number { + if (!stroke) return 0; + if (stroke.alignment === 'inside') return 0; + if (stroke.alignment === 'center') return Math.ceil(stroke.width / 2); + return Math.ceil(stroke.width); +} + +export function getMotionRenderSize(motion: MotionLayerDefinition | undefined): MotionRenderSize { + const shape = motion?.shape; + const width = Math.max(1, Math.ceil(shape?.size.w ?? 1)); + const height = Math.max(1, Math.ceil(shape?.size.h ?? 1)); + const strokePadding = getStrokePadding(motion ? getVisibleStroke(motion) : undefined); + + return { + width: width + strokePadding * 2, + height: height + strokePadding * 2, + strokePadding, + }; +} diff --git a/src/engine/motion/shaders/motionShapes.wgsl b/src/engine/motion/shaders/motionShapes.wgsl new file mode 100644 index 00000000..7c86930d --- /dev/null +++ b/src/engine/motion/shaders/motionShapes.wgsl @@ -0,0 +1,97 @@ +struct VertexOutput { + @builtin(position) position: vec4f, + @location(0) uv: vec2f, +}; + +@vertex +fn vertexMain(@builtin(vertex_index) vertexIndex: u32) -> VertexOutput { + var positions = array( + vec2f(-1.0, -1.0), + vec2f(1.0, -1.0), + vec2f(-1.0, 1.0), + vec2f(-1.0, 1.0), + vec2f(1.0, -1.0), + vec2f(1.0, 1.0) + ); + + var uvs = array( + vec2f(0.0, 1.0), + vec2f(1.0, 1.0), + vec2f(0.0, 0.0), + vec2f(0.0, 0.0), + vec2f(1.0, 1.0), + vec2f(1.0, 0.0) + ); + + var output: VertexOutput; + output.position = vec4f(positions[vertexIndex], 0.0, 1.0); + output.uv = uvs[vertexIndex]; + return output; +} + +struct MotionUniforms { + // shape width/height, output texture width/height + data0: vec4f, + // corner radius, shape type, fill opacity, stroke opacity + data1: vec4f, + fillColor: vec4f, + strokeColor: vec4f, + // stroke width, stroke visible, stroke alignment, unused + data2: vec4f, +}; + +@group(0) @binding(0) var motion: MotionUniforms; + +fn sdRoundBox(point: vec2f, halfSize: vec2f, radius: f32) -> f32 { + let clampedRadius = min(radius, min(halfSize.x, halfSize.y)); + let q = abs(point) - halfSize + vec2f(clampedRadius); + return length(max(q, vec2f(0.0))) + min(max(q.x, q.y), 0.0) - clampedRadius; +} + +fn sdEllipse(point: vec2f, radius: vec2f) -> f32 { + let safeRadius = max(radius, vec2f(0.001)); + let scaled = point / safeRadius; + return (length(scaled) - 1.0) * min(safeRadius.x, safeRadius.y); +} + +fn over(top: vec4f, bottom: vec4f) -> vec4f { + let alpha = top.a + bottom.a * (1.0 - top.a); + let rgb = (top.rgb * top.a + bottom.rgb * bottom.a * (1.0 - top.a)) / max(alpha, 0.0001); + return vec4f(rgb, alpha); +} + +@fragment +fn fragmentMain(input: VertexOutput) -> @location(0) vec4f { + let shapeSize = max(motion.data0.xy, vec2f(1.0)); + let outputSize = max(motion.data0.zw, vec2f(1.0)); + let cornerRadius = motion.data1.x; + let shapeType = motion.data1.y; + let fillOpacity = motion.data1.z; + let strokeOpacity = motion.data1.w; + let strokeWidth = max(0.0, motion.data2.x); + let strokeVisible = motion.data2.y; + let strokeAlignment = motion.data2.z; + + let point = (input.uv - vec2f(0.5)) * outputSize; + let halfSize = shapeSize * 0.5; + let distance = select( + sdRoundBox(point, halfSize, cornerRadius), + sdEllipse(point, halfSize), + shapeType > 0.5 + ); + let aa = max(fwidth(distance), 1.0); + + let fillCoverage = 1.0 - smoothstep(-aa, aa, distance); + let fillAlpha = fillCoverage * motion.fillColor.a * fillOpacity; + let fill = vec4f(motion.fillColor.rgb, fillAlpha); + + let centerStroke = 1.0 - smoothstep(strokeWidth * 0.5 - aa, strokeWidth * 0.5 + aa, abs(distance)); + let insideBand = fillCoverage * smoothstep(-strokeWidth - aa, -strokeWidth + aa, distance); + let outsideBand = smoothstep(-aa, aa, distance) * (1.0 - smoothstep(strokeWidth - aa, strokeWidth + aa, distance)); + let alignedStroke = select(centerStroke, insideBand, strokeAlignment > 0.5 && strokeAlignment < 1.5); + let strokeCoverage = select(alignedStroke, outsideBand, strokeAlignment >= 1.5); + let strokeAlpha = strokeCoverage * motion.strokeColor.a * strokeOpacity * strokeVisible; + let stroke = vec4f(motion.strokeColor.rgb, strokeAlpha); + + return over(stroke, fill); +} diff --git a/src/engine/render/LayerCollector.ts b/src/engine/render/LayerCollector.ts index 76c3d345..68a3daf4 100644 --- a/src/engine/render/LayerCollector.ts +++ b/src/engine/render/LayerCollector.ts @@ -14,12 +14,15 @@ import { wcPipelineMonitor } from '../../services/wcPipelineMonitor'; import { scrubSettleState } from '../../services/scrubSettleState'; import { useTimelineStore } from '../../stores/timeline'; import { getCopiedHtmlVideoPreviewFrame } from './htmlVideoPreviewFallback'; +import type { MotionRenderer } from '../motion/MotionRenderer'; +import { getMotionRenderSize } from '../motion/MotionTypes'; const log = Logger.create('LayerCollector'); const ENABLE_VISUAL_HTML_VIDEO_FALLBACK = false; export interface LayerCollectorDeps { textureManager: TextureManager; + motionRenderer?: MotionRenderer | null; scrubbingCache: ScrubbingCache | null; getLastVideoTime: (key: string) => number | undefined; setLastVideoTime: (key: string, time: number) => void; @@ -271,6 +274,18 @@ export class LayerCollector { return null; } + if (sourceType === 'motion') { + const size = getMotionRenderSize(source.motion); + return { + layer, + isVideo: false, + externalTexture: null, + textureView: null, + sourceWidth: size.width, + sourceHeight: size.height, + }; + } + // Video sources - check decoders in priority order if (sourceType === 'video') { // 1. Try Native Helper decoder (turbo mode) - most efficient diff --git a/src/engine/render/NestedCompRenderer.ts b/src/engine/render/NestedCompRenderer.ts index 21807744..0e8c116e 100644 --- a/src/engine/render/NestedCompRenderer.ts +++ b/src/engine/render/NestedCompRenderer.ts @@ -24,6 +24,8 @@ import { collectActiveSceneSplatEffectors } from '../scene/SceneEffectorUtils'; import { collectScene3DLayers } from '../scene/SceneLayerCollector'; import { resolveRenderableSharedSceneCamera } from '../scene/SceneCameraUtils'; import { getNativeSceneRenderer } from '../native3d/NativeSceneRenderer'; +import type { MotionRenderer } from '../motion/MotionRenderer'; +import { getMotionRenderSize } from '../motion/MotionTypes'; const log = Logger.create('NestedCompRenderer'); const ENABLE_VISUAL_HTML_VIDEO_FALLBACK = false; @@ -53,6 +55,7 @@ export class NestedCompRenderer { private textureManager: TextureManager; private maskTextureManager: MaskTextureManager; private scrubbingCache: ScrubbingCache | null; + private motionRenderer: MotionRenderer | null; private nestedCompTextures: Map = new Map(); // Texture pool for ping-pong buffers, keyed by "widthxheight" @@ -222,7 +225,8 @@ export class NestedCompRenderer { textureManager: TextureManager, maskTextureManager: MaskTextureManager, scrubbingCache: ScrubbingCache | null = null, - colorPipeline: ColorPipeline | null = null + colorPipeline: ColorPipeline | null = null, + motionRenderer: MotionRenderer | null = null ) { this.device = device; this.compositorPipeline = compositorPipeline; @@ -231,6 +235,7 @@ export class NestedCompRenderer { this.maskTextureManager = maskTextureManager; this.scrubbingCache = scrubbingCache; this.colorPipeline = colorPipeline; + this.motionRenderer = motionRenderer; } // Acquire a ping-pong texture pair from pool or create new @@ -703,6 +708,22 @@ export class NestedCompRenderer { continue; } + if (layer.source.type === 'motion') { + const rendered = commandEncoder && this.motionRenderer + ? this.motionRenderer.renderLayer(layer, commandEncoder) + : null; + const size = rendered ?? getMotionRenderSize(layer.source.motion); + result.push({ + layer, + isVideo: false, + externalTexture: null, + textureView: rendered?.textureView ?? null, + sourceWidth: size.width, + sourceHeight: size.height, + }); + continue; + } + // Shared-scene native 3D layers do not contribute a 2D source texture of their own. if (layer.source.type === 'model' || layer.source.type === 'gaussian-splat') { result.push({ diff --git a/src/engine/render/RenderDispatcher.ts b/src/engine/render/RenderDispatcher.ts index 0d23bc1c..9843349e 100644 --- a/src/engine/render/RenderDispatcher.ts +++ b/src/engine/render/RenderDispatcher.ts @@ -50,6 +50,8 @@ import { buildSharedSplatRuntimeRequest, resolveSharedSplatSceneKey, } from '../scene/runtime/SharedSplatRuntimeUtils'; +import type { MotionRenderer } from '../motion/MotionRenderer'; +import { getMotionRenderSize } from '../motion/MotionTypes'; const log = Logger.create('RenderDispatcher'); const GAUSSIAN_PLAYBACK_SORT_FREQUENCY = 6; @@ -183,6 +185,7 @@ export interface RenderDeps { layerCollector: LayerCollector | null; compositor: Compositor | null; nestedCompRenderer: NestedCompRenderer | null; + motionRenderer: MotionRenderer | null; cacheManager: CacheManager; exportCanvasManager: ExportCanvasManager; performanceStats: PerformanceStats; @@ -852,6 +855,7 @@ export class RenderDispatcher { const layerData = d.layerCollector.collect(layers, { textureManager: d.textureManager!, scrubbingCache: d.cacheManager.getScrubbingCache(), + motionRenderer: d.motionRenderer, getLastVideoTime: (key) => d.cacheManager.getLastVideoTime(key), setLastVideoTime: (key, time) => d.cacheManager.setLastVideoTime(key, time), isExporting: d.exportCanvasManager.getIsExporting(), @@ -933,9 +937,19 @@ export class RenderDispatcher { // Pre-render nested compositions (batched with main composite) let hasNestedComps = false; + let hasMotionLayers = false; const preRenderEncoder = device.createCommandEncoder(); for (const data of layerData) { + if (data.layer.source?.type === 'motion') { + hasMotionLayers = true; + const rendered = d.motionRenderer?.renderLayer(data.layer, preRenderEncoder); + const size = rendered ?? getMotionRenderSize(data.layer.source.motion); + data.textureView = rendered?.textureView ?? null; + data.sourceWidth = size.width; + data.sourceHeight = size.height; + } + if (data.layer.source?.nestedComposition) { hasNestedComps = true; const nc = data.layer.source.nestedComposition; @@ -955,7 +969,7 @@ export class RenderDispatcher { if (view) data.textureView = view; } } - if (hasNestedComps) { + if (hasNestedComps || hasMotionLayers) { commandBuffers.push(preRenderEncoder.finish()); } @@ -1149,8 +1163,10 @@ export class RenderDispatcher { const primarySelectedClipId = timelineState.primarySelectedClipId && timelineState.selectedClipIds.has(timelineState.primarySelectedClipId) ? timelineState.primarySelectedClipId : timelineState.selectedClipIds.values().next().value as string | undefined; - const sceneGizmoClipId = engineState.sceneGizmoClipIdOverride ?? - (engineState.previewCameraOverride ? null : primarySelectedClipId ?? null); + const sceneGizmoVisible = engineState.sceneGizmoVisible !== false; + const sceneGizmoClipId = sceneGizmoVisible + ? engineState.sceneGizmoClipIdOverride ?? (engineState.previewCameraOverride ? null : primarySelectedClipId ?? null) + : null; const sceneGizmoClip = sceneGizmoClipId ? timelineState.clips.find((clip) => clip.id === sceneGizmoClipId) ?? null : null; @@ -2249,6 +2265,17 @@ export class RenderDispatcher { layerData.push({ layer, isVideo: false, externalTexture: null, textureView: d.textureManager!.getImageView(texture), sourceWidth: canvas.width, sourceHeight: canvas.height }); } } + if (layer.source.type === 'motion') { + const size = getMotionRenderSize(layer.source.motion); + layerData.push({ + layer, + isVideo: false, + externalTexture: null, + textureView: null, + sourceWidth: size.width, + sourceHeight: size.height, + }); + } } const { width, height } = d.renderTargetManager!.getResolution(); @@ -2274,6 +2301,14 @@ export class RenderDispatcher { } const commandEncoder = device.createCommandEncoder(); + for (const data of layerData) { + if (data.layer.source?.type !== 'motion') continue; + const rendered = d.motionRenderer?.renderLayer(data.layer, commandEncoder); + const size = rendered ?? getMotionRenderSize(data.layer.source.motion); + data.textureView = rendered?.textureView ?? null; + data.sourceWidth = size.width; + data.sourceHeight = size.height; + } // Ping-pong compositing using independent buffers let readView = indPingView; diff --git a/src/services/layerBuilder/LayerBuilderService.ts b/src/services/layerBuilder/LayerBuilderService.ts index 16488d21..93f7885c 100644 --- a/src/services/layerBuilder/LayerBuilderService.ts +++ b/src/services/layerBuilder/LayerBuilderService.ts @@ -25,6 +25,7 @@ import { Logger } from '../logger'; import { flags } from '../../engine/featureFlags'; import { getInterpolatedClipTransform } from '../../utils/keyframeInterpolation'; import { getEffectiveScale } from '../../utils/transformScale'; +import { getInterpolatedMotionLayer } from '../../utils/motionInterpolation'; import { getGaussianSplatSequenceFrame, getGaussianSplatSequenceFrameRuntimeKey, @@ -36,6 +37,7 @@ import { resolveSceneEffectorsEnabled } from '../../engine/scene/SceneEffectorUt import { useTimelineStore } from '../../stores/timeline'; import { useMediaStore } from '../../stores/mediaStore'; import type { MediaFile } from '../../stores/mediaStore/types'; +import { getExpectedProxyFrameCount } from '../../stores/mediaStore/helpers/proxyCompleteness'; import { DEFAULT_TRANSFORM, MAX_NESTING_DEPTH } from '../../stores/timeline/constants'; import { prewarmGaussianSplatRuntime } from '../../engine/scene/runtime/SharedSplatRuntimeCache'; import { resolveSharedSplatUseNativeRenderer } from '../../engine/scene/runtime/SharedSplatRuntimeUtils'; @@ -548,6 +550,14 @@ export class LayerBuilderService { else if (clip.source?.textCanvas) { layer = this.buildTextLayer(clip, layerIndex, ctx, opacityOverride); } + // Motion shape clip + else if (clip.source?.type === 'motion-shape') { + layer = this.buildMotionShapeLayer(clip, layerIndex, ctx, opacityOverride); + } + // Motion null and adjustment clips are timeline/controller data for now. + else if (clip.source?.type === 'motion-null' || clip.source?.type === 'motion-adjustment') { + layer = null; + } // Camera clip (non-rendering scene controller) else if (clip.source?.type === 'camera') { layer = null; @@ -573,6 +583,46 @@ export class LayerBuilderService { return layer; } + private buildMotionShapeLayer(clip: TimelineClip, layerIndex: number, ctx: FrameContext, opacityOverride?: number): Layer | null { + if (!clip.motion || clip.motion.kind !== 'shape') { + return null; + } + + const timeInfo = getClipTimeInfo(ctx, clip); + const transform = this.transformCache.getTransform( + `${ctx.activeCompId}_${layerIndex}_${clip.id}`, + ctx.getInterpolatedTransform(clip.id, timeInfo.clipLocalTime) + ); + const effects = ctx.getInterpolatedEffects(clip.id, timeInfo.clipLocalTime); + const colorCorrection = ctx.getInterpolatedColorCorrection(clip.id, timeInfo.clipLocalTime); + const keyframes = useTimelineStore.getState().clipKeyframes.get(clip.id) ?? []; + const motion = getInterpolatedMotionLayer(clip, keyframes, timeInfo.clipLocalTime) ?? clip.motion; + const finalOpacity = opacityOverride !== undefined + ? transform.opacity * opacityOverride + : transform.opacity; + + const layer: Layer = { + id: `${ctx.activeCompId}_layer_${layerIndex}_${clip.id}`, + name: clip.name, + sourceClipId: clip.id, + visible: true, + opacity: finalOpacity, + blendMode: transform.blendMode as BlendMode, + source: { + type: 'motion', + motion, + }, + effects, + colorCorrection, + position: transform.position, + scale: transform.scale, + rotation: transform.rotation, + }; + + this.addMaskProperties(layer, clip); + return layer; + } + /** * Build nested composition layer */ @@ -784,17 +834,7 @@ export class LayerBuilderService { const proxyFps = mediaFile.proxyFps || 30; const frameIndex = Math.floor(clipTime * proxyFps); - // Check proxy availability - let useProxy = false; - if (mediaFile.proxyStatus === 'ready') { - useProxy = true; - } else if (mediaFile.proxyStatus === 'generating' && (mediaFile.proxyProgress || 0) > 0) { - const totalFrames = Math.ceil((mediaFile.duration || 10) * proxyFps); - const maxGeneratedFrame = Math.floor(totalFrames * ((mediaFile.proxyProgress || 0) / 100)); - useProxy = frameIndex < maxGeneratedFrame; - } - - if (!useProxy) return null; + if (!this.canUseProxyFrame(mediaFile, frameIndex, proxyFps)) return null; // Try to get cached frame const cacheKey = `${mediaFile.id}_${clip.id}`; @@ -846,6 +886,23 @@ export class LayerBuilderService { return null; } + private canUseProxyFrame(mediaFile: MediaFile, frameIndex: number, proxyFps: number): boolean { + if (mediaFile.proxyStatus === 'ready') { + return true; + } + + if (mediaFile.proxyStatus !== 'generating' || (mediaFile.proxyProgress || 0) <= 0) { + return false; + } + + const fallbackDuration = mediaFile.duration || 10; + const totalFrames = + getExpectedProxyFrameCount(fallbackDuration, proxyFps) ?? + Math.ceil(fallbackDuration * proxyFps); + const maxGeneratedFrame = Math.floor(totalFrames * ((mediaFile.proxyProgress || 0) / 100)); + return frameIndex < maxGeneratedFrame; + } + private ensureProxyFrameLoaded( mediaFileId: string, frameIndex: number, @@ -1417,6 +1474,21 @@ export class LayerBuilderService { } if (this.hasRenderableVideoSource(nestedClip.source)) { + const nestedClipTime = this.getNestedClipSourceTime(nestedClip, nestedClipLocalTime); + + if (ctx.proxyEnabled) { + const mediaFile = getMediaFileForClip(ctx, nestedClip); + if (mediaFile?.proxyFps) { + const proxyLayer = this.tryBuildNestedProxyLayer( + nestedClip, + nestedClipTime, + mediaFile, + baseLayer, + ); + if (proxyLayer) return proxyLayer; + } + } + const keepScrubRuntimeActive = ctx.isDraggingPlayhead || scrubSettleState.isPending(nestedClip.id); const useScrubRuntime = keepScrubRuntimeActive; @@ -1442,9 +1514,6 @@ export class LayerBuilderService { runtimeProvider?.isFullMode() ? runtimeProvider : previewRuntimeSource?.webCodecsPlayer ?? nestedClip.source!.webCodecsPlayer; - const nestedClipTime = nestedClip.reversed - ? nestedClip.outPoint - nestedClipLocalTime - : nestedClipLocalTime + nestedClip.inPoint; return { ...baseLayer, source: { @@ -1487,6 +1556,19 @@ export class LayerBuilderService { source: { type: 'text', textCanvas: nestedClip.source.textCanvas }, } as Layer; } + } else if (nestedClip.source?.type === 'motion-shape' && nestedClip.motion?.kind === 'shape') { + return { + ...baseLayer, + source: { + type: 'motion', + motion: getInterpolatedMotionLayer(nestedClip, keyframes, nestedClipLocalTime) ?? nestedClip.motion, + }, + } as Layer; + } else if ( + nestedClip.source?.type === 'motion-null' || + nestedClip.source?.type === 'motion-adjustment' + ) { + return null; } else if (nestedClip.source?.type === 'model') { const nestedSourceTime = nestedClip.reversed ? nestedClip.outPoint - nestedClipLocalTime @@ -1526,6 +1608,85 @@ export class LayerBuilderService { return null; } + private getNestedClipSourceTime(nestedClip: TimelineClip, nestedClipLocalTime: number): number { + const inPoint = nestedClip.inPoint ?? 0; + const outPoint = nestedClip.outPoint ?? nestedClip.duration; + return nestedClip.reversed + ? outPoint - nestedClipLocalTime + : nestedClipLocalTime + inPoint; + } + + private tryBuildNestedProxyLayer( + nestedClip: TimelineClip, + nestedClipTime: number, + mediaFile: MediaFile, + baseLayer: Omit, + ): Layer | null { + const proxyFps = mediaFile.proxyFps || 30; + const frameIndex = Math.floor(nestedClipTime * proxyFps); + if (!this.canUseProxyFrame(mediaFile, frameIndex, proxyFps)) return null; + + const cacheKey = `${mediaFile.id}_${nestedClip.id}`; + const exactFrame = proxyFrameCache.getCachedFrame(mediaFile.id, frameIndex, proxyFps); + if (exactFrame) { + this.proxyFramesRef.set(cacheKey, { frameIndex, image: exactFrame }); + return this.buildNestedProxyImageLayer(baseLayer, exactFrame, mediaFile, frameIndex, nestedClipTime, 'nested-proxy-frame'); + } + + this.ensureProxyFrameLoaded(mediaFile.id, frameIndex, nestedClipTime, proxyFps, cacheKey); + + const nearestFrame = proxyFrameCache.getNearestCachedFrameEntry(mediaFile.id, frameIndex, 30); + if (nearestFrame) { + return this.buildNestedProxyImageLayer( + baseLayer, + nearestFrame.image, + mediaFile, + nearestFrame.frameIndex, + nestedClipTime, + 'nested-proxy-frame-nearest', + ); + } + + const heldFrame = this.proxyFramesRef.get(cacheKey); + if (heldFrame?.image) { + return this.buildNestedProxyImageLayer( + baseLayer, + heldFrame.image, + mediaFile, + heldFrame.frameIndex, + nestedClipTime, + 'nested-proxy-frame-hold', + ); + } + + return null; + } + + private buildNestedProxyImageLayer( + baseLayer: Omit, + imageElement: HTMLImageElement, + mediaFile: MediaFile, + frameIndex: number, + targetMediaTime: number, + previewPath: string, + ): Layer { + const proxyFps = mediaFile.proxyFps || 30; + return { + ...baseLayer, + source: { + type: 'image', + imageElement, + mediaFileId: mediaFile.id, + intrinsicWidth: imageElement.naturalWidth || imageElement.width, + intrinsicHeight: imageElement.naturalHeight || imageElement.height, + mediaTime: frameIndex / proxyFps, + targetMediaTime, + previewPath, + proxyFrameIndex: frameIndex, + }, + }; + } + /** * Preload proxy frames for upcoming nested compositions */ diff --git a/src/services/project/projectLoad.ts b/src/services/project/projectLoad.ts index 44402237..d9c52ea1 100644 --- a/src/services/project/projectLoad.ts +++ b/src/services/project/projectLoad.ts @@ -720,6 +720,8 @@ function convertProjectCompositionToStore( solidColor: c.solidColor, // Math scene clip support mathScene: c.mathScene ? structuredClone(c.mathScene) : undefined, + // Motion design clip support + motion: c.motion ? structuredClone(c.motion) : undefined, vectorAnimationSettings: c.vectorAnimationSettings, // 3D layer support is3D: c.is3D, @@ -1585,6 +1587,37 @@ async function reloadNestedCompositionClips(): Promise { const nestedTracks = composition.timelineData.tracks; for (const nestedSerializedClip of composition.timelineData.clips) { + if ( + ( + nestedSerializedClip.sourceType === 'motion-shape' || + nestedSerializedClip.sourceType === 'motion-null' || + nestedSerializedClip.sourceType === 'motion-adjustment' + ) && + nestedSerializedClip.motion + ) { + nestedClips.push({ + id: `nested-${compClip.id}-${nestedSerializedClip.id}`, + trackId: nestedSerializedClip.trackId, + name: nestedSerializedClip.name || 'Motion', + file: new File([JSON.stringify(nestedSerializedClip.motion)], `${nestedSerializedClip.sourceType}.msmotion`, { type: 'application/json' }), + startTime: nestedSerializedClip.startTime, + duration: nestedSerializedClip.duration, + inPoint: nestedSerializedClip.inPoint, + outPoint: nestedSerializedClip.outPoint, + source: { + type: nestedSerializedClip.sourceType, + naturalDuration: nestedSerializedClip.duration, + }, + motion: structuredClone(nestedSerializedClip.motion), + thumbnails: nestedSerializedClip.thumbnails, + transform: nestedSerializedClip.transform, + effects: nestedSerializedClip.effects || [], + masks: nestedSerializedClip.masks || [], + isLoading: false, + }); + continue; + } + if (nestedSerializedClip.sourceType === 'math-scene' && nestedSerializedClip.mathScene) { const canvas = mathSceneRenderer.createCanvas(); const nestedClip: TimelineClip = { diff --git a/src/services/project/projectSave.ts b/src/services/project/projectSave.ts index 32b59846..6900672c 100644 --- a/src/services/project/projectSave.ts +++ b/src/services/project/projectSave.ts @@ -241,6 +241,8 @@ function convertCompositions(compositions: Composition[]): ProjectComposition[] solidColor: c.solidColor || undefined, // Math scene clip support mathScene: c.mathScene ? structuredClone(c.mathScene) : undefined, + // Motion design clip support + motion: c.motion ? structuredClone(c.motion) : undefined, vectorAnimationSettings: c.source?.vectorAnimationSettings || c.vectorAnimationSettings || undefined, // Transcript data transcript: c.transcript || undefined, diff --git a/src/services/project/types/composition.types.ts b/src/services/project/types/composition.types.ts index 840feeab..1df1c930 100644 --- a/src/services/project/types/composition.types.ts +++ b/src/services/project/types/composition.types.ts @@ -10,6 +10,7 @@ import type { GaussianSplatSequenceData, ModelSequenceData, MathSceneDefinition, + MotionLayerDefinition, SceneSegment, Text3DProperties, TextClipProperties, @@ -72,7 +73,7 @@ export interface ProjectClip { compositionId?: string; // Additional clip metadata (for restoration) - sourceType?: 'video' | 'audio' | 'image' | 'text' | 'solid' | 'model' | 'camera' | 'gaussian-avatar' | 'gaussian-splat' | 'splat-effector' | 'math-scene' | 'lottie' | 'rive'; + sourceType?: 'video' | 'audio' | 'image' | 'text' | 'solid' | 'model' | 'camera' | 'gaussian-avatar' | 'gaussian-splat' | 'splat-effector' | 'math-scene' | 'motion-shape' | 'motion-null' | 'motion-adjustment' | 'lottie' | 'rive'; naturalDuration?: number; linkedClipId?: string; linkedGroupId?: string; @@ -98,6 +99,9 @@ export interface ProjectClip { // Math scene clip support mathScene?: MathSceneDefinition; + // Motion design clip support + motion?: MotionLayerDefinition; + vectorAnimationSettings?: VectorAnimationClipSettings; // Transcript data diff --git a/src/services/properties/PropertyRegistry.ts b/src/services/properties/PropertyRegistry.ts new file mode 100644 index 00000000..0fba757f --- /dev/null +++ b/src/services/properties/PropertyRegistry.ts @@ -0,0 +1,132 @@ +import type { + PropertyDescriptor, + PropertyDescriptorProvider, + PropertyDescriptorResolver, + PropertySearchOptions, + PropertyValue, +} from '../../types/propertyRegistry'; +import type { TimelineClip } from '../../types'; + +function normalizeSearchText(value: string): string { + return value.trim().toLowerCase(); +} + +function descriptorMatchesQuery(descriptor: PropertyDescriptor, query: string): boolean { + if (!query) return true; + + const haystack = [ + descriptor.path, + descriptor.label, + descriptor.group, + ...(descriptor.ui?.aliases ?? []), + ].map(normalizeSearchText); + + const tokens = normalizeSearchText(query) + .split(/\s+/) + .filter(Boolean); + + return tokens.every((token) => haystack.some((candidate) => candidate.includes(token))); +} + +function sortDescriptors(a: PropertyDescriptor, b: PropertyDescriptor): number { + const groupCompare = a.group.localeCompare(b.group); + if (groupCompare !== 0) return groupCompare; + return a.label.localeCompare(b.label); +} + +export class PropertyRegistry { + private descriptors = new Map(); + private resolvers = new Map(); + private providers = new Map(); + + register(descriptor: PropertyDescriptor): void { + this.descriptors.set(descriptor.path, descriptor as PropertyDescriptor); + } + + registerMany(descriptors: PropertyDescriptor[]): void { + descriptors.forEach((descriptor) => this.register(descriptor)); + } + + registerResolver(id: string, resolver: PropertyDescriptorResolver): void { + if (!this.resolvers.has(id)) { + this.resolvers.set(id, resolver); + } + } + + registerProvider(id: string, provider: PropertyDescriptorProvider): void { + if (!this.providers.has(id)) { + this.providers.set(id, provider); + } + } + + has(path: string): boolean { + return this.descriptors.has(path); + } + + clear(): void { + this.descriptors.clear(); + this.resolvers.clear(); + this.providers.clear(); + } + + getDescriptor(path: string, clip?: TimelineClip): PropertyDescriptor | undefined { + const exact = this.descriptors.get(path); + if (exact) { + return exact; + } + + for (const resolver of this.resolvers.values()) { + const resolved = resolver(path, clip); + if (resolved) { + return resolved; + } + } + + return undefined; + } + + getAllDescriptors(clip?: TimelineClip): PropertyDescriptor[] { + const descriptors = new Map(this.descriptors); + + if (clip) { + for (const provider of this.providers.values()) { + provider(clip).forEach((descriptor) => { + descriptors.set(descriptor.path, descriptor); + }); + } + } + + return Array.from(descriptors.values()).sort(sortDescriptors); + } + + search(options: PropertySearchOptions = {}): PropertyDescriptor[] { + return this.getAllDescriptors(options.clip) + .filter((descriptor) => { + if (options.group && descriptor.group !== options.group) return false; + if (options.animatable !== undefined && descriptor.animatable !== options.animatable) return false; + return descriptorMatchesQuery(descriptor, options.query ?? ''); + }); + } + + readValue( + clip: TimelineClip, + path: string, + ): T | undefined { + const descriptor = this.getDescriptor(path, clip); + return descriptor?.read?.(clip, path) as T | undefined; + } + + writeValue( + clip: TimelineClip, + path: string, + value: T, + ): TimelineClip { + const descriptor = this.getDescriptor(path, clip); + if (!descriptor?.write) { + throw new Error(`Property is not writable: ${path}`); + } + return descriptor.write(clip, value, path); + } +} + +export const propertyRegistry = new PropertyRegistry(); diff --git a/src/services/properties/index.ts b/src/services/properties/index.ts new file mode 100644 index 00000000..9ca1b56c --- /dev/null +++ b/src/services/properties/index.ts @@ -0,0 +1,7 @@ +export { PropertyRegistry, propertyRegistry } from './PropertyRegistry'; +export { registerCoreProperties } from './registerCoreProperties'; + +import { propertyRegistry } from './PropertyRegistry'; +import { registerCoreProperties } from './registerCoreProperties'; + +registerCoreProperties(propertyRegistry); diff --git a/src/services/properties/registerCoreProperties.ts b/src/services/properties/registerCoreProperties.ts new file mode 100644 index 00000000..483205de --- /dev/null +++ b/src/services/properties/registerCoreProperties.ts @@ -0,0 +1,973 @@ +import type { Effect as TimelineEffect, TimelineClip, ClipMask, MaskPathKeyframeValue } from '../../types'; +import { + DEFAULT_PRIMARY_COLOR_PARAMS, + RUNTIME_COLOR_PARAM_DEFS, + ensureColorCorrectionState, + parseColorProperty, + parseMaskProperty, + setColorNodeParamValue, +} from '../../types'; +import { + DEFAULT_VECTOR_ANIMATION_CLIP_SETTINGS, + createVectorAnimationInputProperty, + createVectorAnimationStateProperty, + mergeVectorAnimationSettings, + parseVectorAnimationInputProperty, + parseVectorAnimationStateProperty, +} from '../../types/vectorAnimation'; +import type { + AppearanceItem, + MotionLayerDefinition, + ReplicatorDefinition, + ReplicatorLayout, +} from '../../types/motionDesign'; +import { + DEFAULT_MOTION_SHAPE_SIZE, + createDefaultMotionLayerDefinition, + createDefaultReplicatorDefinition, + isMotionProperty, +} from '../../types/motionDesign'; +import type { EffectDefinition, EffectParam } from '../../effects/types'; +import { getAllEffects, getEffect } from '../../effects'; +import type { PropertyDescriptor, PropertyValueType } from '../../types/propertyRegistry'; +import type { PropertyRegistry } from './PropertyRegistry'; +import { propertyRegistry } from './PropertyRegistry'; + +const colorParamDefsByKey = new Map(RUNTIME_COLOR_PARAM_DEFS.map((def) => [def.key, def])); + +type TransformPatch = Omit, 'position' | 'scale' | 'rotation'> & { + position?: Partial; + scale?: Partial; + rotation?: Partial; +}; + +function updateTransform( + clip: TimelineClip, + patch: TransformPatch, +): TimelineClip { + return { + ...clip, + transform: { + ...clip.transform, + ...patch, + position: patch.position ? { ...clip.transform.position, ...patch.position } : clip.transform.position, + scale: patch.scale ? { ...clip.transform.scale, ...patch.scale } : clip.transform.scale, + rotation: patch.rotation ? { ...clip.transform.rotation, ...patch.rotation } : clip.transform.rotation, + }, + }; +} + +function createTransformDescriptor( + path: string, + label: string, + defaultValue: number, + read: (clip: TimelineClip) => number, + write: (clip: TimelineClip, value: number) => TimelineClip, + ui: NonNullable = {}, +): PropertyDescriptor { + return { + path, + label, + group: 'Transform', + valueType: 'number', + animatable: true, + defaultValue, + ui, + read, + write: (clip, value) => write(clip, value as number), + }; +} + +function mapEffectParamType(param: EffectParam): PropertyValueType { + if (param.type === 'boolean') return 'boolean'; + if (param.type === 'select') return 'enum'; + if (param.type === 'color') return 'color'; + if (param.type === 'point') return 'vector2'; + return 'number'; +} + +function isEffectParamAnimatable(param: EffectParam): boolean { + if (param.animatable !== undefined) return param.animatable; + return param.type === 'number' || param.type === 'color' || param.type === 'point'; +} + +function createEffectDescriptor( + effectDefinition: EffectDefinition, + effect: TimelineEffect | undefined, + paramName: string, + param: EffectParam, +): PropertyDescriptor { + const effectId = effect?.id ?? effectDefinition.id; + const effectName = effect?.name || effectDefinition.name; + return { + path: `effect.${effectId}.${paramName}`, + label: param.label, + group: `Effects / ${effectName}`, + valueType: mapEffectParamType(param), + animatable: isEffectParamAnimatable(param), + defaultValue: param.default, + ui: { + min: param.min, + max: param.max, + step: param.step, + aliases: [effectDefinition.name, effectDefinition.id, paramName], + options: param.options, + }, + read: (clip) => { + const current = clip.effects.find((candidate) => candidate.id === effectId); + return current?.params[paramName] ?? param.default; + }, + write: (clip, value) => ({ + ...clip, + effects: clip.effects.map((candidate) => ( + candidate.id === effectId + ? { ...candidate, params: { ...candidate.params, [paramName]: value as number | boolean | string } } + : candidate + )), + }), + }; +} + +function getEffectDescriptorForPath(path: string, clip?: TimelineClip): PropertyDescriptor | undefined { + const parts = path.split('.'); + if (parts.length !== 3 || parts[0] !== 'effect' || !clip) return undefined; + + const [, effectId, paramName] = parts; + const effect = clip.effects.find((candidate) => candidate.id === effectId); + if (!effect) return undefined; + + const effectDefinition = getEffect(effect.type); + const param = effectDefinition?.params[paramName]; + return effectDefinition && param + ? createEffectDescriptor(effectDefinition, effect, paramName, param) + : undefined; +} + +function getEffectDescriptorsForClip(clip: TimelineClip): PropertyDescriptor[] { + return clip.effects.flatMap((effect) => { + const effectDefinition = getEffect(effect.type); + if (!effectDefinition) return []; + + return Object.entries(effectDefinition.params).map(([paramName, param]) => + createEffectDescriptor(effectDefinition, effect, paramName, param) + ); + }); +} + +function getColorDescriptorForPath(path: string, clip?: TimelineClip): PropertyDescriptor | undefined { + const parsed = parseColorProperty(path); + if (!parsed || !clip?.colorCorrection) return undefined; + + const def = colorParamDefsByKey.get(parsed.paramName as keyof typeof DEFAULT_PRIMARY_COLOR_PARAMS); + if (!def) return undefined; + + const state = ensureColorCorrectionState(clip.colorCorrection); + const version = state.versions.find((candidate) => candidate.id === parsed.versionId); + const node = version?.nodes.find((candidate) => candidate.id === parsed.nodeId); + if (!node || typeof node.params[parsed.paramName] !== 'number') return undefined; + + return { + path, + label: def.label, + group: `Color / ${node.name}`, + valueType: 'number', + animatable: true, + defaultValue: def.defaultValue, + ui: { + min: def.min, + max: def.max, + step: def.step, + aliases: [def.section, node.type, parsed.paramName], + }, + read: (targetClip) => { + const currentState = ensureColorCorrectionState(targetClip.colorCorrection); + const currentVersion = currentState.versions.find((candidate) => candidate.id === parsed.versionId); + const currentNode = currentVersion?.nodes.find((candidate) => candidate.id === parsed.nodeId); + const value = currentNode?.params[parsed.paramName]; + return typeof value === 'number' ? value : def.defaultValue; + }, + write: (targetClip, value) => ({ + ...targetClip, + colorCorrection: setColorNodeParamValue( + ensureColorCorrectionState(targetClip.colorCorrection), + parsed.versionId, + parsed.nodeId, + parsed.paramName, + value as number, + ), + }), + }; +} + +function getColorDescriptorsForClip(clip: TimelineClip): PropertyDescriptor[] { + if (!clip.colorCorrection) return []; + + const state = ensureColorCorrectionState(clip.colorCorrection); + return state.versions.flatMap((version) => + version.nodes.flatMap((node) => + Object.keys(node.params).flatMap((paramName) => { + const descriptor = getColorDescriptorForPath(`color.${version.id}.${node.id}.${paramName}`, clip); + return descriptor ? [descriptor] : []; + }) + ) + ); +} + +function getMaskPathValue(mask: ClipMask): MaskPathKeyframeValue { + return { + closed: mask.closed, + vertices: mask.vertices.map((vertex) => ({ + ...vertex, + handleIn: { ...vertex.handleIn }, + handleOut: { ...vertex.handleOut }, + })), + }; +} + +function getMaskDescriptorForPath(path: string, clip?: TimelineClip): PropertyDescriptor | undefined { + const parsed = parseMaskProperty(path); + if (!parsed || !clip?.masks) return undefined; + + const mask = clip.masks.find((candidate) => candidate.id === parsed.maskId); + if (!mask) return undefined; + + if (parsed.property === 'path') { + return { + path, + label: `${mask.name} Path`, + group: 'Masks', + valueType: 'path', + animatable: true, + defaultValue: getMaskPathValue(mask), + ui: { aliases: ['mask path', mask.name] }, + read: (targetClip) => { + const targetMask = targetClip.masks?.find((candidate) => candidate.id === parsed.maskId); + return targetMask ? getMaskPathValue(targetMask) : undefined; + }, + write: (targetClip, value) => { + const pathValue = value as MaskPathKeyframeValue; + return { + ...targetClip, + masks: targetClip.masks?.map((candidate) => ( + candidate.id === parsed.maskId + ? { + ...candidate, + closed: pathValue.closed, + vertices: pathValue.vertices.map((vertex) => ({ + ...vertex, + handleIn: { ...vertex.handleIn }, + handleOut: { ...vertex.handleOut }, + })), + } + : candidate + )), + }; + }, + }; + } + + const numericProperty = parsed.property as 'position.x' | 'position.y' | 'feather' | 'featherQuality'; + const labelByProperty: Record = { + 'position.x': `${mask.name} X`, + 'position.y': `${mask.name} Y`, + feather: `${mask.name} Feather`, + featherQuality: `${mask.name} Feather Quality`, + }; + + return { + path, + label: labelByProperty[numericProperty], + group: 'Masks', + valueType: 'number', + animatable: true, + defaultValue: numericProperty.startsWith('position.') ? 0 : numericProperty === 'featherQuality' ? 1 : 0, + ui: { + min: numericProperty === 'feather' ? 0 : numericProperty === 'featherQuality' ? 1 : undefined, + max: numericProperty === 'featherQuality' ? 100 : undefined, + step: numericProperty === 'featherQuality' ? 1 : 0.1, + aliases: [mask.name, numericProperty], + }, + read: (targetClip) => { + const targetMask = targetClip.masks?.find((candidate) => candidate.id === parsed.maskId); + if (!targetMask) return undefined; + if (numericProperty === 'position.x') return targetMask.position.x; + if (numericProperty === 'position.y') return targetMask.position.y; + return targetMask[numericProperty]; + }, + write: (targetClip, value) => ({ + ...targetClip, + masks: targetClip.masks?.map((candidate) => { + if (candidate.id !== parsed.maskId) return candidate; + if (numericProperty === 'position.x') { + return { ...candidate, position: { ...candidate.position, x: value as number } }; + } + if (numericProperty === 'position.y') { + return { ...candidate, position: { ...candidate.position, y: value as number } }; + } + return { ...candidate, [numericProperty]: value as number }; + }), + }), + }; +} + +function getMaskDescriptorsForClip(clip: TimelineClip): PropertyDescriptor[] { + return (clip.masks ?? []).flatMap((mask) => [ + getMaskDescriptorForPath(`mask.${mask.id}.path`, clip), + getMaskDescriptorForPath(`mask.${mask.id}.position.x`, clip), + getMaskDescriptorForPath(`mask.${mask.id}.position.y`, clip), + getMaskDescriptorForPath(`mask.${mask.id}.feather`, clip), + getMaskDescriptorForPath(`mask.${mask.id}.featherQuality`, clip), + ].filter((descriptor): descriptor is PropertyDescriptor => Boolean(descriptor))); +} + +function getVectorDescriptorForPath(path: string, clip?: TimelineClip): PropertyDescriptor | undefined { + if (clip?.source?.type !== 'lottie') return undefined; + + const stateProperty = parseVectorAnimationStateProperty(path); + if (stateProperty) { + const settings = mergeVectorAnimationSettings(clip.source.vectorAnimationSettings); + return { + path, + label: `${stateProperty.stateMachineName} State`, + group: 'Vector Animation', + valueType: 'enum', + animatable: true, + defaultValue: settings.stateMachineState ?? '', + ui: { aliases: ['lottie state', stateProperty.stateMachineName] }, + read: (targetClip) => mergeVectorAnimationSettings(targetClip.source?.vectorAnimationSettings).stateMachineState ?? '', + write: (targetClip, value) => ({ + ...targetClip, + source: targetClip.source + ? { + ...targetClip.source, + vectorAnimationSettings: { + ...DEFAULT_VECTOR_ANIMATION_CLIP_SETTINGS, + ...targetClip.source.vectorAnimationSettings, + stateMachineName: stateProperty.stateMachineName, + stateMachineState: String(value), + stateMachineStateCues: undefined, + }, + } + : targetClip.source, + }), + }; + } + + const inputProperty = parseVectorAnimationInputProperty(path); + if (!inputProperty) return undefined; + + const settings = mergeVectorAnimationSettings(clip.source.vectorAnimationSettings); + const currentValue = settings.stateMachineInputValues?.[inputProperty.inputName] ?? 0; + return { + path, + label: inputProperty.inputName, + group: 'Vector Animation', + valueType: typeof currentValue === 'boolean' ? 'boolean' : typeof currentValue === 'string' ? 'enum' : 'number', + animatable: typeof currentValue !== 'string', + defaultValue: currentValue, + ui: { aliases: ['lottie input', inputProperty.stateMachineName] }, + read: (targetClip) => { + const targetSettings = mergeVectorAnimationSettings(targetClip.source?.vectorAnimationSettings); + return targetSettings.stateMachineInputValues?.[inputProperty.inputName] ?? currentValue; + }, + write: (targetClip, value) => { + const targetSettings = mergeVectorAnimationSettings(targetClip.source?.vectorAnimationSettings); + return { + ...targetClip, + source: targetClip.source + ? { + ...targetClip.source, + vectorAnimationSettings: { + ...targetSettings, + stateMachineName: inputProperty.stateMachineName, + stateMachineInputValues: { + ...(targetSettings.stateMachineInputValues ?? {}), + [inputProperty.inputName]: value as boolean | number | string, + }, + }, + } + : targetClip.source, + }; + }, + }; +} + +function getVectorDescriptorsForClip(clip: TimelineClip): PropertyDescriptor[] { + if (clip.source?.type !== 'lottie') return []; + + const settings = mergeVectorAnimationSettings(clip.source.vectorAnimationSettings); + const descriptors: PropertyDescriptor[] = []; + if (settings.stateMachineName) { + const stateDescriptor = getVectorDescriptorForPath( + createVectorAnimationStateProperty(settings.stateMachineName), + clip, + ); + if (stateDescriptor) descriptors.push(stateDescriptor); + + Object.keys(settings.stateMachineInputValues ?? {}).forEach((inputName) => { + const descriptor = getVectorDescriptorForPath( + createVectorAnimationInputProperty(settings.stateMachineName!, inputName), + clip, + ); + if (descriptor) descriptors.push(descriptor); + }); + } + return descriptors; +} + +function cloneMotion(motion: MotionLayerDefinition | undefined): MotionLayerDefinition { + return structuredClone(motion ?? createDefaultMotionLayerDefinition('shape')) as MotionLayerDefinition; +} + +function withMotion(clip: TimelineClip, updater: (motion: MotionLayerDefinition) => MotionLayerDefinition): TimelineClip { + return { + ...clip, + motion: updater(cloneMotion(clip.motion)), + }; +} + +function updateShape( + clip: TimelineClip, + updater: (motion: MotionLayerDefinition) => MotionLayerDefinition, +): TimelineClip { + return withMotion(clip, updater); +} + +function ensureReplicator(motion: MotionLayerDefinition): ReplicatorDefinition { + return motion.replicator ? structuredClone(motion.replicator) : createDefaultReplicatorDefinition(); +} + +function createGridLayout(layout: ReplicatorLayout): Extract { + if (layout.mode === 'grid') return structuredClone(layout); + return createDefaultReplicatorDefinition().layout as Extract; +} + +function createMotionShapeDescriptor( + path: 'shape.size.w' | 'shape.size.h' | 'shape.cornerRadius', + label: string, +): PropertyDescriptor { + return { + path, + label, + group: 'Motion / Shape', + valueType: 'number', + animatable: true, + defaultValue: path === 'shape.size.w' + ? DEFAULT_MOTION_SHAPE_SIZE.w + : path === 'shape.size.h' + ? DEFAULT_MOTION_SHAPE_SIZE.h + : 0, + ui: { + min: 0, + step: 1, + aliases: ['motion', 'shape'], + }, + read: (clip) => { + if (path === 'shape.size.w') return clip.motion?.shape?.size.w ?? DEFAULT_MOTION_SHAPE_SIZE.w; + if (path === 'shape.size.h') return clip.motion?.shape?.size.h ?? DEFAULT_MOTION_SHAPE_SIZE.h; + return clip.motion?.shape?.cornerRadius ?? 0; + }, + write: (clip, value) => updateShape(clip, (motion) => ({ + ...motion, + shape: { + ...(motion.shape ?? createDefaultMotionLayerDefinition('shape').shape!), + size: { + ...(motion.shape?.size ?? DEFAULT_MOTION_SHAPE_SIZE), + ...(path === 'shape.size.w' ? { w: value as number } : {}), + ...(path === 'shape.size.h' ? { h: value as number } : {}), + }, + ...(path === 'shape.cornerRadius' ? { cornerRadius: value as number } : {}), + }, + })), + }; +} + +function getAppearanceItem(motion: MotionLayerDefinition | undefined, itemId: string): AppearanceItem | undefined { + return motion?.appearance?.items.find((item) => item.id === itemId); +} + +function createAppearanceDescriptor(path: string, clip: TimelineClip): PropertyDescriptor | undefined { + const match = /^appearance\.([^.]+)\.(.+)$/.exec(path); + if (!match) return undefined; + + const [, itemId, field] = match; + const item = getAppearanceItem(clip.motion, itemId); + if (!item) return undefined; + + const common = { + path, + group: `Motion / Appearance / ${item.name}`, + ui: { aliases: ['motion', 'appearance', item.kind, item.name] }, + }; + + if (field === 'opacity') { + return { + ...common, + label: `${item.name} Opacity`, + valueType: 'number', + animatable: true, + defaultValue: 1, + ui: { ...common.ui, min: 0, max: 1, step: 0.01 }, + read: (targetClip) => getAppearanceItem(targetClip.motion, itemId)?.opacity ?? 1, + write: (targetClip, value) => withMotion(targetClip, (motion) => ({ + ...motion, + appearance: motion.appearance + ? { + ...motion.appearance, + items: motion.appearance.items.map((candidate) => ( + candidate.id === itemId ? { ...candidate, opacity: value as number } : candidate + )), + } + : motion.appearance, + })), + }; + } + + const colorMatch = /^color\.(r|g|b|a)$/.exec(field); + if (colorMatch && (item.kind === 'color-fill' || item.kind === 'stroke')) { + const channel = colorMatch[1] as 'r' | 'g' | 'b' | 'a'; + return { + ...common, + label: `${item.name} ${channel.toUpperCase()}`, + valueType: 'number', + animatable: true, + defaultValue: channel === 'a' ? 1 : 0, + ui: { ...common.ui, min: 0, max: 1, step: 0.01 }, + read: (targetClip) => { + const targetItem = getAppearanceItem(targetClip.motion, itemId); + return targetItem && (targetItem.kind === 'color-fill' || targetItem.kind === 'stroke') + ? targetItem.color[channel] + : undefined; + }, + write: (targetClip, value) => withMotion(targetClip, (motion) => ({ + ...motion, + appearance: motion.appearance + ? { + ...motion.appearance, + items: motion.appearance.items.map((candidate) => ( + candidate.id === itemId && (candidate.kind === 'color-fill' || candidate.kind === 'stroke') + ? { ...candidate, color: { ...candidate.color, [channel]: value as number } } + : candidate + )), + } + : motion.appearance, + })), + }; + } + + if (field === 'stroke.width' && item.kind === 'stroke') { + return { + ...common, + label: `${item.name} Width`, + valueType: 'number', + animatable: true, + defaultValue: item.width, + ui: { ...common.ui, min: 0, step: 0.5 }, + read: (targetClip) => { + const targetItem = getAppearanceItem(targetClip.motion, itemId); + return targetItem?.kind === 'stroke' ? targetItem.width : undefined; + }, + write: (targetClip, value) => withMotion(targetClip, (motion) => ({ + ...motion, + appearance: motion.appearance + ? { + ...motion.appearance, + items: motion.appearance.items.map((candidate) => ( + candidate.id === itemId && candidate.kind === 'stroke' + ? { ...candidate, width: value as number } + : candidate + )), + } + : motion.appearance, + })), + }; + } + + if (field === 'stroke.alignment' && item.kind === 'stroke') { + return { + ...common, + label: `${item.name} Alignment`, + valueType: 'enum', + animatable: false, + defaultValue: item.alignment, + ui: { + ...common.ui, + options: [ + { value: 'center', label: 'Center' }, + { value: 'inside', label: 'Inside' }, + { value: 'outside', label: 'Outside' }, + ], + }, + read: (targetClip) => { + const targetItem = getAppearanceItem(targetClip.motion, itemId); + return targetItem?.kind === 'stroke' ? targetItem.alignment : undefined; + }, + write: (targetClip, value) => withMotion(targetClip, (motion) => ({ + ...motion, + appearance: motion.appearance + ? { + ...motion.appearance, + items: motion.appearance.items.map((candidate) => ( + candidate.id === itemId && candidate.kind === 'stroke' + ? { ...candidate, alignment: value as 'center' | 'inside' | 'outside' } + : candidate + )), + } + : motion.appearance, + })), + }; + } + + return undefined; +} + +function getReplicatorDescriptorForPath(path: string, clip?: TimelineClip): PropertyDescriptor | undefined { + if (!isMotionProperty(path) || !path.startsWith('replicator.')) return undefined; + + const defaultReplicator = createDefaultReplicatorDefinition(); + const current = clip?.motion?.replicator ?? defaultReplicator; + const grid = createGridLayout(current.layout); + + const specs: Record number | boolean | string; + write: (replicator: ReplicatorDefinition, value: unknown) => ReplicatorDefinition; + ui?: PropertyDescriptor['ui']; + }> = { + 'replicator.enabled': { + label: 'Enabled', + valueType: 'boolean', + defaultValue: false, + animatable: false, + read: (replicator) => replicator.enabled, + write: (replicator, value) => ({ ...replicator, enabled: Boolean(value) }), + }, + 'replicator.layout.mode': { + label: 'Layout', + valueType: 'enum', + defaultValue: 'grid', + animatable: false, + read: (replicator) => replicator.layout.mode, + write: (replicator, value) => ({ + ...replicator, + layout: String(value) === 'grid' ? createGridLayout(replicator.layout) : replicator.layout, + }), + ui: { options: [{ value: 'grid', label: 'Grid' }] }, + }, + 'replicator.count.x': { + label: 'Count X', + valueType: 'number', + defaultValue: grid.count.x, + animatable: true, + read: (replicator) => createGridLayout(replicator.layout).count.x, + write: (replicator, value) => { + const layout = createGridLayout(replicator.layout); + return { ...replicator, layout: { ...layout, count: { ...layout.count, x: Math.max(1, Math.round(value as number)) } } }; + }, + ui: { min: 1, step: 1 }, + }, + 'replicator.count.y': { + label: 'Count Y', + valueType: 'number', + defaultValue: grid.count.y, + animatable: true, + read: (replicator) => createGridLayout(replicator.layout).count.y, + write: (replicator, value) => { + const layout = createGridLayout(replicator.layout); + return { ...replicator, layout: { ...layout, count: { ...layout.count, y: Math.max(1, Math.round(value as number)) } } }; + }, + ui: { min: 1, step: 1 }, + }, + 'replicator.spacing.x': { + label: 'Spacing X', + valueType: 'number', + defaultValue: grid.spacing.x, + animatable: true, + read: (replicator) => createGridLayout(replicator.layout).spacing.x, + write: (replicator, value) => { + const layout = createGridLayout(replicator.layout); + return { ...replicator, layout: { ...layout, spacing: { ...layout.spacing, x: value as number } } }; + }, + ui: { step: 1 }, + }, + 'replicator.spacing.y': { + label: 'Spacing Y', + valueType: 'number', + defaultValue: grid.spacing.y, + animatable: true, + read: (replicator) => createGridLayout(replicator.layout).spacing.y, + write: (replicator, value) => { + const layout = createGridLayout(replicator.layout); + return { ...replicator, layout: { ...layout, spacing: { ...layout.spacing, y: value as number } } }; + }, + ui: { step: 1 }, + }, + 'replicator.offset.position.x': { + label: 'Offset X', + valueType: 'number', + defaultValue: defaultReplicator.offset.position.x, + animatable: true, + read: (replicator) => replicator.offset.position.x, + write: (replicator, value) => ({ ...replicator, offset: { ...replicator.offset, position: { ...replicator.offset.position, x: value as number } } }), + ui: { step: 1 }, + }, + 'replicator.offset.position.y': { + label: 'Offset Y', + valueType: 'number', + defaultValue: defaultReplicator.offset.position.y, + animatable: true, + read: (replicator) => replicator.offset.position.y, + write: (replicator, value) => ({ ...replicator, offset: { ...replicator.offset, position: { ...replicator.offset.position, y: value as number } } }), + ui: { step: 1 }, + }, + 'replicator.offset.rotation': { + label: 'Offset Rotation', + valueType: 'number', + defaultValue: defaultReplicator.offset.rotation, + animatable: true, + read: (replicator) => replicator.offset.rotation, + write: (replicator, value) => ({ ...replicator, offset: { ...replicator.offset, rotation: value as number } }), + ui: { unit: 'deg', step: 0.1 }, + }, + 'replicator.offset.scale.x': { + label: 'Offset Scale X', + valueType: 'number', + defaultValue: defaultReplicator.offset.scale.x, + animatable: true, + read: (replicator) => replicator.offset.scale.x, + write: (replicator, value) => ({ ...replicator, offset: { ...replicator.offset, scale: { ...replicator.offset.scale, x: value as number } } }), + ui: { step: 0.01 }, + }, + 'replicator.offset.scale.y': { + label: 'Offset Scale Y', + valueType: 'number', + defaultValue: defaultReplicator.offset.scale.y, + animatable: true, + read: (replicator) => replicator.offset.scale.y, + write: (replicator, value) => ({ ...replicator, offset: { ...replicator.offset, scale: { ...replicator.offset.scale, y: value as number } } }), + ui: { step: 0.01 }, + }, + 'replicator.offset.opacity': { + label: 'Offset Opacity', + valueType: 'number', + defaultValue: defaultReplicator.offset.opacity, + animatable: true, + read: (replicator) => replicator.offset.opacity, + write: (replicator, value) => ({ ...replicator, offset: { ...replicator.offset, opacity: value as number } }), + ui: { min: 0, max: 1, step: 0.01 }, + }, + }; + + const spec = specs[path]; + if (!spec) return undefined; + + return { + path, + label: spec.label, + group: 'Motion / Replicator', + valueType: spec.valueType, + animatable: spec.animatable, + defaultValue: spec.defaultValue, + ui: { aliases: ['motion', 'replicator'], ...spec.ui }, + read: (targetClip) => spec.read(targetClip.motion?.replicator ?? defaultReplicator), + write: (targetClip, value) => withMotion(targetClip, (motion) => { + const replicator = ensureReplicator(motion); + return { + ...motion, + replicator: spec.write(replicator, value), + }; + }), + }; +} + +function getMotionDescriptorForPath(path: string, clip?: TimelineClip): PropertyDescriptor | undefined { + if (!isMotionProperty(path)) return undefined; + + if (path === 'shape.size.w') return createMotionShapeDescriptor(path, 'Width'); + if (path === 'shape.size.h') return createMotionShapeDescriptor(path, 'Height'); + if (path === 'shape.cornerRadius') return createMotionShapeDescriptor(path, 'Corner Radius'); + if (path.startsWith('appearance.') && clip) return createAppearanceDescriptor(path, clip); + return getReplicatorDescriptorForPath(path, clip); +} + +function getMotionDescriptorsForClip(clip: TimelineClip): PropertyDescriptor[] { + const descriptors: PropertyDescriptor[] = [ + createMotionShapeDescriptor('shape.size.w', 'Width'), + createMotionShapeDescriptor('shape.size.h', 'Height'), + createMotionShapeDescriptor('shape.cornerRadius', 'Corner Radius'), + ]; + + clip.motion?.appearance?.items.forEach((item) => { + [ + `appearance.${item.id}.opacity`, + ...(item.kind === 'color-fill' || item.kind === 'stroke' + ? [ + `appearance.${item.id}.color.r`, + `appearance.${item.id}.color.g`, + `appearance.${item.id}.color.b`, + `appearance.${item.id}.color.a`, + ] + : []), + ...(item.kind === 'stroke' + ? [ + `appearance.${item.id}.stroke.width`, + `appearance.${item.id}.stroke.alignment`, + ] + : []), + ].forEach((path) => { + const descriptor = createAppearanceDescriptor(path, clip); + if (descriptor) descriptors.push(descriptor); + }); + }); + + [ + 'replicator.enabled', + 'replicator.layout.mode', + 'replicator.count.x', + 'replicator.count.y', + 'replicator.spacing.x', + 'replicator.spacing.y', + 'replicator.offset.position.x', + 'replicator.offset.position.y', + 'replicator.offset.rotation', + 'replicator.offset.scale.x', + 'replicator.offset.scale.y', + 'replicator.offset.opacity', + ].forEach((path) => { + const descriptor = getReplicatorDescriptorForPath(path, clip); + if (descriptor) descriptors.push(descriptor); + }); + + return descriptors; +} + +function registerTransformProperties(registry: PropertyRegistry): void { + registry.registerMany([ + createTransformDescriptor( + 'opacity', + 'Opacity', + 1, + (clip) => clip.transform.opacity, + (clip, value) => updateTransform(clip, { opacity: value }), + { min: 0, max: 1, step: 0.01, aliases: ['alpha', 'transparency'] }, + ), + createTransformDescriptor( + 'position.x', + 'Position X', + 0, + (clip) => clip.transform.position.x, + (clip, value) => updateTransform(clip, { position: { x: value } }), + { step: 1, aliases: ['x'] }, + ), + createTransformDescriptor( + 'position.y', + 'Position Y', + 0, + (clip) => clip.transform.position.y, + (clip, value) => updateTransform(clip, { position: { y: value } }), + { step: 1, aliases: ['y'] }, + ), + createTransformDescriptor( + 'position.z', + 'Position Z', + 0, + (clip) => clip.transform.position.z, + (clip, value) => updateTransform(clip, { position: { z: value } }), + { step: 1, aliases: ['z', 'depth'] }, + ), + createTransformDescriptor( + 'scale.all', + 'Scale', + 1, + (clip) => clip.transform.scale.all ?? clip.transform.scale.x, + (clip, value) => updateTransform(clip, { scale: { all: value, x: value, y: value, z: clip.transform.scale.z } }), + { step: 0.01, aliases: ['size'] }, + ), + createTransformDescriptor( + 'scale.x', + 'Scale X', + 1, + (clip) => clip.transform.scale.x, + (clip, value) => updateTransform(clip, { scale: { x: value } }), + { step: 0.01 }, + ), + createTransformDescriptor( + 'scale.y', + 'Scale Y', + 1, + (clip) => clip.transform.scale.y, + (clip, value) => updateTransform(clip, { scale: { y: value } }), + { step: 0.01 }, + ), + createTransformDescriptor( + 'scale.z', + 'Scale Z', + 1, + (clip) => clip.transform.scale.z ?? 1, + (clip, value) => updateTransform(clip, { scale: { z: value } }), + { step: 0.01 }, + ), + createTransformDescriptor( + 'rotation.x', + 'Rotation X', + 0, + (clip) => clip.transform.rotation.x, + (clip, value) => updateTransform(clip, { rotation: { x: value } }), + { unit: 'deg', step: 0.1 }, + ), + createTransformDescriptor( + 'rotation.y', + 'Rotation Y', + 0, + (clip) => clip.transform.rotation.y, + (clip, value) => updateTransform(clip, { rotation: { y: value } }), + { unit: 'deg', step: 0.1 }, + ), + createTransformDescriptor( + 'rotation.z', + 'Rotation', + 0, + (clip) => clip.transform.rotation.z, + (clip, value) => updateTransform(clip, { rotation: { z: value } }), + { unit: 'deg', step: 0.1, aliases: ['rotation z'] }, + ), + { + path: 'speed', + label: 'Speed', + group: 'Transform', + valueType: 'number', + animatable: true, + defaultValue: 1, + ui: { min: -8, max: 8, step: 0.01, aliases: ['time stretch', 'playback speed'] }, + read: (clip) => clip.speed ?? 1, + write: (clip, value) => ({ ...clip, speed: value as number }), + }, + ]); +} + +function registerEffectTemplates(registry: PropertyRegistry): void { + getAllEffects().forEach((effectDefinition) => { + Object.entries(effectDefinition.params).forEach(([paramName, param]) => { + registry.register(createEffectDescriptor(effectDefinition, undefined, paramName, param)); + }); + }); +} + +export function registerCoreProperties(registry: PropertyRegistry = propertyRegistry): PropertyRegistry { + registerTransformProperties(registry); + registerEffectTemplates(registry); + registry.registerResolver('effect-instance', getEffectDescriptorForPath); + registry.registerProvider('effect-instance', getEffectDescriptorsForClip); + registry.registerResolver('color-correction', getColorDescriptorForPath); + registry.registerProvider('color-correction', getColorDescriptorsForClip); + registry.registerResolver('mask', getMaskDescriptorForPath); + registry.registerProvider('mask', getMaskDescriptorsForClip); + registry.registerResolver('vector-animation', getVectorDescriptorForPath); + registry.registerProvider('vector-animation', getVectorDescriptorsForClip); + registry.registerResolver('motion-design', getMotionDescriptorForPath); + registry.registerProvider('motion-design', getMotionDescriptorsForClip); + return registry; +} diff --git a/src/services/proxyGenerator.ts b/src/services/proxyGenerator.ts index b101624a..b240700a 100644 --- a/src/services/proxyGenerator.ts +++ b/src/services/proxyGenerator.ts @@ -31,6 +31,7 @@ const BACKPRESSURE_POLL_MS = 5; const MIN_FLUSH_TIMEOUT_MS = 30000; const MAX_FLUSH_TIMEOUT_MS = 180000; const FLUSH_TIMEOUT_PER_SAMPLE_MS = 120; +const FRAME_COUNT_EPSILON = 1e-3; interface EncodeQueueItem { frameIndex: number; @@ -73,6 +74,29 @@ function formatMs(ms: number): string { return ms >= 1000 ? `${(ms / 1000).toFixed(2)}s` : `${ms.toFixed(0)}ms`; } +function ceilFrameCount(value: number): number { + const rounded = Math.round(value); + if (Math.abs(value - rounded) <= FRAME_COUNT_EPSILON) { + return rounded; + } + return Math.ceil(value); +} + +export function getFirstPresentationCts(samples: Sample[]): number { + let firstPresentationCts = Number.POSITIVE_INFINITY; + for (const sample of samples) { + if (Number.isFinite(sample.cts) && sample.cts < firstPresentationCts) { + firstPresentationCts = sample.cts; + } + } + return Number.isFinite(firstPresentationCts) ? firstPresentationCts : 0; +} + +export function getNormalizedSampleTimestampUs(sample: Sample, firstPresentationCts: number): number { + const normalizedCts = Math.max(0, sample.cts - firstPresentationCts); + return (normalizedCts / sample.timescale) * 1_000_000; +} + interface AVCConfigurationBox { AVCProfileIndication: number; profile_compatibility: number; @@ -323,7 +347,7 @@ class ProxyGeneratorWebCodecs { this.proxyFps = Number.isFinite(sourceFps) && sourceFps > 0 ? Math.min(PROXY_FPS, Math.round(sourceFps * 100) / 100) : PROXY_FPS; - this.totalFrames = Math.ceil(this.duration * this.proxyFps); + this.totalFrames = ceilFrameCount(this.duration * this.proxyFps); log.info(`Duration: ${this.duration.toFixed(3)}s, totalFrames: ${this.totalFrames}, samples: ${expectedSamples}, proxyFps: ${this.proxyFps.toFixed(2)}`); @@ -708,9 +732,16 @@ class ProxyGeneratorWebCodecs { const decoder = this.decoder; const sortedSamples = [...this.samples].sort((a, b) => a.dts - b.dts); + const firstPresentationCts = getFirstPresentationCts(sortedSamples); const keyframeCount = sortedSamples.filter(s => s.is_sync).length; log.info(`Decoding ${sortedSamples.length} samples (${keyframeCount} keyframes)...`); + if (firstPresentationCts > 0) { + log.debug('Normalizing proxy sample timestamps', { + firstPresentationCts, + firstPresentationSeconds: firstPresentationCts / sortedSamples[0].timescale, + }); + } const firstKeyframeIdx = sortedSamples.findIndex(s => s.is_sync); if (firstKeyframeIdx === -1) throw new Error('No keyframes found'); @@ -745,7 +776,7 @@ class ProxyGeneratorWebCodecs { const chunk = new EncodedVideoChunk({ type: sample.is_sync ? 'key' : 'delta', - timestamp: (sample.cts / sample.timescale) * 1_000_000, + timestamp: getNormalizedSampleTimestampUs(sample, firstPresentationCts), duration: (sample.duration / sample.timescale) * 1_000_000, data: sample.data, }); diff --git a/src/stores/engineStore.ts b/src/stores/engineStore.ts index 32dd520b..5643bc6b 100644 --- a/src/stores/engineStore.ts +++ b/src/stores/engineStore.ts @@ -54,6 +54,7 @@ interface EngineState { sceneNavNoKeyframes: boolean; sceneCameraLiveOverrides: Record; previewCameraOverride: SceneCameraConfig | null; + sceneGizmoVisible: boolean; sceneGizmoMode: SceneGizmoMode; sceneGizmoHoveredAxis: SceneGizmoAxis | null; sceneGizmoClipIdOverride: string | null; @@ -74,6 +75,7 @@ interface EngineState { clearSceneCameraLiveOverride: (clipId: string) => void; clearSceneCameraLiveOverrides: () => void; setPreviewCameraOverride: (camera: SceneCameraConfig | null) => void; + setSceneGizmoVisible: (visible: boolean) => void; setSceneGizmoMode: (mode: SceneGizmoMode) => void; setSceneGizmoHoveredAxis: (axis: SceneGizmoAxis | null) => void; setSceneGizmoClipIdOverride: (clipId: string | null) => void; @@ -175,6 +177,7 @@ export const useEngineStore = create()( sceneNavNoKeyframes: false, sceneCameraLiveOverrides: {}, previewCameraOverride: null, + sceneGizmoVisible: true, sceneGizmoMode: 'move', sceneGizmoHoveredAxis: null, sceneGizmoClipIdOverride: null, @@ -283,6 +286,13 @@ export const useEngineStore = create()( set({ previewCameraOverride: camera }); }, + setSceneGizmoVisible: (visible: boolean) => { + set({ + sceneGizmoVisible: visible, + ...(!visible ? { sceneGizmoHoveredAxis: null } : {}), + }); + }, + setSceneGizmoMode: (mode: SceneGizmoMode) => { set({ sceneGizmoMode: mode }); }, diff --git a/src/stores/mediaStore/helpers/proxyCompleteness.ts b/src/stores/mediaStore/helpers/proxyCompleteness.ts index fc7b6441..6afb55e9 100644 --- a/src/stores/mediaStore/helpers/proxyCompleteness.ts +++ b/src/stores/mediaStore/helpers/proxyCompleteness.ts @@ -1,6 +1,15 @@ import { PROXY_FPS } from '../constants'; const PROXY_COMPLETE_THRESHOLD = 0.98; +const FRAME_COUNT_EPSILON = 1e-3; + +function ceilFrameCount(value: number): number { + const rounded = Math.round(value); + if (Math.abs(value - rounded) <= FRAME_COUNT_EPSILON) { + return rounded; + } + return Math.ceil(value); +} export function getExpectedProxyFps(fps?: number): number { if (!fps || !Number.isFinite(fps) || fps <= 0) return PROXY_FPS; @@ -9,7 +18,7 @@ export function getExpectedProxyFps(fps?: number): number { export function getExpectedProxyFrameCount(duration?: number, fps?: number): number | null { if (!duration || !Number.isFinite(duration) || duration <= 0) return null; - return Math.ceil(duration * getExpectedProxyFps(fps)); + return ceilFrameCount(duration * getExpectedProxyFps(fps)); } export function isProxyFrameCountComplete( diff --git a/src/stores/mediaStore/helpers/thumbnailHelpers.ts b/src/stores/mediaStore/helpers/thumbnailHelpers.ts index 98d3ffc0..a6539349 100644 --- a/src/stores/mediaStore/helpers/thumbnailHelpers.ts +++ b/src/stores/mediaStore/helpers/thumbnailHelpers.ts @@ -6,6 +6,10 @@ import { Logger } from '../../../services/logger'; const log = Logger.create('Thumbnail'); +const THUMBNAIL_MAX_WIDTH = 320; +const THUMBNAIL_MAX_HEIGHT = 240; +const THUMBNAIL_QUALITY = 0.72; + /** * Create thumbnail for video or image. */ @@ -15,7 +19,7 @@ export async function createThumbnail( ): Promise { return new Promise((resolve) => { if (type === 'image') { - resolve(URL.createObjectURL(file)); + void createImageThumbnail(file).then(resolve); return; } @@ -25,6 +29,7 @@ export async function createThumbnail( video.src = url; video.muted = true; video.playsInline = true; + video.preload = 'metadata'; const timeout = setTimeout(() => { log.warn('Timeout:', file.name); @@ -38,17 +43,26 @@ export async function createThumbnail( }; video.onloadedmetadata = () => { - video.currentTime = Math.min(1, video.duration * 0.1); + const targetTime = Number.isFinite(video.duration) && video.duration > 0 + ? Math.min(1, video.duration * 0.1) + : 0; + try { + video.currentTime = targetTime; + } catch { + cleanup(); + resolve(undefined); + } }; video.onseeked = () => { const canvas = document.createElement('canvas'); - canvas.width = 160; - canvas.height = 90; + const size = getThumbnailCanvasSize(video.videoWidth || 16, video.videoHeight || 9); + canvas.width = size.width; + canvas.height = size.height; const ctx = canvas.getContext('2d'); if (ctx) { ctx.drawImage(video, 0, 0, canvas.width, canvas.height); - resolve(canvas.toDataURL('image/jpeg', 0.7)); + resolve(canvasToThumbnailDataUrl(canvas)); } else { resolve(undefined); } @@ -67,6 +81,86 @@ export async function createThumbnail( }); } +async function createImageThumbnail(file: File): Promise { + return new Promise((resolve) => { + const image = new Image(); + const url = URL.createObjectURL(file); + + const timeout = setTimeout(() => { + log.warn('Image thumbnail timeout:', file.name); + cleanup(); + resolve(undefined); + }, THUMBNAIL_TIMEOUT); + + const cleanup = () => { + clearTimeout(timeout); + URL.revokeObjectURL(url); + image.onload = null; + image.onerror = null; + }; + + image.onload = () => { + const width = image.naturalWidth || image.width; + const height = image.naturalHeight || image.height; + if (!width || !height) { + cleanup(); + resolve(undefined); + return; + } + + const size = getThumbnailCanvasSize(width, height); + const canvas = document.createElement('canvas'); + canvas.width = size.width; + canvas.height = size.height; + const ctx = canvas.getContext('2d'); + + if (!ctx) { + cleanup(); + resolve(undefined); + return; + } + + ctx.drawImage(image, 0, 0, canvas.width, canvas.height); + const thumbnail = canvasToThumbnailDataUrl(canvas); + cleanup(); + resolve(thumbnail); + }; + + image.onerror = () => { + cleanup(); + resolve(undefined); + }; + + image.decoding = 'async'; + image.src = url; + }); +} + +function getThumbnailCanvasSize(sourceWidth: number, sourceHeight: number): { width: number; height: number } { + if (!Number.isFinite(sourceWidth) || !Number.isFinite(sourceHeight) || sourceWidth <= 0 || sourceHeight <= 0) { + return { width: THUMBNAIL_MAX_WIDTH, height: Math.round(THUMBNAIL_MAX_WIDTH * 9 / 16) }; + } + + const scale = Math.min( + THUMBNAIL_MAX_WIDTH / sourceWidth, + THUMBNAIL_MAX_HEIGHT / sourceHeight, + 1, + ); + + return { + width: Math.max(1, Math.round(sourceWidth * scale)), + height: Math.max(1, Math.round(sourceHeight * scale)), + }; +} + +function canvasToThumbnailDataUrl(canvas: HTMLCanvasElement): string { + const webp = canvas.toDataURL('image/webp', THUMBNAIL_QUALITY); + if (webp.startsWith('data:image/webp')) { + return webp; + } + return canvas.toDataURL('image/jpeg', THUMBNAIL_QUALITY); +} + /** * Handle thumbnail deduplication - check for existing, save new. * UNIFIED: Replaces 3 duplicate blocks in original code. diff --git a/src/stores/mediaStore/slices/fileManageSlice.ts b/src/stores/mediaStore/slices/fileManageSlice.ts index aa52a4cd..9e1d25b4 100644 --- a/src/stores/mediaStore/slices/fileManageSlice.ts +++ b/src/stores/mediaStore/slices/fileManageSlice.ts @@ -60,7 +60,7 @@ export const createFileManageSlice: MediaSliceCreator = (set, if (refreshThumbnail) { if (mediaFile.type === 'image') { - thumbnailUrl = URL.createObjectURL(mediaFile.file); + thumbnailUrl = await createThumbnail(mediaFile.file, 'image'); } else if (mediaFile.type === 'video') { thumbnailUrl = await createThumbnail(mediaFile.file, 'video'); } diff --git a/src/stores/timeline/clipboardSlice.ts b/src/stores/timeline/clipboardSlice.ts index 2e265e9b..ebda123b 100644 --- a/src/stores/timeline/clipboardSlice.ts +++ b/src/stores/timeline/clipboardSlice.ts @@ -90,6 +90,7 @@ export const createClipboardSlice: SliceCreator = (set, get) = mathScene: clip.source?.type === 'math-scene' && clip.mathScene ? structuredClone(clip.mathScene) : undefined, + motion: clip.motion ? structuredClone(clip.motion) : undefined, // Visual data - reuse existing thumbnails and waveforms thumbnails: clip.thumbnails ? [...clip.thumbnails] : undefined, waveform: clip.waveform ? [...clip.waveform] : undefined, @@ -138,6 +139,10 @@ export const createClipboardSlice: SliceCreator = (set, get) = const isPrimitiveMeshClip = clipData.sourceType === 'model' && !!clipData.meshType; const isCameraClip = clipData.sourceType === 'camera'; const isSplatEffectorClip = clipData.sourceType === 'splat-effector'; + const isMotionClip = + clipData.sourceType === 'motion-shape' || + clipData.sourceType === 'motion-null' || + clipData.sourceType === 'motion-adjustment'; const requiresAsyncMediaLoad = !clipData.isComposition && clipData.sourceType !== 'text' && @@ -145,7 +150,8 @@ export const createClipboardSlice: SliceCreator = (set, get) = clipData.sourceType !== 'math-scene' && !isPrimitiveMeshClip && !isCameraClip && - !isSplatEffectorClip; + !isSplatEffectorClip && + !isMotionClip; // Find a track of the same type as the original let targetTrackId = clipData.trackId; @@ -215,6 +221,10 @@ export const createClipboardSlice: SliceCreator = (set, get) = mathSceneRenderer.render(clipData.mathScene!, canvas, 0, clipData.duration); return canvas; })(), + } : isMotionClip && clipData.motion ? { + type: clipData.sourceType, + mediaFileId: clipData.mediaFileId, + naturalDuration: clipData.naturalDuration ?? clipData.duration, } : clipData.sourceType === 'lottie' && clipData.mediaFileId ? { type: 'lottie' as const, mediaFileId: clipData.mediaFileId, @@ -254,6 +264,7 @@ export const createClipboardSlice: SliceCreator = (set, get) = text3DProperties, solidColor: clipData.solidColor, mathScene: clipData.mathScene ? structuredClone(clipData.mathScene) : undefined, + motion: clipData.motion ? structuredClone(clipData.motion) : undefined, // Reuse existing thumbnails and waveforms from copied clip thumbnails: clipData.thumbnails ? [...clipData.thumbnails] : undefined, waveform: clipData.waveform ? [...clipData.waveform] : undefined, @@ -346,6 +357,14 @@ export const createClipboardSlice: SliceCreator = (set, get) = continue; } + if ( + newClip.source?.type === 'motion-shape' || + newClip.source?.type === 'motion-null' || + newClip.source?.type === 'motion-adjustment' + ) { + continue; + } + // Handle solid clips - regenerate canvas if (newClip.source?.type === 'solid') { const originalClipData = clipboardData.find(cd => idMapping.get(cd.id) === newClip.id); diff --git a/src/stores/timeline/helpers/idGenerator.ts b/src/stores/timeline/helpers/idGenerator.ts index cb4b9c0f..536d35d3 100644 --- a/src/stores/timeline/helpers/idGenerator.ts +++ b/src/stores/timeline/helpers/idGenerator.ts @@ -63,6 +63,13 @@ export function generateMathSceneClipId(): string { return generateClipId('clip-math'); } +/** + * Generate a unique ID for motion design clips. + */ +export function generateMotionClipId(kind: 'shape' | 'null' | 'adjustment' | 'group' = 'shape'): string { + return generateClipId(`clip-motion-${kind}`); +} + /** * Generate a unique ID for mesh clips. */ diff --git a/src/stores/timeline/index.ts b/src/stores/timeline/index.ts index 7aed560b..0f0e62e3 100644 --- a/src/stores/timeline/index.ts +++ b/src/stores/timeline/index.ts @@ -11,6 +11,7 @@ import { createClipSlice } from './clipSlice'; import { createTextClipSlice } from './textClipSlice'; import { createSolidClipSlice } from './solidClipSlice'; import { createMathSceneClipSlice } from './mathSceneClipSlice'; +import { createMotionClipSlice } from './motionClipSlice'; import { createMeshClipSlice } from './meshClipSlice'; import { createCameraClipSlice } from './cameraClipSlice'; import { createSplatEffectorClipSlice } from './splatEffectorClipSlice'; @@ -50,6 +51,7 @@ export const useTimelineStore = create()( const textClipActions = createTextClipSlice(set, get); const solidClipActions = createSolidClipSlice(set, get); const mathSceneClipActions = createMathSceneClipSlice(set, get); + const motionClipActions = createMotionClipSlice(set, get); const meshClipActions = createMeshClipSlice(set, get); const cameraClipActions = createCameraClipSlice(set, get); const splatEffectorClipActions = createSplatEffectorClipSlice(set, get); @@ -265,6 +267,7 @@ export const useTimelineStore = create()( ...textClipActions, ...solidClipActions, ...mathSceneClipActions, + ...motionClipActions, ...meshClipActions, ...cameraClipActions, ...splatEffectorClipActions, diff --git a/src/stores/timeline/keyframeSlice.ts b/src/stores/timeline/keyframeSlice.ts index 833f08d0..e420bca9 100644 --- a/src/stores/timeline/keyframeSlice.ts +++ b/src/stores/timeline/keyframeSlice.ts @@ -33,6 +33,8 @@ import { vectorAnimationInputValueToNumber, type VectorAnimationClipSettings, } from '../../types/vectorAnimation'; +import { isMotionProperty } from '../../types/motionDesign'; +import { propertyRegistry } from '../../services/properties'; import { normalizeEasingType } from '../../utils/easing'; import { composeTransforms } from '../../utils/transformComposition'; import { calculateSourceTime, getSpeedAtTime, calculateTimelineDuration } from '../../utils/speedIntegration'; @@ -1034,6 +1036,18 @@ export const createKeyframeSlice: SliceCreator = (set, get) => return; } + if (isMotionProperty(property)) { + const descriptor = propertyRegistry.getDescriptor(property, clip); + if (descriptor?.write) { + const nextClip = propertyRegistry.writeValue(clip, property, value); + set({ + clips: clips.map(c => c.id === clipId ? nextClip : c), + }); + get().invalidateCache(); + } + return; + } + // Handle speed property (directly on clip, not transform) if (property === 'speed') { const { invalidateCache, updateDuration } = get(); @@ -1285,6 +1299,14 @@ export const createKeyframeSlice: SliceCreator = (set, get) => colorProperty.paramName, currentValue ); + } else if (isMotionProperty(property)) { + const descriptor = propertyRegistry.getDescriptor(property, clip); + if (descriptor?.write) { + const nextClip = propertyRegistry.writeValue(clip, property, currentValue); + set({ + clips: get().clips.map(c => c.id === clipId ? nextClip : c), + }); + } } else if (property === 'speed') { const { updateDuration } = get(); const sourceDuration = clip.outPoint - clip.inPoint; diff --git a/src/stores/timeline/motionClipSlice.ts b/src/stores/timeline/motionClipSlice.ts new file mode 100644 index 00000000..8ad8b17c --- /dev/null +++ b/src/stores/timeline/motionClipSlice.ts @@ -0,0 +1,214 @@ +import type { TimelineClip } from '../../types'; +import type { MotionClipActions, SliceCreator } from './types'; +import { createDefaultMotionLayerDefinition } from '../../types/motionDesign'; +import { DEFAULT_TRANSFORM } from './constants'; +import { generateMotionClipId } from './helpers/idGenerator'; +import { engine } from '../../engine/WebGPUEngine'; +import { layerBuilder } from '../../services/layerBuilder'; +import { Logger } from '../../services/logger'; + +const log = Logger.create('MotionClipSlice'); + +function colorFromHex(hex: string | undefined): { r: number; g: number; b: number; a: number } { + const fallback = { r: 1, g: 1, b: 1, a: 1 }; + if (!hex) return fallback; + + const normalized = hex.trim().replace('#', ''); + const value = normalized.length === 3 + ? normalized.split('').map((part) => part + part).join('') + : normalized.slice(0, 6); + if (!/^[0-9a-fA-F]{6}$/.test(value)) { + return fallback; + } + + return { + r: parseInt(value.slice(0, 2), 16) / 255, + g: parseInt(value.slice(2, 4), 16) / 255, + b: parseInt(value.slice(4, 6), 16) / 255, + a: 1, + }; +} + +export const createMotionClipSlice: SliceCreator = (set, get) => ({ + addMotionShapeClip: (trackId, startTime, options = {}) => { + const { clips, tracks, updateDuration, invalidateCache } = get(); + const track = tracks.find((candidate) => candidate.id === trackId); + + if (!track || track.type !== 'video') { + log.warn('Motion shape clips can only be added to video tracks'); + return null; + } + + const duration = options.duration ?? 5; + const motion = createDefaultMotionLayerDefinition('shape', { + primitive: options.primitive, + size: options.size, + fillColor: options.fillColor, + }); + const clipId = generateMotionClipId('shape'); + const shapeClip: TimelineClip = { + id: clipId, + trackId, + name: options.name ?? 'Motion Shape', + file: new File([JSON.stringify(motion)], 'motion-shape.msmotion', { type: 'application/json' }), + startTime, + duration, + inPoint: 0, + outPoint: duration, + source: { + type: 'motion-shape', + naturalDuration: duration, + }, + motion, + transform: { ...DEFAULT_TRANSFORM }, + effects: [], + isLoading: false, + }; + + set({ clips: [...clips, shapeClip] }); + updateDuration(); + invalidateCache(); + layerBuilder.invalidateCache(); + engine.requestRender(); + + log.debug('Created motion shape clip', { clipId, primitive: motion.shape?.primitive }); + return clipId; + }, + + addMotionNullClip: (trackId, startTime, duration = 5) => { + const { clips, tracks, updateDuration, invalidateCache } = get(); + const track = tracks.find((candidate) => candidate.id === trackId); + + if (!track || track.type !== 'video') { + log.warn('Motion null clips can only be added to video tracks'); + return null; + } + + const motion = createDefaultMotionLayerDefinition('null'); + const clipId = generateMotionClipId('null'); + const nullClip: TimelineClip = { + id: clipId, + trackId, + name: 'Null', + file: new File([JSON.stringify(motion)], 'motion-null.msmotion', { type: 'application/json' }), + startTime, + duration, + inPoint: 0, + outPoint: duration, + source: { + type: 'motion-null', + naturalDuration: duration, + }, + motion, + transform: { ...DEFAULT_TRANSFORM }, + effects: [], + isLoading: false, + }; + + set({ clips: [...clips, nullClip] }); + updateDuration(); + invalidateCache(); + layerBuilder.invalidateCache(); + engine.requestRender(); + + log.debug('Created motion null clip', { clipId }); + return clipId; + }, + + addMotionAdjustmentClip: (trackId, startTime, duration = 5) => { + const { clips, tracks, updateDuration, invalidateCache } = get(); + const track = tracks.find((candidate) => candidate.id === trackId); + + if (!track || track.type !== 'video') { + log.warn('Motion adjustment clips can only be added to video tracks'); + return null; + } + + const motion = createDefaultMotionLayerDefinition('adjustment'); + const clipId = generateMotionClipId('adjustment'); + const adjustmentClip: TimelineClip = { + id: clipId, + trackId, + name: 'Adjustment', + file: new File([JSON.stringify(motion)], 'motion-adjustment.msmotion', { type: 'application/json' }), + startTime, + duration, + inPoint: 0, + outPoint: duration, + source: { + type: 'motion-adjustment', + naturalDuration: duration, + }, + motion, + transform: { ...DEFAULT_TRANSFORM }, + effects: [], + isLoading: false, + }; + + set({ clips: [...clips, adjustmentClip] }); + updateDuration(); + invalidateCache(); + layerBuilder.invalidateCache(); + engine.requestRender(); + + log.debug('Created motion adjustment clip', { clipId }); + return clipId; + }, + + convertSolidToMotionShape: (clipId) => { + const { clips, invalidateCache } = get(); + const clip = clips.find((candidate) => candidate.id === clipId); + + if (!clip || clip.source?.type !== 'solid') { + log.warn('Only solid clips can be converted to motion shapes', { clipId }); + return null; + } + + const motion = createDefaultMotionLayerDefinition('shape', { + primitive: 'rectangle', + fillColor: colorFromHex(clip.solidColor), + }); + const convertedClip: TimelineClip = { + ...clip, + name: clip.name || 'Motion Shape', + file: new File([JSON.stringify(motion)], 'motion-shape.msmotion', { type: 'application/json' }), + source: { + ...(clip.source ?? {}), + type: 'motion-shape', + textCanvas: undefined, + naturalDuration: clip.duration, + }, + motion, + solidColor: undefined, + isLoading: false, + }; + + set({ + clips: clips.map((candidate) => candidate.id === clipId ? convertedClip : candidate), + }); + invalidateCache(); + layerBuilder.invalidateCache(); + engine.requestRender(); + + log.debug('Converted solid clip to motion shape', { clipId }); + return clipId; + }, + + updateMotionLayer: (clipId, updater) => { + const { clips, invalidateCache } = get(); + const clip = clips.find((candidate) => candidate.id === clipId); + if (!clip?.motion) return; + + set({ + clips: clips.map((candidate) => { + if (candidate.id !== clipId || !candidate.motion) { + return candidate; + } + return { ...candidate, motion: updater(structuredClone(candidate.motion)) }; + }), + }); + invalidateCache(); + layerBuilder.invalidateCache(); + engine.requestRender(); + }, +}); diff --git a/src/stores/timeline/serializationUtils.ts b/src/stores/timeline/serializationUtils.ts index 590e853c..b502ef88 100644 --- a/src/stores/timeline/serializationUtils.ts +++ b/src/stores/timeline/serializationUtils.ts @@ -147,6 +147,7 @@ export const createSerializationUtils: SliceCreator = (set, mathScene: clip.source?.type === 'math-scene' && clip.mathScene ? structuredClone(clip.mathScene) : undefined, + motion: clip.motion ? structuredClone(clip.motion) : undefined, // Clip label color // 3D layer support is3D: clip.is3D || undefined, @@ -358,6 +359,44 @@ export const createSerializationUtils: SliceCreator = (set, isLoading: false, }; }; + const isMotionSourceType = (sourceType: SerializableClip['sourceType']): sourceType is 'motion-shape' | 'motion-null' | 'motion-adjustment' => ( + sourceType === 'motion-shape' || + sourceType === 'motion-null' || + sourceType === 'motion-adjustment' + ); + const createMotionClip = (serializedClip: SerializableClip, clipId = serializedClip.id): TimelineClip | null => { + if (!isMotionSourceType(serializedClip.sourceType) || !serializedClip.motion) { + return null; + } + + return { + id: clipId, + trackId: serializedClip.trackId, + name: serializedClip.name || 'Motion', + file: new File([JSON.stringify(serializedClip.motion)], `${serializedClip.sourceType}.msmotion`, { type: 'application/json' }), + mediaFileId: serializedClip.mediaFileId || undefined, + startTime: serializedClip.startTime, + duration: serializedClip.duration, + inPoint: serializedClip.inPoint, + outPoint: serializedClip.outPoint, + source: { + type: serializedClip.sourceType, + mediaFileId: serializedClip.mediaFileId || undefined, + naturalDuration: serializedClip.duration, + }, + motion: structuredClone(serializedClip.motion), + thumbnails: serializedClip.thumbnails, + linkedClipId: serializedClip.linkedClipId, + linkedGroupId: serializedClip.linkedGroupId, + transform: serializedClip.transform, + effects: serializedClip.effects || [], + colorCorrection: serializedClip.colorCorrection ? structuredClone(serializedClip.colorCorrection) : undefined, + masks: serializedClip.masks, + speed: serializedClip.speed, + preservesPitch: serializedClip.preservesPitch, + isLoading: false, + }; + }; for (const serializedClip of data.clips) { // Handle composition clips specially @@ -550,6 +589,12 @@ export const createSerializationUtils: SliceCreator = (set, continue; } // Regular media clip at this sub-level + const motionClip = createMotionClip(nsc, `nested-${parentClipId}-${nsc.id}`); + if (motionClip) { + result.push(motionClip); + continue; + } + if (nsc.sourceType === 'model' && nsc.meshType) { result.push(createPrimitiveMeshClip(nsc, `nested-${parentClipId}-${nsc.id}`)); continue; @@ -1080,6 +1125,16 @@ export const createSerializationUtils: SliceCreator = (set, continue; } + const motionClip = createMotionClip(serializedClip); + if (motionClip) { + set(state => ({ + clips: [...state.clips, motionClip], + })); + + log.debug('Restored motion clip', { clip: serializedClip.name, sourceType: serializedClip.sourceType }); + continue; + } + // Math Scene clips - restore from serializable scene definition if (serializedClip.sourceType === 'math-scene' && serializedClip.mathScene) { const { mathSceneRenderer } = await import('../../services/mathScene/MathSceneRenderer'); diff --git a/src/stores/timeline/types.ts b/src/stores/timeline/types.ts index 441c2184..48e75e0e 100644 --- a/src/stores/timeline/types.ts +++ b/src/stores/timeline/types.ts @@ -26,6 +26,7 @@ import type { Layer, SerializableClip, } from '../../types'; +import type { MotionColor, MotionLayerDefinition, ShapePrimitive } from '../../types/motionDesign'; import type { Composition } from '../mediaStore'; import type { VectorAnimationClipSettings } from '../../types/vectorAnimation'; import type { MarkerMIDIBinding } from '../../types/midi'; @@ -234,6 +235,22 @@ export interface MathSceneClipActions { updateMathParameter: (clipId: string, parameterId: string, patch: Partial) => void; } +export interface MotionShapeClipOptions { + primitive?: ShapePrimitive; + size?: { w: number; h: number }; + fillColor?: MotionColor; + duration?: number; + name?: string; +} + +export interface MotionClipActions { + addMotionShapeClip: (trackId: string, startTime: number, options?: MotionShapeClipOptions) => string | null; + addMotionNullClip: (trackId: string, startTime: number, duration?: number) => string | null; + addMotionAdjustmentClip: (trackId: string, startTime: number, duration?: number) => string | null; + convertSolidToMotionShape: (clipId: string) => string | null; + updateMotionLayer: (clipId: string, updater: (motion: MotionLayerDefinition) => MotionLayerDefinition) => void; +} + // Mesh clip actions (extracted to meshClipSlice) export interface MeshClipActions { addMeshClip: (trackId: string, startTime: number, meshType: import('../mediaStore/types').MeshPrimitiveType, duration?: number, skipMediaItem?: boolean) => string | null; @@ -321,7 +338,7 @@ export interface CoreClipActions { } // Combined ClipActions = all sub-interfaces -export type ClipActions = CoreClipActions & TextClipActions & SolidClipActions & MathSceneClipActions & MeshClipActions & CameraClipActions & SplatEffectorClipActions & ClipEffectActions & ColorCorrectionActions & LinkedGroupActions & DownloadClipActions; +export type ClipActions = CoreClipActions & TextClipActions & SolidClipActions & MathSceneClipActions & MotionClipActions & MeshClipActions & CameraClipActions & SplatEffectorClipActions & ClipEffectActions & ColorCorrectionActions & LinkedGroupActions & DownloadClipActions; // Playback actions interface export interface PlaybackActions { @@ -492,6 +509,7 @@ export interface ClipboardClipData { text3DProperties?: import('../../types').Text3DProperties; solidColor?: string; mathScene?: MathSceneDefinition; + motion?: MotionLayerDefinition; vectorAnimationSettings?: VectorAnimationClipSettings; cameraSettings?: import('../mediaStore/types').SceneCameraSettings; meshType?: import('../mediaStore/types').MeshPrimitiveType; diff --git a/src/types/index.ts b/src/types/index.ts index 365f162f..39c7a280 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -7,8 +7,10 @@ import type { VectorAnimationStateProperty, } from './vectorAnimation'; import type { ColorCorrectionState, RuntimeColorGrade } from './colorCorrection'; +import type { MotionLayerDefinition, MotionProperty } from './motionDesign'; export * from './colorCorrection'; +export * from './motionDesign'; export type TimelineSourceType = | 'video' @@ -22,6 +24,9 @@ export type TimelineSourceType = | 'gaussian-splat' | 'splat-effector' | 'math-scene' + | 'motion-shape' + | 'motion-null' + | 'motion-adjustment' | VectorAnimationProvider; export type ModelSequencePlaybackMode = 'clamp' | 'loop'; @@ -145,7 +150,7 @@ export type BlendMode = | 'alpha-add'; export interface LayerSource { - type: 'video' | 'image' | 'camera' | 'color' | 'text' | 'solid' | 'model' | 'gaussian-avatar' | 'gaussian-splat'; + type: 'video' | 'image' | 'camera' | 'color' | 'text' | 'solid' | 'model' | 'gaussian-avatar' | 'gaussian-splat' | 'motion'; modelUrl?: string; // Blob URL to 3D model file (OBJ/glTF/GLB) modelFileName?: string; modelSequence?: ModelSequenceData; @@ -192,6 +197,8 @@ export interface LayerSource { textCanvas?: HTMLCanvasElement; textProperties?: TextClipProperties; text3DProperties?: Text3DProperties; + // Motion design support + motion?: MotionLayerDefinition; } // Data for pre-rendering nested compositions @@ -652,6 +659,7 @@ export interface TimelineClip { runtimeSessionKey?: string; } | null; mathScene?: MathSceneDefinition; + motion?: MotionLayerDefinition; thumbnails?: string[]; // Array of data URLs for filmstrip preview mediaFileId?: string; // Reference to MediaFile for audio/proxy lookup (top-level for YouTube downloads) linkedClipId?: string; // ID of linked clip (e.g., audio linked to video) @@ -786,6 +794,7 @@ export interface SerializableClip { solidColor?: string; vectorAnimationSettings?: VectorAnimationClipSettings; mathScene?: MathSceneDefinition; + motion?: MotionLayerDefinition; // Transition support transitionIn?: TimelineTransition; transitionOut?: TimelineTransition; @@ -865,7 +874,7 @@ export type MaskNumericProperty = `mask.${string}.${MaskNumericPropertyName}`; export type MaskProperty = MaskPathProperty | MaskNumericProperty; // Combined animatable property type -export type AnimatableProperty = TransformProperty | CameraProperty | EffectProperty | ColorProperty | MaskProperty | VectorAnimationInputProperty | VectorAnimationStateProperty; +export type AnimatableProperty = TransformProperty | CameraProperty | EffectProperty | ColorProperty | MaskProperty | VectorAnimationInputProperty | VectorAnimationStateProperty | MotionProperty; export function isCameraProperty(property: string): property is CameraProperty { return /^camera\.(fov|near|far|resolutionWidth|resolutionHeight)$/.test(property); diff --git a/src/types/motionDesign.ts b/src/types/motionDesign.ts new file mode 100644 index 00000000..dcbd8b7f --- /dev/null +++ b/src/types/motionDesign.ts @@ -0,0 +1,324 @@ +import type { BlendMode } from './index'; + +export type MotionLayerKind = 'shape' | 'null' | 'adjustment' | 'group'; +export type ShapePrimitive = 'rectangle' | 'ellipse' | 'polygon' | 'star'; + +export interface MotionColor { + r: number; + g: number; + b: number; + a: number; +} + +export interface MotionVector2 { + x: number; + y: number; +} + +export interface MotionLayerDefinition { + version: 1; + kind: MotionLayerKind; + shape?: ShapeDefinition; + appearance?: AppearanceStack; + replicator?: ReplicatorDefinition; + ui?: MotionLayerUiState; +} + +export interface MotionLayerUiState { + labelColor?: string; + locked?: boolean; + pinnedProperties?: string[]; + propertiesSearch?: string; +} + +export interface ShapeDefinition { + primitive: ShapePrimitive; + size: { w: number; h: number }; + cornerRadius?: number; + polygon?: { + points: number; + radius: number; + cornerRadius: number; + }; + star?: { + points: number; + outerRadius: number; + innerRadius: number; + cornerRadius: number; + }; +} + +export type AppearanceKind = 'color-fill' | 'stroke' | 'linear-gradient' | 'radial-gradient' | 'texture-fill'; + +export interface AppearanceItemBase { + id: string; + kind: AppearanceKind; + name: string; + visible: boolean; + opacity: number; + blendMode?: BlendMode; +} + +export interface ColorFillAppearance extends AppearanceItemBase { + kind: 'color-fill'; + color: MotionColor; +} + +export interface StrokeAppearance extends AppearanceItemBase { + kind: 'stroke'; + color: MotionColor; + width: number; + alignment: 'center' | 'inside' | 'outside'; +} + +export interface GradientStop { + id: string; + offset: number; + color: MotionColor; +} + +export interface LinearGradientAppearance extends AppearanceItemBase { + kind: 'linear-gradient'; + stops: GradientStop[]; + start: MotionVector2; + end: MotionVector2; +} + +export interface RadialGradientAppearance extends AppearanceItemBase { + kind: 'radial-gradient'; + stops: GradientStop[]; + center: MotionVector2; + radius: number; +} + +export interface TextureFillAppearance extends AppearanceItemBase { + kind: 'texture-fill'; + mediaFileId?: string; + fit: 'contain' | 'cover' | 'fill' | 'stretch' | 'tile'; + transform: { + position: MotionVector2; + scale: MotionVector2; + rotation: number; + }; + time?: number; +} + +export type AppearanceItem = + | ColorFillAppearance + | StrokeAppearance + | LinearGradientAppearance + | RadialGradientAppearance + | TextureFillAppearance; + +export interface AppearanceStack { + version: 1; + items: AppearanceItem[]; + selectedItemId?: string; +} + +export interface ReplicatorDefinition { + enabled: boolean; + layout: ReplicatorLayout; + offset: ReplicatorOffset; + distribution?: ReplicatorDistribution; + modifiers: ReplicatorModifier[]; + falloff?: ReplicatorFalloff; + maxInstances?: number; +} + +export type ReplicatorLayout = + | { + mode: 'grid'; + count: { x: number; y: number }; + spacing: MotionVector2; + patternOffset?: MotionVector2; + } + | { + mode: 'linear'; + count: number; + spacing: number; + direction: MotionVector2; + } + | { + mode: 'radial'; + count: number; + radius: number; + startAngle: number; + endAngle: number; + autoOrient: boolean; + }; + +export interface ReplicatorOffset { + position: MotionVector2; + rotation: number; + scale: MotionVector2; + opacity: number; + mode: 'cumulative' | 'absolute'; +} + +export interface ReplicatorDistribution { + seed?: number; + randomizeOrder?: boolean; +} + +export interface ReplicatorModifier { + id: string; + kind: 'random' | 'noise' | 'oscillator' | 'field'; + enabled: boolean; + seed?: number; + targetProperties: string[]; + params: Record; +} + +export interface ReplicatorFalloff { + shapeClipId: string; + feather: number; + invert: boolean; + clip: boolean; +} + +export type MotionShapeProperty = + | 'shape.size.w' + | 'shape.size.h' + | 'shape.cornerRadius'; + +export type MotionAppearanceProperty = + | `appearance.${string}.opacity` + | `appearance.${string}.color.r` + | `appearance.${string}.color.g` + | `appearance.${string}.color.b` + | `appearance.${string}.color.a` + | `appearance.${string}.stroke.width` + | `appearance.${string}.stroke.alignment`; + +export type MotionReplicatorProperty = + | 'replicator.enabled' + | 'replicator.layout.mode' + | 'replicator.count.x' + | 'replicator.count.y' + | 'replicator.spacing.x' + | 'replicator.spacing.y' + | 'replicator.offset.position.x' + | 'replicator.offset.position.y' + | 'replicator.offset.rotation' + | 'replicator.offset.scale.x' + | 'replicator.offset.scale.y' + | 'replicator.offset.opacity'; + +export type MotionProperty = MotionShapeProperty | MotionAppearanceProperty | MotionReplicatorProperty; + +export const DEFAULT_MOTION_COLOR: MotionColor = { r: 1, g: 1, b: 1, a: 1 }; +export const DEFAULT_MOTION_SHAPE_SIZE = { w: 320, h: 180 }; + +export function createMotionAppearanceId(prefix = 'appearance'): string { + return `${prefix}_${Date.now()}_${Math.random().toString(36).slice(2, 7)}`; +} + +export function createColorFillAppearance( + color: MotionColor = DEFAULT_MOTION_COLOR, + id = createMotionAppearanceId('fill'), +): ColorFillAppearance { + return { + id, + kind: 'color-fill', + name: 'Fill', + visible: true, + opacity: 1, + color: { ...color }, + }; +} + +export function createStrokeAppearance( + color: MotionColor = { r: 0, g: 0, b: 0, a: 1 }, + id = createMotionAppearanceId('stroke'), +): StrokeAppearance { + return { + id, + kind: 'stroke', + name: 'Stroke', + visible: false, + opacity: 1, + color: { ...color }, + width: 4, + alignment: 'center', + }; +} + +export function createDefaultAppearanceStack(fillColor?: MotionColor): AppearanceStack { + const fill = createColorFillAppearance(fillColor); + return { + version: 1, + items: [fill], + selectedItemId: fill.id, + }; +} + +export function createDefaultShapeDefinition( + primitive: ShapePrimitive = 'rectangle', + size = DEFAULT_MOTION_SHAPE_SIZE, +): ShapeDefinition { + return { + primitive, + size: { ...size }, + cornerRadius: primitive === 'rectangle' ? 0 : undefined, + polygon: { points: 6, radius: Math.min(size.w, size.h) / 2, cornerRadius: 0 }, + star: { points: 5, outerRadius: Math.min(size.w, size.h) / 2, innerRadius: Math.min(size.w, size.h) / 4, cornerRadius: 0 }, + }; +} + +export function createDefaultReplicatorDefinition(): ReplicatorDefinition { + return { + enabled: false, + layout: { + mode: 'grid', + count: { x: 3, y: 3 }, + spacing: { x: 120, y: 120 }, + patternOffset: { x: 0, y: 0 }, + }, + offset: { + position: { x: 0, y: 0 }, + rotation: 0, + scale: { x: 1, y: 1 }, + opacity: 1, + mode: 'cumulative', + }, + modifiers: [], + maxInstances: 10000, + }; +} + +export function createDefaultMotionLayerDefinition( + kind: MotionLayerKind, + options: { + primitive?: ShapePrimitive; + size?: { w: number; h: number }; + fillColor?: MotionColor; + } = {}, +): MotionLayerDefinition { + if (kind === 'shape') { + return { + version: 1, + kind, + shape: createDefaultShapeDefinition(options.primitive, options.size), + appearance: createDefaultAppearanceStack(options.fillColor), + replicator: createDefaultReplicatorDefinition(), + ui: {}, + }; + } + + return { + version: 1, + kind, + ui: {}, + }; +} + +export function isMotionProperty(property: string): property is MotionProperty { + return ( + property === 'shape.size.w' || + property === 'shape.size.h' || + property === 'shape.cornerRadius' || + /^appearance\.[^.]+\.(opacity|color\.(r|g|b|a)|stroke\.(width|alignment))$/.test(property) || + /^replicator\.(enabled|layout\.mode|count\.(x|y)|spacing\.(x|y)|offset\.(position\.(x|y)|rotation|scale\.(x|y)|opacity))$/.test(property) + ); +} diff --git a/src/types/propertyRegistry.ts b/src/types/propertyRegistry.ts new file mode 100644 index 00000000..7d229711 --- /dev/null +++ b/src/types/propertyRegistry.ts @@ -0,0 +1,46 @@ +import type { TimelineClip } from './index'; + +export type PropertyValueType = + | 'number' + | 'boolean' + | 'color' + | 'enum' + | 'vector2' + | 'gradient' + | 'path'; + +export type PropertyValue = unknown; + +export interface PropertyDescriptor { + path: string; + label: string; + group: string; + valueType: PropertyValueType; + animatable: boolean; + defaultValue: T; + ui?: { + min?: number; + max?: number; + step?: number; + unit?: string; + aliases?: string[]; + compact?: boolean; + options?: Array<{ value: string | number | boolean; label: string }>; + }; + read?: (clip: TimelineClip, path: string) => T | undefined; + write?: (clip: TimelineClip, value: PropertyValue, path: string) => TimelineClip; +} + +export interface PropertySearchOptions { + clip?: TimelineClip; + query?: string; + group?: string; + animatable?: boolean; +} + +export type PropertyDescriptorResolver = ( + path: string, + clip?: TimelineClip, +) => PropertyDescriptor | undefined; + +export type PropertyDescriptorProvider = (clip: TimelineClip) => PropertyDescriptor[]; diff --git a/src/utils/motionInterpolation.ts b/src/utils/motionInterpolation.ts new file mode 100644 index 00000000..eb46db10 --- /dev/null +++ b/src/utils/motionInterpolation.ts @@ -0,0 +1,47 @@ +import type { Keyframe, TimelineClip } from '../types'; +import type { MotionLayerDefinition } from '../types/motionDesign'; +import { isMotionProperty } from '../types/motionDesign'; +import { propertyRegistry } from '../services/properties'; +import { interpolateKeyframes } from './keyframeInterpolation'; + +export function getInterpolatedMotionLayer( + clip: TimelineClip, + keyframes: Keyframe[], + clipLocalTime: number, +): MotionLayerDefinition | undefined { + if (!clip.motion) { + return undefined; + } + + const motionProperties = Array.from(new Set( + keyframes + .map((keyframe) => keyframe.property) + .filter(isMotionProperty), + )); + + if (motionProperties.length === 0) { + return clip.motion; + } + + let workingClip: TimelineClip = { + ...clip, + motion: structuredClone(clip.motion), + }; + + for (const property of motionProperties) { + const descriptor = propertyRegistry.getDescriptor(property, workingClip); + if (!descriptor?.animatable || descriptor.valueType !== 'number') { + continue; + } + + const defaultValue = propertyRegistry.readValue(workingClip, property); + if (typeof defaultValue !== 'number' || !Number.isFinite(defaultValue)) { + continue; + } + + const value = interpolateKeyframes(keyframes, property, clipLocalTime, defaultValue); + workingClip = propertyRegistry.writeValue(workingClip, property, value); + } + + return workingClip.motion; +} diff --git a/tests/helpers/storeFactory.ts b/tests/helpers/storeFactory.ts index c758bd0f..b596f709 100644 --- a/tests/helpers/storeFactory.ts +++ b/tests/helpers/storeFactory.ts @@ -19,6 +19,7 @@ import { createMaskSlice } from '../../src/stores/timeline/maskSlice'; import { createClipSlice } from '../../src/stores/timeline/clipSlice'; import { createTextClipSlice } from '../../src/stores/timeline/textClipSlice'; import { createSolidClipSlice } from '../../src/stores/timeline/solidClipSlice'; +import { createMotionClipSlice } from '../../src/stores/timeline/motionClipSlice'; import { createClipEffectSlice } from '../../src/stores/timeline/clipEffectSlice'; import { createColorCorrectionSlice } from '../../src/stores/timeline/colorCorrectionSlice'; import { createLinkedGroupSlice } from '../../src/stores/timeline/linkedGroupSlice'; @@ -101,6 +102,7 @@ export function createTestTimelineStore(overrides?: Partial) { const clipActions = createClipSlice(set, get); const textClipActions = createTextClipSlice(set, get); const solidClipActions = createSolidClipSlice(set, get); + const motionClipActions = createMotionClipSlice(set, get); const clipEffectActions = createClipEffectSlice(set, get); const colorCorrectionActions = createColorCorrectionSlice(set, get); const linkedGroupActions = createLinkedGroupSlice(set, get); @@ -265,6 +267,7 @@ export function createTestTimelineStore(overrides?: Partial) { ...clipActions, ...textClipActions, ...solidClipActions, + ...motionClipActions, ...clipEffectActions, ...colorCorrectionActions, ...linkedGroupActions, diff --git a/tests/stores/mediaStore/fileManageSlice.test.ts b/tests/stores/mediaStore/fileManageSlice.test.ts index 7f7ba51d..f47a7cd5 100644 --- a/tests/stores/mediaStore/fileManageSlice.test.ts +++ b/tests/stores/mediaStore/fileManageSlice.test.ts @@ -8,6 +8,15 @@ import { describe, it, expect, beforeEach, vi } from 'vitest'; import { create } from 'zustand'; import type { MediaState, MediaFile, MediaFolder, TextItem, SolidItem, Composition } from '../../../src/stores/mediaStore/types'; + +const thumbnailMocks = vi.hoisted(() => ({ + createThumbnail: vi.fn(async () => 'blob:http://localhost/refreshed-thumb'), +})); + +vi.mock('../../../src/stores/mediaStore/helpers/thumbnailHelpers', () => ({ + createThumbnail: thumbnailMocks.createThumbnail, +})); + import { createFileManageSlice, type FileManageActions } from '../../../src/stores/mediaStore/slices/fileManageSlice'; import { createFolderSlice, type FolderActions } from '../../../src/stores/mediaStore/slices/folderSlice'; import { createSelectionSlice, type SelectionActions } from '../../../src/stores/mediaStore/slices/selectionSlice'; @@ -215,6 +224,8 @@ describe('MediaStore - File Management', () => { beforeEach(() => { store = createTestStore(); + thumbnailMocks.createThumbnail.mockClear(); + thumbnailMocks.createThumbnail.mockResolvedValue('blob:http://localhost/refreshed-thumb'); }); // --- Adding media files --- @@ -871,8 +882,7 @@ describe('MediaStore - File Management', () => { describe('refreshFileUrls', () => { it('should recreate image blob urls from the current file object', async () => { const createObjectURL = vi.spyOn(URL, 'createObjectURL') - .mockReturnValueOnce('blob:http://localhost/refreshed-file') - .mockReturnValueOnce('blob:http://localhost/refreshed-thumb'); + .mockReturnValueOnce('blob:http://localhost/refreshed-file'); const revokeObjectURL = vi.spyOn(URL, 'revokeObjectURL').mockImplementation(() => {}); const file = makeMediaFile({ id: 'img-1', @@ -891,6 +901,7 @@ describe('MediaStore - File Management', () => { expect(result).toBe(true); expect(updated.url).toBe('blob:http://localhost/refreshed-file'); expect(updated.thumbnailUrl).toBe('blob:http://localhost/refreshed-thumb'); + expect(thumbnailMocks.createThumbnail).toHaveBeenCalledWith(file.file, 'image'); expect(revokeObjectURL).toHaveBeenCalledWith('blob:http://localhost/original-file'); expect(revokeObjectURL).toHaveBeenCalledWith('blob:http://localhost/original-thumb'); diff --git a/tests/unit/PreviewSourceMonitor.test.tsx b/tests/unit/PreviewSourceMonitor.test.tsx index e56d00a8..b01983cb 100644 --- a/tests/unit/PreviewSourceMonitor.test.tsx +++ b/tests/unit/PreviewSourceMonitor.test.tsx @@ -57,6 +57,7 @@ const engineState = { previewCameraOverride: null, activeGaussianSplatLoadProgress: null, setPreviewCameraOverride: vi.fn(), + setSceneGizmoVisible: vi.fn(), setSceneGizmoClipIdOverride: vi.fn(), setSceneNavFpsMoveSpeed: vi.fn(), }; diff --git a/tests/unit/TransformTab.test.tsx b/tests/unit/TransformTab.test.tsx index b873f168..81cde6b1 100644 --- a/tests/unit/TransformTab.test.tsx +++ b/tests/unit/TransformTab.test.tsx @@ -173,6 +173,26 @@ describe('TransformTab position units', () => { expect(mockState.setPropertyValue).toHaveBeenCalledWith('clip-1', 'position.x', 1); }); + it('accepts clip speed values up to 10000 percent', () => { + mockState.sourceType = 'video'; + const { container } = render( + , + ); + + const speedControl = Array.from(container.querySelectorAll('.draggable-number')) + .find((element) => element.textContent === '100%') as HTMLElement; + fireEvent.doubleClick(speedControl); + const input = container.querySelector('input.draggable-number-input') as HTMLInputElement; + fireEvent.change(input, { target: { value: '10000' } }); + fireEvent.keyDown(input, { key: 'Enter' }); + + expect(mockState.setPropertyValue).toHaveBeenCalledWith('clip-1', 'speed', 100); + }); + it('edits 3D video plane positions in scene units', () => { mockState.sourceType = 'video'; const { container } = render( diff --git a/tests/unit/layerBuilderService.test.ts b/tests/unit/layerBuilderService.test.ts index 8ecad1ff..0e5aa171 100644 --- a/tests/unit/layerBuilderService.test.ts +++ b/tests/unit/layerBuilderService.test.ts @@ -12,6 +12,7 @@ import { } from '../../src/services/mediaRuntime/runtimePlayback'; import { mediaRuntimeRegistry } from '../../src/services/mediaRuntime/registry'; import { scrubSettleState } from '../../src/services/scrubSettleState'; +import { proxyFrameCache } from '../../src/services/proxyFrameCache'; import { lottieRuntimeManager } from '../../src/services/vectorAnimation/LottieRuntimeManager'; import type { RuntimeFrameProvider } from '../../src/services/mediaRuntime/types'; import type { TimelineClip } from '../../src/types'; @@ -1235,4 +1236,131 @@ describe('LayerBuilderService paused visual provider selection', () => { expect(nestedLayers[0]?.source?.gaussianSplatRuntimeKey).toBeTruthy(); expect(nestedLayers[0]?.source?.gaussianSplatSettings?.render?.useNativeRenderer).toBe(true); }); + + it('uses proxy frames for video clips inside nested composition clips while scrubbing', () => { + const service = new LayerBuilderService(); + const file = new File(['video'], 'nested.mp4', { type: 'video/mp4' }); + const video = document.createElement('video'); + const proxyImage = new Image(); + Object.defineProperty(proxyImage, 'naturalWidth', { configurable: true, value: 640 }); + Object.defineProperty(proxyImage, 'naturalHeight', { configurable: true, value: 360 }); + const mockedProxyFrameCache = proxyFrameCache as typeof proxyFrameCache & { + getCachedFrame: ReturnType; + getNearestCachedFrameEntry: ReturnType; + getFrame: ReturnType; + }; + mockedProxyFrameCache.getCachedFrame = vi.fn().mockReturnValue(proxyImage); + mockedProxyFrameCache.getNearestCachedFrameEntry = vi.fn().mockReturnValue(null); + mockedProxyFrameCache.getFrame = vi.fn().mockResolvedValue(proxyImage); + + const getMediaStateSpy = vi.spyOn(useMediaStore, 'getState').mockReturnValue({ + ...initialMediaState, + activeCompositionId: null, + activeLayerSlots: {}, + layerOpacities: {}, + proxyEnabled: true, + files: [{ + id: 'media-video-1', + name: 'nested.mp4', + type: 'video', + createdAt: 1, + file, + duration: 10, + proxyStatus: 'ready', + proxyFps: 24, + }], + compositions: [], + }); + + useTimelineStore.setState({ + tracks: [ + { + id: 'track-v1', + name: 'Video 1', + type: 'video', + visible: true, + muted: false, + solo: false, + }, + ], + clips: [ + { + id: 'comp-clip-1', + trackId: 'track-v1', + name: 'Comp 1', + startTime: 0, + duration: 10, + inPoint: 0, + outPoint: 10, + effects: [], + transform: { ...DEFAULT_TRANSFORM }, + isComposition: true, + compositionId: 'comp-1', + nestedTracks: [ + { + id: 'nested-track-v1', + name: 'Nested Video', + type: 'video', + visible: true, + muted: false, + solo: false, + }, + ], + nestedClips: [ + { + id: 'nested-video-1', + trackId: 'nested-track-v1', + mediaFileId: 'media-video-1', + name: 'nested.mp4', + file, + startTime: 0, + duration: 10, + inPoint: 0, + outPoint: 10, + effects: [], + transform: { ...DEFAULT_TRANSFORM }, + source: { + type: 'video', + mediaFileId: 'media-video-1', + videoElement: video, + naturalDuration: 10, + }, + isLoading: false, + }, + ], + isLoading: false, + }, + ], + playheadPosition: 1, + isPlaying: false, + isDraggingPlayhead: true, + playbackSpeed: 1, + }); + + const layers = service.buildLayersFromStore(); + const nestedLayers = (layers[0]?.source as NestedCompositionSource | undefined)?.nestedComposition?.layers as Array<{ + source?: { + type?: string; + imageElement?: HTMLImageElement; + proxyFrameIndex?: number; + previewPath?: string; + mediaFileId?: string; + }; + }> | undefined; + + expect(mockedProxyFrameCache.getCachedFrame).toHaveBeenCalledWith('media-video-1', 24, 24); + expect(nestedLayers).toHaveLength(1); + expect(nestedLayers?.[0]?.source).toMatchObject({ + type: 'image', + imageElement: proxyImage, + proxyFrameIndex: 24, + previewPath: 'nested-proxy-frame', + mediaFileId: 'media-video-1', + }); + + delete mockedProxyFrameCache.getCachedFrame; + delete mockedProxyFrameCache.getNearestCachedFrameEntry; + delete mockedProxyFrameCache.getFrame; + getMediaStateSpy.mockRestore(); + }); }); diff --git a/tests/unit/mediaPanelSourceMonitor.test.tsx b/tests/unit/mediaPanelSourceMonitor.test.tsx index 746db444..b49ab130 100644 --- a/tests/unit/mediaPanelSourceMonitor.test.tsx +++ b/tests/unit/mediaPanelSourceMonitor.test.tsx @@ -150,4 +150,16 @@ describe('MediaPanel source monitor opening', () => { expect(mediaState.setSourceMonitorFile).toHaveBeenCalledWith('file-1'); }); + + it('does not mount board assets far outside the viewport', () => { + localStorage.setItem('media-panel-board-viewport', JSON.stringify({ + zoom: 1, + panX: -20000, + panY: -20000, + })); + + const { container } = render(); + + expect(container.querySelectorAll('.media-board-node')).toHaveLength(0); + }); }); diff --git a/tests/unit/motionDesignRendering.test.ts b/tests/unit/motionDesignRendering.test.ts new file mode 100644 index 00000000..ef2ba992 --- /dev/null +++ b/tests/unit/motionDesignRendering.test.ts @@ -0,0 +1,129 @@ +import { describe, expect, it } from 'vitest'; +import type { ClipTransform, Keyframe, TimelineClip } from '../../src/types'; +import { + createDefaultMotionLayerDefinition, + createStrokeAppearance, +} from '../../src/types/motionDesign'; +import { createMotionUniformArray } from '../../src/engine/motion/MotionBuffers'; +import { getMotionRenderSize } from '../../src/engine/motion/MotionTypes'; +import { getInterpolatedMotionLayer } from '../../src/utils/motionInterpolation'; +import { createTestTimelineStore } from '../helpers/storeFactory'; + +function makeTransform(): ClipTransform { + return { + opacity: 1, + blendMode: 'normal', + position: { x: 0, y: 0, z: 0 }, + scale: { x: 1, y: 1 }, + rotation: { x: 0, y: 0, z: 0 }, + }; +} + +function makeMotionClip(motion = createDefaultMotionLayerDefinition('shape')): TimelineClip { + return { + id: 'motion-clip', + trackId: 'video-1', + name: 'Motion', + file: new File([], 'motion.msmotion'), + startTime: 0, + duration: 5, + inPoint: 0, + outPoint: 5, + source: { type: 'motion-shape', naturalDuration: 5 }, + motion, + transform: makeTransform(), + effects: [], + isLoading: false, + }; +} + +describe('motion design rendering helpers', () => { + it('sizes motion render targets with outside stroke padding', () => { + const motion = createDefaultMotionLayerDefinition('shape', { + size: { w: 100, h: 50 }, + }); + motion.appearance?.items.push({ + ...createStrokeAppearance({ r: 1, g: 0, b: 0, a: 1 }), + visible: true, + width: 12, + alignment: 'outside', + }); + + expect(getMotionRenderSize(motion)).toEqual({ + width: 124, + height: 74, + strokePadding: 12, + }); + }); + + it('packs shape, fill, and stroke values into renderer uniforms', () => { + const motion = createDefaultMotionLayerDefinition('shape', { + primitive: 'ellipse', + size: { w: 100, h: 50 }, + fillColor: { r: 0.25, g: 0.5, b: 0.75, a: 1 }, + }); + motion.appearance?.items.push({ + ...createStrokeAppearance({ r: 1, g: 0, b: 0, a: 1 }), + visible: true, + width: 8, + alignment: 'center', + }); + + const uniforms = createMotionUniformArray(motion, getMotionRenderSize(motion)); + + expect(Array.from(uniforms.slice(0, 6))).toEqual([100, 50, 108, 58, 0, 1]); + expect(Array.from(uniforms.slice(8, 12))).toEqual([0.25, 0.5, 0.75, 1]); + expect(Array.from(uniforms.slice(16, 19))).toEqual([8, 1, 0]); + }); + + it('interpolates numeric motion properties through the property registry', () => { + const clip = makeMotionClip(createDefaultMotionLayerDefinition('shape', { + size: { w: 100, h: 50 }, + })); + const keyframes: Keyframe[] = [ + { + id: 'kf-1', + clipId: clip.id, + time: 0, + property: 'shape.size.w', + value: 100, + easing: 'linear', + }, + { + id: 'kf-2', + clipId: clip.id, + time: 2, + property: 'shape.size.w', + value: 300, + easing: 'linear', + }, + ]; + + const interpolated = getInterpolatedMotionLayer(clip, keyframes, 1); + + expect(interpolated?.shape?.size.w).toBe(200); + expect(clip.motion?.shape?.size.w).toBe(100); + }); + + it('converts solid clips to motion rectangle clips while keeping timeline identity', () => { + const store = createTestTimelineStore(); + const clipId = store.getState().addSolidClip('video-1', 2, '#336699', 4, true); + + expect(clipId).toBeTruthy(); + const convertedId = store.getState().convertSolidToMotionShape(clipId!); + const converted = store.getState().clips.find((clip) => clip.id === clipId); + const fill = converted?.motion?.appearance?.items[0]; + + expect(convertedId).toBe(clipId); + expect(converted?.source?.type).toBe('motion-shape'); + expect(converted?.startTime).toBe(2); + expect(converted?.duration).toBe(4); + expect(converted?.motion?.shape?.primitive).toBe('rectangle'); + expect(fill?.kind).toBe('color-fill'); + if (fill?.kind === 'color-fill') { + expect(fill.color.r).toBeCloseTo(0.2, 3); + expect(fill.color.g).toBeCloseTo(0.4, 3); + expect(fill.color.b).toBeCloseTo(0.6, 3); + } + }); +}); diff --git a/tests/unit/propertyRegistry.test.ts b/tests/unit/propertyRegistry.test.ts new file mode 100644 index 00000000..3c8dad10 --- /dev/null +++ b/tests/unit/propertyRegistry.test.ts @@ -0,0 +1,139 @@ +import { describe, expect, it } from 'vitest'; +import type { ClipTransform, SerializableClip, TimelineClip } from '../../src/types'; +import { createDefaultMotionLayerDefinition } from '../../src/types/motionDesign'; +import { PropertyRegistry } from '../../src/services/properties/PropertyRegistry'; +import { registerCoreProperties } from '../../src/services/properties/registerCoreProperties'; + +function makeTransform(overrides?: Partial): ClipTransform { + return { + opacity: 1, + blendMode: 'normal', + position: { x: 0, y: 0, z: 0 }, + scale: { x: 1, y: 1 }, + rotation: { x: 0, y: 0, z: 0 }, + ...overrides, + }; +} + +function makeClip(overrides?: Partial): TimelineClip { + return { + id: 'clip-1', + trackId: 'video-1', + name: 'Clip', + file: new File([], 'clip.dat'), + startTime: 0, + duration: 5, + inPoint: 0, + outPoint: 5, + source: { type: 'video', naturalDuration: 5 }, + transform: makeTransform(), + effects: [], + isLoading: false, + ...overrides, + }; +} + +function createRegistry(): PropertyRegistry { + return registerCoreProperties(new PropertyRegistry()); +} + +describe('PropertyRegistry', () => { + it('describes and writes transform properties without mutating the source clip', () => { + const registry = createRegistry(); + const clip = makeClip({ transform: makeTransform({ position: { x: 12, y: 0, z: 0 } }) }); + + const descriptor = registry.getDescriptor('position.x', clip); + expect(descriptor?.label).toBe('Position X'); + expect(registry.readValue(clip, 'position.x')).toBe(12); + + const updated = registry.writeValue(clip, 'position.x', 42); + expect(updated.transform.position.x).toBe(42); + expect(clip.transform.position.x).toBe(12); + }); + + it('searches registered labels and aliases', () => { + const registry = createRegistry(); + const matches = registry.search({ query: 'alpha' }); + + expect(matches.some((descriptor) => descriptor.path === 'opacity')).toBe(true); + }); + + it('resolves effect instance parameters from the current clip', () => { + const registry = createRegistry(); + const clip = makeClip({ + effects: [ + { + id: 'fx-1', + type: 'brightness', + name: 'Brightness', + enabled: true, + params: { amount: 0.25 }, + }, + ], + }); + + const descriptor = registry.getDescriptor('effect.fx-1.amount', clip); + expect(descriptor?.label).toBe('Amount'); + expect(descriptor?.group).toBe('Effects / Brightness'); + expect(registry.readValue(clip, 'effect.fx-1.amount')).toBe(0.25); + + const updated = registry.writeValue(clip, 'effect.fx-1.amount', 0.75); + expect(updated.effects[0].params.amount).toBe(0.75); + expect(clip.effects[0].params.amount).toBe(0.25); + }); + + it('describes and writes motion shape and appearance properties', () => { + const registry = createRegistry(); + const motion = createDefaultMotionLayerDefinition('shape', { + size: { w: 200, h: 100 }, + fillColor: { r: 0.2, g: 0.3, b: 0.4, a: 1 }, + }); + const fillId = motion.appearance?.items[0].id; + const clip = makeClip({ + source: { type: 'motion-shape', naturalDuration: 5 }, + motion, + }); + + expect(registry.readValue(clip, 'shape.size.w')).toBe(200); + const resized = registry.writeValue(clip, 'shape.size.w', 640); + expect(resized.motion?.shape?.size.w).toBe(640); + expect(clip.motion?.shape?.size.w).toBe(200); + + expect(fillId).toBeDefined(); + const colorPath = `appearance.${fillId}.color.r`; + expect(registry.getDescriptor(colorPath, clip)?.label).toBe('Fill R'); + const recolored = registry.writeValue(clip, colorPath, 0.9); + const recoloredFill = recolored.motion?.appearance?.items[0]; + expect(recoloredFill?.kind).toBe('color-fill'); + if (recoloredFill?.kind === 'color-fill') { + expect(recoloredFill.color.r).toBe(0.9); + } + }); + + it('keeps motion definitions JSON-serializable on clips', () => { + const motion = createDefaultMotionLayerDefinition('shape', { + primitive: 'ellipse', + size: { w: 320, h: 240 }, + }); + const clip: SerializableClip = { + id: 'motion-clip', + trackId: 'video-1', + name: 'Ellipse', + mediaFileId: '', + startTime: 0, + duration: 5, + inPoint: 0, + outPoint: 5, + sourceType: 'motion-shape', + transform: makeTransform(), + effects: [], + motion, + }; + + const restored: SerializableClip = JSON.parse(JSON.stringify(clip)); + expect(restored.motion?.version).toBe(1); + expect(restored.motion?.kind).toBe('shape'); + expect(restored.motion?.shape?.primitive).toBe('ellipse'); + expect(restored.motion?.appearance?.items).toHaveLength(1); + }); +}); diff --git a/tests/unit/proxyCompleteness.test.ts b/tests/unit/proxyCompleteness.test.ts new file mode 100644 index 00000000..4fa88b00 --- /dev/null +++ b/tests/unit/proxyCompleteness.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from 'vitest'; +import type { Sample } from '../../src/engine/webCodecsTypes'; +import { + getExpectedProxyFrameCount, + isProxyFrameCountComplete, +} from '../../src/stores/mediaStore/helpers/proxyCompleteness'; +import { + getFirstPresentationCts, + getNormalizedSampleTimestampUs, +} from '../../src/services/proxyGenerator'; + +function makeSample(cts: number): Sample { + return { + number: 0, + track_id: 1, + data: new ArrayBuffer(0), + size: 0, + cts, + dts: 0, + duration: 512, + is_sync: false, + timescale: 12288, + }; +} + +describe('proxy completeness', () => { + it('does not add an extra expected frame for tiny duration rounding noise', () => { + expect(getExpectedProxyFrameCount(5.041666666666667, 24)).toBe(121); + expect(getExpectedProxyFrameCount(5.041667, 24)).toBe(121); + }); + + it('keeps real fractional frame durations rounded up', () => { + expect(getExpectedProxyFrameCount(5.05, 24)).toBe(122); + }); + + it('applies the 98 percent threshold to the stable expected frame count', () => { + expect(isProxyFrameCountComplete(117, 5.041667, 24)).toBe(false); + expect(isProxyFrameCountComplete(119, 5.041667, 24)).toBe(true); + }); +}); + +describe('proxy sample timestamps', () => { + it('normalizes composition-time offsets before deriving frame indices', () => { + const samples = [makeSample(2048), makeSample(2560), makeSample(61440)]; + const firstPresentationCts = getFirstPresentationCts(samples); + + expect(firstPresentationCts).toBe(2048); + expect(getNormalizedSampleTimestampUs(samples[0], firstPresentationCts)).toBe(0); + expect(getNormalizedSampleTimestampUs(samples[1], firstPresentationCts)).toBeCloseTo(41666.67, 1); + expect(Math.round((getNormalizedSampleTimestampUs(samples[2], firstPresentationCts) / 1_000_000) * 24)).toBe(116); + }); +}); diff --git a/tests/unit/renderDispatcher.test.ts b/tests/unit/renderDispatcher.test.ts index d4c71094..07b69072 100644 --- a/tests/unit/renderDispatcher.test.ts +++ b/tests/unit/renderDispatcher.test.ts @@ -149,6 +149,9 @@ describe('RenderDispatcher empty playback hold', () => { useEngineStore.setState({ sceneNavClipId: null, sceneNavFpsMode: false, + sceneGizmoVisible: true, + sceneGizmoClipIdOverride: null, + sceneGizmoHoveredAxis: null, }); useMediaStore.setState({ files: [], @@ -399,6 +402,76 @@ describe('RenderDispatcher empty playback hold', () => { expect(layerData[0]?.layer.rotation).toEqual({ x: 0, y: 0, z: 0 }); }); + it('uses the preview scene-handle toggle for the native scene gizmo pass', () => { + const { dispatcher, deps } = createDispatcher(false); + deps.sceneRenderer = { + isInitialized: true, + renderScene: vi.fn(() => ({ label: 'shared-scene-view' })), + }; + + useTimelineStore.setState({ + selectedClipIds: new Set(['native-splat-clip']), + primarySelectedClipId: 'native-splat-clip', + clips: [{ + id: 'native-splat-clip', + trackId: 'track-1', + startTime: 0, + duration: 1, + transform: { + position: { x: 0, y: 0, z: 0 }, + scale: { x: 1, y: 1, z: 1 }, + rotation: { x: 0, y: 0, z: 0 }, + opacity: 1, + blendMode: 'normal', + }, + source: { type: 'gaussian-splat' }, + }], + }); + + const createLayerData = () => [ + { + layer: { + id: 'native-splat-layer', + sourceClipId: 'native-splat-clip', + name: 'Native Splat', + visible: true, + opacity: 1, + blendMode: 'normal', + is3D: true, + position: { x: 0, y: 0, z: 0 }, + scale: { x: 1, y: 1, z: 1 }, + rotation: { x: 0, y: 0, z: 0 }, + source: { + type: 'gaussian-splat', + gaussianSplatUrl: 'blob:native-splat', + gaussianSplatFileName: 'native.ply', + gaussianSplatSettings: { + render: { + useNativeRenderer: true, + }, + }, + }, + }, + isVideo: false, + externalTexture: null, + textureView: null, + sourceWidth: 1920, + sourceHeight: 1080, + }, + ] as unknown as LayerRenderData[]; + + useEngineStore.getState().setSceneGizmoVisible(true); + dispatcher.process3DLayers(createLayerData(), {} as GPUDevice, 1920, 1080); + expect(deps.sceneRenderer.renderScene.mock.calls[0][5]).toMatchObject({ + clipId: 'native-splat-clip', + }); + + deps.sceneRenderer.renderScene.mockClear(); + useEngineStore.getState().setSceneGizmoVisible(false); + dispatcher.process3DLayers(createLayerData(), {} as GPUDevice, 1920, 1080); + expect(deps.sceneRenderer.renderScene.mock.calls[0][5]).toBeNull(); + }); + it('routes pure native gaussian-splat scenes through the shared scene renderer', () => { const { dispatcher, deps } = createDispatcher(false); deps.sceneRenderer = { From 4d7241395a23a2469eefaddde45c9928599ad154 Mon Sep 17 00:00:00 2001 From: Sportinger Date: Wed, 6 May 2026 10:11:50 +0200 Subject: [PATCH 2/3] Improve motion design and media persistence --- docs/Features/GPU-Engine.md | 1 + docs/Features/Keyframes.md | 5 + docs/Features/Motion-Design.md | 8 +- docs/Features/Timeline.md | 4 + src/App.css | 8 + src/components/panels/MediaPanel.tsx | 182 +++++++++++- .../panels/properties/MotionShapeTab.tsx | 92 ++++++- src/components/timeline/Timeline.tsx | 3 +- .../timeline/TimelineContextMenu.tsx | 20 ++ src/components/timeline/TrackContextMenu.tsx | 19 ++ src/engine/core/WebGPUContext.ts | 13 +- src/engine/motion/MotionBuffers.ts | 29 ++ src/engine/motion/MotionPipeline.ts | 27 +- src/engine/motion/MotionRenderer.ts | 16 +- src/engine/motion/MotionTypes.ts | 141 +++++++++- src/engine/motion/shaders/motionShapes.wgsl | 94 ++++--- src/engine/render/RenderLoop.ts | 13 +- src/engine/stats/PerformanceStats.ts | 5 +- src/services/project/projectLifecycle.ts | 97 ++++++- src/services/project/projectLoad.ts | 258 +++++++++--------- src/services/project/projectSave.ts | 44 ++- .../mediaStore/slices/fileManageSlice.ts | 76 +++++- .../stores/mediaStore/fileManageSlice.test.ts | 4 + tests/unit/mediaPanelSourceMonitor.test.tsx | 1 + tests/unit/motionDesignRendering.test.ts | 73 ++++- tests/unit/performanceStats.test.ts | 14 + tests/unit/projectMediaPersistence.test.ts | 176 ++++++++++++ 27 files changed, 1213 insertions(+), 210 deletions(-) diff --git a/docs/Features/GPU-Engine.md b/docs/Features/GPU-Engine.md index d25950fa..adca8b94 100644 --- a/docs/Features/GPU-Engine.md +++ b/docs/Features/GPU-Engine.md @@ -59,6 +59,7 @@ The engine currently supports: - `motion-shape` clips render through `src/engine/motion/MotionRenderer.ts`. - Rectangle and ellipse primitives are drawn with analytic WGSL SDFs into transparent `rgba8unorm` textures. +- Grid-replicated motion shapes use a per-shape instance buffer and instanced draws in the same shader path, capped at 100 instances for the current MVP. - The resulting texture view is composited through the normal `CompositorPipeline`, so masks, effects, blend mode, nested comps, preview targets, and export share the same downstream path. ### Masks diff --git a/docs/Features/Keyframes.md b/docs/Features/Keyframes.md index 54e788d6..3abe9f99 100644 --- a/docs/Features/Keyframes.md +++ b/docs/Features/Keyframes.md @@ -96,6 +96,11 @@ appearance.{appearanceId}.color.g appearance.{appearanceId}.color.b appearance.{appearanceId}.color.a appearance.{appearanceId}.stroke.width +replicator.count.x +replicator.count.y +replicator.spacing.x +replicator.spacing.y +replicator.offset.opacity ``` Numeric motion properties are interpolated before `MotionRenderer` draws the shape texture, so preview, nested compositions, and export evaluate the same frame state. Enum-like fields such as primitive and stroke alignment are currently static controls. diff --git a/docs/Features/Motion-Design.md b/docs/Features/Motion-Design.md index 6bad3e08..4116d4f6 100644 --- a/docs/Features/Motion-Design.md +++ b/docs/Features/Motion-Design.md @@ -14,14 +14,18 @@ The motion design system follows `docs/plans/motion-design-system-plan.md`. It i - `src/services/properties/PropertyRegistry.ts` describes transform, effect, color, mask, vector-animation, and motion properties without owning Zustand state. - `src/stores/timeline/motionClipSlice.ts` can create rectangle/ellipse shape clips, null clips, adjustment clips, update motion definitions, and convert solid clips to motion rectangle clips. - `src/components/panels/properties/MotionShapeTab.tsx` exposes primitive, size, corner radius, fill, and stroke controls for motion shape clips. +- Video track-header context menus can create Motion Rectangle and Motion Ellipse clips at the playhead. +- Solid clip context menus can convert the selected solid to a motion shape while preserving its clip id and timing. +- The Motion tab exposes a first Grid Replicator section with enable, count, spacing, and opacity fade controls. - `src/engine/motion/MotionRenderer.ts` renders rectangle and ellipse primitives into transparent `rgba8unorm` textures using analytic WGSL SDFs. +- The renderer supports grid-replicated rectangle/ellipse shapes through a per-shape instance buffer and instanced draws, capped at 100 instances for the current MVP. - `LayerBuilderService`, `NestedCompRenderer`, `RenderDispatcher`, and `ExportLayerBuilder` pass motion shape layers through the same compositor path as image/text/video textures. - Numeric motion properties are evaluated through the keyframe store via the property registry before rendering. ## Not Yet Implemented -- Replicators are represented in the schema and registry, but no GPU instancing pipeline is wired. +- Replicators have a grid MVP for shape clips, but no random/noise modifiers, radial/linear layouts, falloff, or direct media replicators are wired yet. - Texture fills, gradients, appearance blend modes, polygon/star rendering, viewport motion paths, and graph mode are not implemented yet. - Adjustment layers remain blocked on the render graph work. -The next implementation slice should add user-facing creation affordances, pinned motion property lanes, and the first replicator controls while keeping adjustment layers deferred until the render graph work is ready. +The next implementation slice should add pinned motion property lanes or media texture fills while keeping adjustment layers deferred until the render graph work is ready. diff --git a/docs/Features/Timeline.md b/docs/Features/Timeline.md index c98ddc58..0ce1bcf8 100644 --- a/docs/Features/Timeline.md +++ b/docs/Features/Timeline.md @@ -65,7 +65,10 @@ getTrackChildren() // Query child tracks ### Motion Shape - Rectangle and ellipse shape clips are timeline clips with JSON motion definitions. +- Video track-header context menus can add Motion Rectangle or Motion Ellipse clips at the current playhead position. +- Solid clip context menus expose Convert Solid to Motion Shape. - The Motion tab exposes primitive, size, radius, fill, and stroke controls. +- Motion shape replicator controls can enable a grid, edit X/Y counts, edit X/Y spacing, and keyframe the per-instance fade. - Solid clips can be converted in the store to motion rectangle clips while preserving timeline identity, timing, transform, effects, and keyframes. - Motion shape rendering uses WebGPU SDF textures, then the normal compositor stack. @@ -188,6 +191,7 @@ Soloing multiple tracks is supported. Non-solo tracks dim visually when any solo ### Track Header Context Menu +- Video tracks can create Motion Rectangle, Motion Ellipse, or Math Scene clips at the playhead. - `Duplicate Track` currently creates a new empty track of the same type. - `Delete` is blocked for the last remaining track of that type. - Deleting a populated track shows the affected clip count in the menu label/tooltip. diff --git a/src/App.css b/src/App.css index 420e6efc..ff8ca7cc 100644 --- a/src/App.css +++ b/src/App.css @@ -9170,6 +9170,7 @@ input[type="checkbox"] { display: flex; flex-direction: column; flex: 1; + min-height: 0; } /* Column headers */ @@ -9412,6 +9413,13 @@ input[type="checkbox"] { user-select: none; flex: 1; min-height: 0; + overflow: auto; + contain: layout paint; +} + +.media-classic-virtual-spacer { + flex: 0 0 auto; + pointer-events: none; } /* View mode toggle buttons */ diff --git a/src/components/panels/MediaPanel.tsx b/src/components/panels/MediaPanel.tsx index 302a9649..98271fb7 100644 --- a/src/components/panels/MediaPanel.tsx +++ b/src/components/panels/MediaPanel.tsx @@ -32,6 +32,14 @@ type ColumnId = 'label' | 'name' | 'duration' | 'resolution' | 'fps' | 'containe type MediaPanelViewMode = 'classic' | 'icons' | 'board'; type MediaBoardItem = Exclude; +const CLASSIC_ROW_HEIGHT = 20; +const CLASSIC_OVERSCAN_ROWS = 12; + +interface ClassicListRow { + item: ProjectItem; + depth: number; +} + interface MediaBoardViewport { zoom: number; panX: number; @@ -539,6 +547,7 @@ export function MediaPanel() { const proxyFolderName = useMediaStore(state => state.proxyFolderName); const activeCompositionId = useMediaStore(state => state.activeCompositionId); const refreshFileUrls = useMediaStore(state => state.refreshFileUrls); + const ensureFileThumbnail = useMediaStore(state => state.ensureFileThumbnail); // Actions from getState() - stable, no subscription needed const { @@ -555,7 +564,6 @@ export function MediaPanel() { toggleFolderExpanded, setSelection, addToSelection, - getItemsByFolder, openCompositionTab, updateComposition, generateProxy, @@ -607,6 +615,7 @@ export function MediaPanel() { const [dragOverFolderId, setDragOverFolderId] = useState(null); const [internalDragId, setInternalDragId] = useState(null); const [isExternalDragOver, setIsExternalDragOver] = useState(false); + const [classicListViewport, setClassicListViewport] = useState({ scrollTop: 0, height: 0 }); const [labelPickerItemId, setLabelPickerItemId] = useState(null); const [labelPickerPos, setLabelPickerPos] = useState<{ x: number; y: number } | null>(null); const [viewMode, setViewMode] = useState(loadMediaPanelViewMode); @@ -683,6 +692,34 @@ export function MediaPanel() { return () => resizeObserver.disconnect(); }, [viewMode]); + useLayoutEffect(() => { + if (viewMode !== 'classic') return; + + const list = itemListRef.current; + if (!list) return; + + const updateViewport = () => { + setClassicListViewport((current) => { + const next = { + scrollTop: list.scrollTop, + height: list.clientHeight, + }; + return current.scrollTop === next.scrollTop && current.height === next.height ? current : next; + }); + }; + + updateViewport(); + + if (typeof ResizeObserver === 'undefined') { + window.addEventListener('resize', updateViewport); + return () => window.removeEventListener('resize', updateViewport); + } + + const resizeObserver = new ResizeObserver(updateViewport); + resizeObserver.observe(list); + return () => resizeObserver.disconnect(); + }, [viewMode]); + useEffect(() => () => { if (boardInteractionFrameRef.current !== null) { window.cancelAnimationFrame(boardInteractionFrameRef.current); @@ -1086,6 +1123,17 @@ export function MediaPanel() { return [...folderItems, ...nonFolderItems]; }, [sortColumn, sortDirection, getSortValue]); + const handleClassicListScroll = useCallback((event: React.UIEvent) => { + const target = event.currentTarget; + setClassicListViewport((current) => { + const next = { + scrollTop: target.scrollTop, + height: target.clientHeight, + }; + return current.scrollTop === next.scrollTop && current.height === next.height ? current : next; + }); + }, []); + // Close dropdown when clicking outside useEffect(() => { if (!addDropdownOpen) return; @@ -2063,8 +2111,8 @@ export function MediaPanel() { } }; - // Render a single item - const renderItem = (item: ProjectItem, depth: number = 0) => { + // Render a single classic-list row. Tree traversal is virtualized separately. + const renderClassicRow = (item: ProjectItem, depth: number = 0) => { const isFolder = 'isExpanded' in item; const isSelected = selectedIds.includes(item.id); const isRenaming = renamingId === item.id; @@ -2097,11 +2145,6 @@ export function MediaPanel() { ))}
- {isFolder && isExpanded && ( -
- {sortItems(getItemsByFolder(item.id)).map(child => renderItem(child, depth + 1))} -
- )} ); }; @@ -2111,7 +2154,7 @@ export function MediaPanel() { const parts: string[] = [item.name]; if (isFolder) { - const children = getItemsByFolder(item.id); + const children = getItemsForParent(item.id); parts.push(`${children.length} item${children.length !== 1 ? 's' : ''}`); } else if (isComp) { const comp = item as Composition; @@ -2156,7 +2199,7 @@ export function MediaPanel() { const duration = mediaFile?.duration || comp?.duration; // Folder item count - const folderCount = isFolder ? getItemsByFolder(item.id).length : 0; + const folderCount = isFolder ? getItemsForParent(item.id).length : 0; return (
@@ -2203,8 +2246,6 @@ export function MediaPanel() { ); }; - // Get root items (with sorting applied) - const rootItems = sortItems(getItemsByFolder(null)); const totalItems = ( files.length + compositions.length + @@ -2216,6 +2257,71 @@ export function MediaPanel() { splatEffectorItems.length ); + const projectItemsByParentId = useMemo(() => { + const itemsByParentId = new Map(); + const append = (item: ProjectItem) => { + const parentId = item.parentId ?? null; + const items = itemsByParentId.get(parentId); + if (items) { + items.push(item); + } else { + itemsByParentId.set(parentId, [item]); + } + }; + + folders.forEach(append); + compositions.forEach(append); + textItems.forEach(append); + solidItems.forEach(append); + meshItems.forEach(append); + cameraItems.forEach(append); + splatEffectorItems.forEach(append); + files.forEach(append); + + return itemsByParentId; + }, [files, compositions, folders, textItems, solidItems, meshItems, cameraItems, splatEffectorItems]); + + const getItemsForParent = useCallback( + (parentId: string | null) => projectItemsByParentId.get(parentId) ?? [], + [projectItemsByParentId], + ); + + const classicExpandedFolderIdSet = useMemo(() => new Set(expandedFolderIds), [expandedFolderIds]); + const classicRows = useMemo(() => { + const rows: ClassicListRow[] = []; + const appendRows = (items: ProjectItem[], depth: number) => { + for (const item of sortItems(items)) { + rows.push({ item, depth }); + if ('isExpanded' in item && classicExpandedFolderIdSet.has(item.id)) { + appendRows(getItemsForParent(item.id), depth + 1); + } + } + }; + + appendRows(getItemsForParent(null), 0); + return rows; + }, [ + sortItems, + getItemsForParent, + classicExpandedFolderIdSet, + ]); + + const classicVisibleRange = useMemo(() => { + const height = Math.max(classicListViewport.height, CLASSIC_ROW_HEIGHT); + const start = Math.max(0, Math.floor(classicListViewport.scrollTop / CLASSIC_ROW_HEIGHT) - CLASSIC_OVERSCAN_ROWS); + const visibleCount = Math.ceil(height / CLASSIC_ROW_HEIGHT) + CLASSIC_OVERSCAN_ROWS * 2; + const end = Math.min(classicRows.length, start + visibleCount); + return { start, end }; + }, [classicListViewport.height, classicListViewport.scrollTop, classicRows.length]); + + const classicVisibleRows = useMemo( + () => classicRows.slice(classicVisibleRange.start, classicVisibleRange.end), + [classicRows, classicVisibleRange.end, classicVisibleRange.start], + ); + + const classicTopSpacerHeight = classicVisibleRange.start * CLASSIC_ROW_HEIGHT; + const classicBottomSpacerHeight = Math.max(0, (classicRows.length - classicVisibleRange.end) * CLASSIC_ROW_HEIGHT); + const mediaBoardItems = useMemo(() => ([ ...files, ...compositions, @@ -2658,6 +2764,47 @@ export function MediaPanel() { )) ), [mediaBoardLayout.placements, mediaBoardVisibleRect, selectedIdSet]); + const visibleMediaBoardThumbnailKey = useMemo(() => { + if (!mediaBoardRenderLod.showImages) return ''; + return visibleMediaBoardPlacements + .map((placement) => placement.item) + .filter((item): item is MediaFile => ( + isImportedMediaFileItem(item) + && !item.thumbnailUrl + && !item.isImporting + && (item.type === 'image' || item.type === 'video') + )) + .slice(0, 48) + .map((item) => item.id) + .join('\n'); + }, [mediaBoardRenderLod.showImages, visibleMediaBoardPlacements]); + + useEffect(() => { + if (viewMode !== 'board' || !visibleMediaBoardThumbnailKey) return; + + const thumbnailIds = visibleMediaBoardThumbnailKey.split('\n').filter(Boolean); + let cancelled = false; + let nextIndex = 0; + const workerCount = Math.min(3, thumbnailIds.length); + + const runWorker = async () => { + while (!cancelled) { + const id = thumbnailIds[nextIndex]; + nextIndex += 1; + if (!id) return; + await ensureFileThumbnail(id); + } + }; + + for (let index = 0; index < workerCount; index += 1) { + void runWorker(); + } + + return () => { + cancelled = true; + }; + }, [ensureFileThumbnail, viewMode, visibleMediaBoardThumbnailKey]); + const screenToMediaBoard = useCallback((clientX: number, clientY: number) => { const rect = boardCanvasRef.current?.getBoundingClientRect(); if (!rect) return { x: 0, y: 0 }; @@ -3918,7 +4065,7 @@ export function MediaPanel() { ); // Grid view: items for current folder + breadcrumb path - const gridItems = sortItems(getItemsByFolder(gridFolderId)); + const gridItems = sortItems(getItemsForParent(gridFolderId)); const gridBreadcrumb: Array<{ id: string | null; name: string }> = []; if (gridFolderId) { // Build path from root to current folder @@ -4118,6 +4265,7 @@ export function MediaPanel() {
{ const target = e.target as HTMLElement; @@ -4125,7 +4273,13 @@ export function MediaPanel() { }} style={{ position: 'relative' }} > - {rootItems.map(item => renderItem(item))} + {classicTopSpacerHeight > 0 && ( +
+ )} + {classicVisibleRows.map(({ item, depth }) => renderClassicRow(item, depth))} + {classicBottomSpacerHeight > 0 && ( +
+ )} {/* Marquee selection rectangle */} {marquee && (() => { const left = Math.min(marquee.startX, marquee.currentX); diff --git a/src/components/panels/properties/MotionShapeTab.tsx b/src/components/panels/properties/MotionShapeTab.tsx index 126b258a..12c3375d 100644 --- a/src/components/panels/properties/MotionShapeTab.tsx +++ b/src/components/panels/properties/MotionShapeTab.tsx @@ -5,10 +5,11 @@ import type { AppearanceItem, ColorFillAppearance, MotionColor, + ReplicatorLayout, ShapePrimitive, StrokeAppearance, } from '../../../types/motionDesign'; -import { createColorFillAppearance, createStrokeAppearance } from '../../../types/motionDesign'; +import { createColorFillAppearance, createDefaultReplicatorDefinition, createStrokeAppearance } from '../../../types/motionDesign'; import { DraggableNumber, KeyframeToggle } from './shared'; interface MotionShapeTabProps { @@ -87,6 +88,11 @@ function updateAppearanceItem( return items.map((item) => item.id === itemId ? updater(item as T) : item); } +function getGridLayout(layout: ReplicatorLayout | undefined): Extract { + if (layout?.mode === 'grid') return layout; + return createDefaultReplicatorDefinition().layout as Extract; +} + export function MotionShapeTab({ clipId }: MotionShapeTabProps) { const clip = useTimelineStore(state => state.clips.find(candidate => candidate.id === clipId)); const updateMotionLayer = useTimelineStore(state => state.updateMotionLayer); @@ -97,6 +103,8 @@ export function MotionShapeTab({ clipId }: MotionShapeTabProps) { const appearanceItems = motion?.appearance?.items ?? []; const fill = appearanceItems.find((item): item is ColorFillAppearance => item.kind === 'color-fill'); const stroke = appearanceItems.find((item): item is StrokeAppearance => item.kind === 'stroke'); + const replicator = motion?.replicator ?? createDefaultReplicatorDefinition(); + const gridLayout = getGridLayout(replicator.layout); const updatePrimitive = useCallback((primitive: ShapePrimitive) => { updateMotionLayer(clipId, (current) => ({ @@ -191,6 +199,20 @@ export function MotionShapeTab({ clipId }: MotionShapeTabProps) { : current); }, [clipId, stroke, updateMotionLayer]); + const setReplicatorEnabled = useCallback((enabled: boolean) => { + updateMotionLayer(clipId, (current) => { + const currentReplicator = current.replicator ?? createDefaultReplicatorDefinition(); + return { + ...current, + replicator: { + ...currentReplicator, + enabled, + layout: getGridLayout(currentReplicator.layout), + }, + }; + }); + }, [clipId, updateMotionLayer]); + if (!clip || !motion || !shape) { return

Select a motion shape clip

; } @@ -308,6 +330,74 @@ export function MotionShapeTab({ clipId }: MotionShapeTabProps) { /> )}
+ +
+
+ + setReplicatorEnabled(event.target.checked)} + /> + +
+ {replicator.enabled && ( + <> + + + + +
+ + Fade + setPropertyValue(clipId, 'replicator.offset.opacity', clamp01(value / 100))} + min={0} + max={100} + suffix="%" + defaultValue={100} + /> +
+ + )} +
); } diff --git a/src/components/timeline/Timeline.tsx b/src/components/timeline/Timeline.tsx index 473744b1..ff83974a 100644 --- a/src/components/timeline/Timeline.tsx +++ b/src/components/timeline/Timeline.tsx @@ -200,7 +200,7 @@ export function Timeline() { addClip, addCompClip, addTextClip, addSolidClip, addMeshClip, addCameraClip, addSplatEffectorClip, moveClip, trimClip, removeClip, selectClip, unlinkGroup, splitClip, splitClipAtPlayhead, toggleClipReverse, updateClipTransform, setClipParent, generateWaveformForClip, - addClipEffect, + addClipEffect, convertSolidToMotionShape, } = store; // Transform getters @@ -1582,6 +1582,7 @@ export function Timeline() { toggleClipReverse={toggleClipReverse} unlinkGroup={unlinkGroup} generateWaveformForClip={generateWaveformForClip} + convertSolidToMotionShape={convertSolidToMotionShape} setMulticamDialogOpen={setMulticamDialogOpen} showInExplorer={showInExplorer} /> diff --git a/src/components/timeline/TimelineContextMenu.tsx b/src/components/timeline/TimelineContextMenu.tsx index 39b7b21c..9e11cf14 100644 --- a/src/components/timeline/TimelineContextMenu.tsx +++ b/src/components/timeline/TimelineContextMenu.tsx @@ -30,6 +30,7 @@ interface TimelineContextMenuProps { toggleClipReverse: (clipId: string) => void; unlinkGroup: (clipId: string) => void; generateWaveformForClip: (clipId: string) => void; + convertSolidToMotionShape: (clipId: string) => string | null; setMulticamDialogOpen: (open: boolean) => void; // File explorer @@ -47,6 +48,7 @@ export function TimelineContextMenu({ toggleClipReverse, unlinkGroup, generateWaveformForClip, + convertSolidToMotionShape, setMulticamDialogOpen, showInExplorer, }: TimelineContextMenuProps) { @@ -158,6 +160,7 @@ export function TimelineContextMenu({ const mediaFile = getMediaFileForClip(contextMenu.clipId); const clip = clipMap.get(contextMenu.clipId); const isVideo = clip?.source?.type === 'video'; + const isSolid = clip?.source?.type === 'solid'; const isGenerating = mediaFile?.proxyStatus === 'generating'; const hasProxy = mediaFile?.proxyStatus === 'ready'; @@ -287,6 +290,23 @@ export function TimelineContextMenu({ Split at Playhead (C)
+ {isSolid && ( + <> +
+
{ + if (contextMenu.clipId) { + convertSolidToMotionShape(contextMenu.clipId); + } + setContextMenu(null); + }} + > + Convert Solid to Motion Shape +
+ + )} + {/* Multicam options */} {selectedClipIds.size > 1 && (
{ + if (menu.trackType !== 'video') return; + const { playheadPosition, addMotionShapeClip, selectClip } = useTimelineStore.getState(); + const clipId = addMotionShapeClip(menu.trackId, playheadPosition, { + primitive, + name: primitive === 'ellipse' ? 'Motion Ellipse' : 'Motion Rectangle', + }); + if (clipId) { + selectClip(clipId); + } + onClose(); + }; + const handleDeleteTrack = () => { useTimelineStore.getState().removeTrack(menu.trackId); onClose(); @@ -104,6 +117,12 @@ export function TrackContextMenu({ menu, onClose }: TrackContextMenuProps) {
+ Add Math Scene
+
handleAddMotionShape('rectangle')}> + + Add Motion Rectangle +
+
handleAddMotionShape('ellipse')}> + + Add Motion Ellipse +
)}
diff --git a/src/engine/core/WebGPUContext.ts b/src/engine/core/WebGPUContext.ts index a97f786a..f96a6a44 100644 --- a/src/engine/core/WebGPUContext.ts +++ b/src/engine/core/WebGPUContext.ts @@ -4,6 +4,11 @@ import { Logger } from '../../services/logger'; const log = Logger.create('WebGPUContext'); +const ADAPTER_WITH_PREFERENCE_TIMEOUT_MS = 2000; +const ADAPTER_FALLBACK_TIMEOUT_MS = 5000; +const DEVICE_WITH_LIMITS_TIMEOUT_MS = 2000; +const DEVICE_FALLBACK_TIMEOUT_MS = 5000; + export type DeviceLostCallback = (reason: string) => void; export type DeviceRestoredCallback = () => void; export type GPUPowerPreference = 'high-performance' | 'low-power'; @@ -83,7 +88,7 @@ export class WebGPUContext { log.info(`Requesting adapter with powerPreference: ${this.currentPowerPreference}`); this.adapter = await this.withTimeout( navigator.gpu.requestAdapter({ powerPreference: this.currentPowerPreference }), - 5000, + ADAPTER_WITH_PREFERENCE_TIMEOUT_MS, 'requestAdapter (with powerPreference)', ); @@ -92,7 +97,7 @@ export class WebGPUContext { log.warn('First adapter request failed, retrying without powerPreference...'); this.adapter = await this.withTimeout( navigator.gpu.requestAdapter(), - 5000, + ADAPTER_FALLBACK_TIMEOUT_MS, 'requestAdapter (no preference)', ); } @@ -117,7 +122,7 @@ export class WebGPUContext { requiredFeatures: [], requiredLimits, }), - 5000, + DEVICE_WITH_LIMITS_TIMEOUT_MS, 'requestDevice (with limits)', ); } catch (e) { @@ -130,7 +135,7 @@ export class WebGPUContext { log.warn('Retrying device request without requiredLimits...'); this.device = await this.withTimeout( this.adapter.requestDevice(), - 5000, + DEVICE_FALLBACK_TIMEOUT_MS, 'requestDevice (no limits)', ); } diff --git a/src/engine/motion/MotionBuffers.ts b/src/engine/motion/MotionBuffers.ts index 100d6b6f..47aff49d 100644 --- a/src/engine/motion/MotionBuffers.ts +++ b/src/engine/motion/MotionBuffers.ts @@ -72,3 +72,32 @@ export function createMotionUniformArray( return data; } + +export function createMotionInstanceArray( + size: MotionRenderSize, +): Float32Array { + const replicator = size.replicator; + const countX = Math.max(1, replicator.countX); + const countY = Math.max(1, replicator.countY); + const data = new Float32Array(replicator.instanceCount * 4); + const gridCenterX = (countX - 1) * 0.5; + const gridCenterY = (countY - 1) * 0.5; + let cursor = 0; + + for (let y = 0; y < countY; y += 1) { + const rowOffsetX = y % 2 === 1 ? replicator.patternOffsetX : 0; + const rowOffsetY = y % 2 === 1 ? replicator.patternOffsetY : 0; + for (let x = 0; x < countX; x += 1) { + const instanceIndex = y * countX + x; + data[cursor] = (x - gridCenterX) * replicator.spacingX + rowOffsetX - replicator.boundsCenterX; + data[cursor + 1] = (y - gridCenterY) * replicator.spacingY + rowOffsetY - replicator.boundsCenterY; + data[cursor + 2] = instanceIndex === 0 + ? 1 + : Math.pow(replicator.offsetOpacity, instanceIndex); + data[cursor + 3] = 0; + cursor += 4; + } + } + + return data; +} diff --git a/src/engine/motion/MotionPipeline.ts b/src/engine/motion/MotionPipeline.ts index 7f05e3fc..f01afa74 100644 --- a/src/engine/motion/MotionPipeline.ts +++ b/src/engine/motion/MotionPipeline.ts @@ -16,7 +16,7 @@ export class MotionPipeline { label: 'motion-shape-bind-group-layout', entries: [{ binding: 0, - visibility: GPUShaderStage.FRAGMENT, + visibility: GPUShaderStage.VERTEX | GPUShaderStage.FRAGMENT, buffer: { type: 'uniform' }, }], }); @@ -42,11 +42,34 @@ export class MotionPipeline { vertex: { module, entryPoint: 'vertexMain', + buffers: [{ + arrayStride: 4 * 4, + stepMode: 'instance', + attributes: [{ + shaderLocation: 0, + offset: 0, + format: 'float32x4', + }], + }], }, fragment: { module, entryPoint: 'fragmentMain', - targets: [{ format: MOTION_RENDER_TEXTURE_FORMAT }], + targets: [{ + format: MOTION_RENDER_TEXTURE_FORMAT, + blend: { + color: { + operation: 'add', + srcFactor: 'src-alpha', + dstFactor: 'one-minus-src-alpha', + }, + alpha: { + operation: 'add', + srcFactor: 'one', + dstFactor: 'one-minus-src-alpha', + }, + }, + }], }, primitive: { topology: 'triangle-list', diff --git a/src/engine/motion/MotionRenderer.ts b/src/engine/motion/MotionRenderer.ts index 3966dc88..269457ca 100644 --- a/src/engine/motion/MotionRenderer.ts +++ b/src/engine/motion/MotionRenderer.ts @@ -1,6 +1,6 @@ import type { Layer } from '../core/types'; import type { MotionLayerDefinition } from '../../types/motionDesign'; -import { createMotionUniformArray } from './MotionBuffers'; +import { createMotionInstanceArray, createMotionUniformArray } from './MotionBuffers'; import { getMotionRenderSize, MOTION_RENDER_TEXTURE_FORMAT, @@ -33,7 +33,9 @@ export class MotionRenderer { const size = getMotionRenderSize(motion); const cache = this.getOrCreateCache(layer, size.width, size.height); const uniforms = createMotionUniformArray(motion, size); + const instances = createMotionInstanceArray(size); this.device.queue.writeBuffer(cache.uniformBuffer, 0, uniforms as GPUAllowSharedBufferSource); + this.device.queue.writeBuffer(cache.instanceBuffer, 0, instances as GPUAllowSharedBufferSource); const pass = commandEncoder.beginRenderPass({ label: 'motion-shape-render-pass', @@ -46,7 +48,8 @@ export class MotionRenderer { }); pass.setPipeline(this.pipeline.getPipeline()); pass.setBindGroup(0, cache.bindGroup); - pass.draw(6); + pass.setVertexBuffer(0, cache.instanceBuffer); + pass.draw(6, size.replicator.instanceCount); pass.end(); return { @@ -59,6 +62,7 @@ export class MotionRenderer { for (const cache of this.caches.values()) { cache.texture.destroy(); cache.uniformBuffer.destroy(); + cache.instanceBuffer.destroy(); } this.caches.clear(); } @@ -77,6 +81,7 @@ export class MotionRenderer { if (existing) { existing.texture.destroy(); existing.uniformBuffer.destroy(); + existing.instanceBuffer.destroy(); this.caches.delete(key); } @@ -92,6 +97,11 @@ export class MotionRenderer { size: 20 * 4, usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, }); + const instanceBuffer = this.device.createBuffer({ + label: `motion-shape-instances-${key}`, + size: 4 * 4 * 100, + usage: GPUBufferUsage.VERTEX | GPUBufferUsage.COPY_DST, + }); const bindGroup = this.device.createBindGroup({ label: `motion-shape-bind-group-${key}`, layout: this.pipeline.getBindGroupLayout(), @@ -101,7 +111,7 @@ export class MotionRenderer { }], }); - const cache = { texture, view, uniformBuffer, bindGroup, width, height }; + const cache = { texture, view, uniformBuffer, instanceBuffer, bindGroup, width, height }; this.caches.set(key, cache); return cache; } diff --git a/src/engine/motion/MotionTypes.ts b/src/engine/motion/MotionTypes.ts index 475f1889..d3a33d22 100644 --- a/src/engine/motion/MotionTypes.ts +++ b/src/engine/motion/MotionTypes.ts @@ -1,11 +1,29 @@ -import type { MotionLayerDefinition, StrokeAppearance } from '../../types/motionDesign'; +import type { MotionLayerDefinition, ReplicatorLayout, StrokeAppearance } from '../../types/motionDesign'; export const MOTION_RENDER_TEXTURE_FORMAT: GPUTextureFormat = 'rgba8unorm'; +export const MOTION_REPLICATOR_SHADER_MAX_INSTANCES = 100; export interface MotionRenderSize { width: number; height: number; strokePadding: number; + replicator: MotionReplicatorRenderState; +} + +export interface MotionReplicatorRenderState { + enabled: boolean; + countX: number; + countY: number; + spacingX: number; + spacingY: number; + patternOffsetX: number; + patternOffsetY: number; + offsetOpacity: number; + instanceCount: number; + boundsCenterX: number; + boundsCenterY: number; + boundsWidth: number; + boundsHeight: number; } export interface MotionRenderResult extends MotionRenderSize { @@ -16,6 +34,7 @@ export interface MotionClipGpuCache { texture: GPUTexture; view: GPUTextureView; uniformBuffer: GPUBuffer; + instanceBuffer: GPUBuffer; bindGroup: GPUBindGroup; width: number; height: number; @@ -37,15 +56,131 @@ function getStrokePadding(stroke: StrokeAppearance | undefined): number { return Math.ceil(stroke.width); } +function finiteOr(value: number | undefined, fallback: number): number { + return typeof value === 'number' && Number.isFinite(value) ? value : fallback; +} + +function clampCount(value: number | undefined, max: number): number { + return Math.max(1, Math.min(max, Math.round(finiteOr(value, 1)))); +} + +function getGridLayout(layout: ReplicatorLayout | undefined): Extract | undefined { + return layout?.mode === 'grid' ? layout : undefined; +} + +function getGridBounds(params: { + countX: number; + countY: number; + spacingX: number; + spacingY: number; + patternOffsetX: number; + patternOffsetY: number; +}): { centerX: number; centerY: number; width: number; height: number } { + const { countX, countY, spacingX, spacingY, patternOffsetX, patternOffsetY } = params; + const gridCenterX = (countX - 1) * 0.5; + const gridCenterY = (countY - 1) * 0.5; + let minX = 0; + let maxX = 0; + let minY = 0; + let maxY = 0; + let initialized = false; + + for (let y = 0; y < countY; y += 1) { + const rowOffsetX = y % 2 === 1 ? patternOffsetX : 0; + const rowOffsetY = y % 2 === 1 ? patternOffsetY : 0; + for (let x = 0; x < countX; x += 1) { + const offsetX = (x - gridCenterX) * spacingX + rowOffsetX; + const offsetY = (y - gridCenterY) * spacingY + rowOffsetY; + if (!initialized) { + minX = offsetX; + maxX = offsetX; + minY = offsetY; + maxY = offsetY; + initialized = true; + } else { + minX = Math.min(minX, offsetX); + maxX = Math.max(maxX, offsetX); + minY = Math.min(minY, offsetY); + maxY = Math.max(maxY, offsetY); + } + } + } + + return { + centerX: (minX + maxX) * 0.5, + centerY: (minY + maxY) * 0.5, + width: maxX - minX, + height: maxY - minY, + }; +} + +export function getMotionReplicatorRenderState(motion: MotionLayerDefinition | undefined): MotionReplicatorRenderState { + const replicator = motion?.replicator; + const layout = getGridLayout(replicator?.layout); + if (!replicator?.enabled || !layout) { + return { + enabled: false, + countX: 1, + countY: 1, + spacingX: 0, + spacingY: 0, + patternOffsetX: 0, + patternOffsetY: 0, + offsetOpacity: 1, + instanceCount: 1, + boundsCenterX: 0, + boundsCenterY: 0, + boundsWidth: 0, + boundsHeight: 0, + }; + } + + const maxInstances = clampCount(replicator.maxInstances, MOTION_REPLICATOR_SHADER_MAX_INSTANCES); + const countX = clampCount(layout.count.x, maxInstances); + const countY = clampCount(layout.count.y, Math.max(1, Math.floor(maxInstances / countX))); + const spacingX = finiteOr(layout.spacing.x, 0) + finiteOr(replicator.offset.position.x, 0); + const spacingY = finiteOr(layout.spacing.y, 0) + finiteOr(replicator.offset.position.y, 0); + const patternOffsetX = finiteOr(layout.patternOffset?.x, 0); + const patternOffsetY = finiteOr(layout.patternOffset?.y, 0); + const bounds = getGridBounds({ + countX, + countY, + spacingX, + spacingY, + patternOffsetX, + patternOffsetY, + }); + + return { + enabled: true, + countX, + countY, + spacingX, + spacingY, + patternOffsetX, + patternOffsetY, + offsetOpacity: Math.max(0, Math.min(1, finiteOr(replicator.offset.opacity, 1))), + instanceCount: countX * countY, + boundsCenterX: bounds.centerX, + boundsCenterY: bounds.centerY, + boundsWidth: bounds.width, + boundsHeight: bounds.height, + }; +} + export function getMotionRenderSize(motion: MotionLayerDefinition | undefined): MotionRenderSize { const shape = motion?.shape; const width = Math.max(1, Math.ceil(shape?.size.w ?? 1)); const height = Math.max(1, Math.ceil(shape?.size.h ?? 1)); const strokePadding = getStrokePadding(motion ? getVisibleStroke(motion) : undefined); + const replicator = getMotionReplicatorRenderState(motion); + const replicatedWidth = Math.ceil(replicator.boundsWidth); + const replicatedHeight = Math.ceil(replicator.boundsHeight); return { - width: width + strokePadding * 2, - height: height + strokePadding * 2, + width: width + strokePadding * 2 + replicatedWidth, + height: height + strokePadding * 2 + replicatedHeight, strokePadding, + replicator, }; } diff --git a/src/engine/motion/shaders/motionShapes.wgsl b/src/engine/motion/shaders/motionShapes.wgsl index 7c86930d..7df1bc62 100644 --- a/src/engine/motion/shaders/motionShapes.wgsl +++ b/src/engine/motion/shaders/motionShapes.wgsl @@ -1,19 +1,43 @@ struct VertexOutput { @builtin(position) position: vec4f, - @location(0) uv: vec2f, + @location(0) localPoint: vec2f, + @location(1) instanceOpacity: f32, }; -@vertex -fn vertexMain(@builtin(vertex_index) vertexIndex: u32) -> VertexOutput { - var positions = array( - vec2f(-1.0, -1.0), - vec2f(1.0, -1.0), - vec2f(-1.0, 1.0), - vec2f(-1.0, 1.0), - vec2f(1.0, -1.0), - vec2f(1.0, 1.0) - ); +struct MotionUniforms { + // shape width/height, output texture width/height + data0: vec4f, + // corner radius, shape type, fill opacity, stroke opacity + data1: vec4f, + fillColor: vec4f, + strokeColor: vec4f, + // stroke width, stroke visible, stroke alignment, unused + data2: vec4f, +}; + +@group(0) @binding(0) var motion: MotionUniforms; +fn strokePadding() -> f32 { + let strokeWidth = max(0.0, motion.data2.x); + let strokeVisible = motion.data2.y; + let strokeAlignment = motion.data2.z; + if (strokeVisible < 0.5) { + return 0.0; + } + if (strokeAlignment >= 1.5) { + return strokeWidth; + } + if (strokeAlignment > 0.5) { + return 0.0; + } + return strokeWidth * 0.5; +} + +@vertex +fn vertexMain( + @builtin(vertex_index) vertexIndex: u32, + @location(0) instanceData: vec4f +) -> VertexOutput { var uvs = array( vec2f(0.0, 1.0), vec2f(1.0, 1.0), @@ -23,25 +47,25 @@ fn vertexMain(@builtin(vertex_index) vertexIndex: u32) -> VertexOutput { vec2f(1.0, 0.0) ); + let uv = uvs[vertexIndex]; + let shapeSize = max(motion.data0.xy, vec2f(1.0)); + let outputSize = max(motion.data0.zw, vec2f(1.0)); + let drawSize = shapeSize + vec2f(strokePadding() * 2.0); + let localPoint = (uv - vec2f(0.5)) * drawSize; + let outputPoint = localPoint + instanceData.xy; + var output: VertexOutput; - output.position = vec4f(positions[vertexIndex], 0.0, 1.0); - output.uv = uvs[vertexIndex]; + output.position = vec4f( + outputPoint.x / outputSize.x * 2.0, + -outputPoint.y / outputSize.y * 2.0, + 0.0, + 1.0 + ); + output.localPoint = localPoint; + output.instanceOpacity = instanceData.z; return output; } -struct MotionUniforms { - // shape width/height, output texture width/height - data0: vec4f, - // corner radius, shape type, fill opacity, stroke opacity - data1: vec4f, - fillColor: vec4f, - strokeColor: vec4f, - // stroke width, stroke visible, stroke alignment, unused - data2: vec4f, -}; - -@group(0) @binding(0) var motion: MotionUniforms; - fn sdRoundBox(point: vec2f, halfSize: vec2f, radius: f32) -> f32 { let clampedRadius = min(radius, min(halfSize.x, halfSize.y)); let q = abs(point) - halfSize + vec2f(clampedRadius); @@ -60,10 +84,8 @@ fn over(top: vec4f, bottom: vec4f) -> vec4f { return vec4f(rgb, alpha); } -@fragment -fn fragmentMain(input: VertexOutput) -> @location(0) vec4f { +fn sampleShape(localPoint: vec2f, instanceOpacity: f32) -> vec4f { let shapeSize = max(motion.data0.xy, vec2f(1.0)); - let outputSize = max(motion.data0.zw, vec2f(1.0)); let cornerRadius = motion.data1.x; let shapeType = motion.data1.y; let fillOpacity = motion.data1.z; @@ -72,17 +94,16 @@ fn fragmentMain(input: VertexOutput) -> @location(0) vec4f { let strokeVisible = motion.data2.y; let strokeAlignment = motion.data2.z; - let point = (input.uv - vec2f(0.5)) * outputSize; let halfSize = shapeSize * 0.5; let distance = select( - sdRoundBox(point, halfSize, cornerRadius), - sdEllipse(point, halfSize), + sdRoundBox(localPoint, halfSize, cornerRadius), + sdEllipse(localPoint, halfSize), shapeType > 0.5 ); let aa = max(fwidth(distance), 1.0); let fillCoverage = 1.0 - smoothstep(-aa, aa, distance); - let fillAlpha = fillCoverage * motion.fillColor.a * fillOpacity; + let fillAlpha = fillCoverage * motion.fillColor.a * fillOpacity * instanceOpacity; let fill = vec4f(motion.fillColor.rgb, fillAlpha); let centerStroke = 1.0 - smoothstep(strokeWidth * 0.5 - aa, strokeWidth * 0.5 + aa, abs(distance)); @@ -90,8 +111,13 @@ fn fragmentMain(input: VertexOutput) -> @location(0) vec4f { let outsideBand = smoothstep(-aa, aa, distance) * (1.0 - smoothstep(strokeWidth - aa, strokeWidth + aa, distance)); let alignedStroke = select(centerStroke, insideBand, strokeAlignment > 0.5 && strokeAlignment < 1.5); let strokeCoverage = select(alignedStroke, outsideBand, strokeAlignment >= 1.5); - let strokeAlpha = strokeCoverage * motion.strokeColor.a * strokeOpacity * strokeVisible; + let strokeAlpha = strokeCoverage * motion.strokeColor.a * strokeOpacity * strokeVisible * instanceOpacity; let stroke = vec4f(motion.strokeColor.rgb, strokeAlpha); return over(stroke, fill); } + +@fragment +fn fragmentMain(input: VertexOutput) -> @location(0) vec4f { + return sampleShape(input.localPoint, input.instanceOpacity); +} diff --git a/src/engine/render/RenderLoop.ts b/src/engine/render/RenderLoop.ts index 418a509b..461481fb 100644 --- a/src/engine/render/RenderLoop.ts +++ b/src/engine/render/RenderLoop.ts @@ -29,6 +29,7 @@ export class RenderLoop { // engine goes idle after 1s and scrubbing produces black frames. // Cleared when setIsPlaying(true) is called (first play warms up videos). private idleSuppressed = false; + private idleSuppressedSince = 0; // Frame rate limiting private hasActiveVideo = false; @@ -43,6 +44,7 @@ export class RenderLoop { private renderCount = 0; private readonly IDLE_TIMEOUT = 1000; // 1s before idle + private readonly IDLE_SUPPRESSION_TIMEOUT = 3000; // bounded reload warmup private readonly VIDEO_FRAME_TIME = 16.67; // ~60fps target private readonly SCRUB_FRAME_TIME = 33; // ~30fps during scrubbing (avoids wasted renders while video seeks) private readonly WATCHDOG_INTERVAL = 2000; // Check every 2s @@ -75,7 +77,15 @@ export class RenderLoop { const rafGap = lastTimestamp > 0 ? timestamp - lastTimestamp : 0; lastTimestamp = timestamp; - // Idle detection (suppressed until first play to allow video GPU warmup) + if ( + this.idleSuppressed + && timestamp - this.idleSuppressedSince > this.IDLE_SUPPRESSION_TIMEOUT + ) { + this.idleSuppressed = false; + log.info('Idle suppression lifted (warmup timeout)'); + } + + // Idle detection (briefly suppressed after reload to allow video GPU warmup) if (!this.idleSuppressed) { const timeSinceActivity = timestamp - this.lastActivityTime; if (!this.isIdle && !this.renderRequested && timeSinceActivity > this.IDLE_TIMEOUT) { @@ -268,6 +278,7 @@ export class RenderLoop { */ suppressIdle(): void { this.idleSuppressed = true; + this.idleSuppressedSince = performance.now(); this.isIdle = false; log.info('Idle suppressed (waiting for first play)'); } diff --git a/src/engine/stats/PerformanceStats.ts b/src/engine/stats/PerformanceStats.ts index 11fefb21..3788129b 100644 --- a/src/engine/stats/PerformanceStats.ts +++ b/src/engine/stats/PerformanceStats.ts @@ -137,6 +137,7 @@ export class PerformanceStats { const avgFrameTime = this.frameTimeCount > 0 ? sum / this.frameTimeCount : 0; const cadenceFps = this.getCadenceFps(); const displayFps = isIdle ? 0 : cadenceFps || this.fps; + const dropsLastSecond = isIdle ? 0 : this.detailedStats.dropsLastSecond; return { fps: displayFps, @@ -151,8 +152,8 @@ export class PerformanceStats { }, drops: { count: this.detailedStats.dropsTotal, - lastSecond: this.detailedStats.dropsLastSecond, - reason: this.detailedStats.lastDropReason, + lastSecond: dropsLastSecond, + reason: dropsLastSecond > 0 ? this.detailedStats.lastDropReason : 'none', }, layerCount: this.lastLayerCount, targetFps: 60, diff --git a/src/services/project/projectLifecycle.ts b/src/services/project/projectLifecycle.ts index 60b3e8bd..971d2efd 100644 --- a/src/services/project/projectLifecycle.ts +++ b/src/services/project/projectLifecycle.ts @@ -1,7 +1,8 @@ // Project Lifecycle — create, open, close, auto-sync import { Logger } from '../logger'; -import { useMediaStore } from '../../stores/mediaStore'; +import { useMediaStore, type MediaFile, type Composition, type MediaFolder } from '../../stores/mediaStore'; +import type { MediaState } from '../../stores/mediaStore/types'; import { useTimelineStore } from '../../stores/timeline'; import { useYouTubeStore } from '../../stores/youtubeStore'; import { useDockStore } from '../../stores/dockStore'; @@ -25,6 +26,80 @@ let beforeUnloadHandler: (() => void) | null = null; const DEFAULT_CONTINUOUS_SAVE_DELAY_MS = 1000; +type MediaAutoSyncSelection = Pick< + MediaState, + 'files' | 'compositions' | 'folders' | 'slotAssignments' | 'slotClipSettings' +>; + +function shallowTupleEqual(a: T, b: T): boolean { + if (a === b) return true; + if (a.length !== b.length) return false; + return a.every((value, index) => Object.is(value, b[index])); +} + +function isPersistedMediaFileEqual(a: MediaFile, b: MediaFile): boolean { + return a.id === b.id + && a.name === b.name + && a.type === b.type + && a.parentId === b.parentId + && a.createdAt === b.createdAt + && a.filePath === b.filePath + && a.projectPath === b.projectPath + && a.fileHash === b.fileHash + && a.duration === b.duration + && a.width === b.width + && a.height === b.height + && a.fps === b.fps + && a.codec === b.codec + && a.audioCodec === b.audioCodec + && a.container === b.container + && a.bitrate === b.bitrate + && a.fileSize === b.fileSize + && a.hasAudio === b.hasAudio + && a.splatCount === b.splatCount + && a.totalSplatCount === b.totalSplatCount + && a.splatFrameCount === b.splatFrameCount + && a.proxyStatus === b.proxyStatus + && a.proxyFrameCount === b.proxyFrameCount + && a.proxyFps === b.proxyFps + && a.labelColor === b.labelColor + && a.vectorAnimation === b.vectorAnimation + && a.modelSequence === b.modelSequence + && a.gaussianSplatSequence === b.gaussianSplatSequence; +} + +function arePersistedMediaFilesEqual(a: MediaFile[], b: MediaFile[]): boolean { + if (a === b) return true; + if (a.length !== b.length) return false; + return a.every((file, index) => isPersistedMediaFileEqual(file, b[index]!)); +} + +function areCompositionsEqual(a: Composition[], b: Composition[]): boolean { + return a === b; +} + +function areFoldersEqual(a: MediaFolder[], b: MediaFolder[]): boolean { + return a === b; +} + +function selectMediaAutoSyncState(state: MediaState): MediaAutoSyncSelection { + return { + files: state.files, + compositions: state.compositions, + folders: state.folders, + slotAssignments: state.slotAssignments, + slotClipSettings: state.slotClipSettings, + }; +} + +function isMediaAutoSyncSelectionEqual(a: MediaAutoSyncSelection, b: MediaAutoSyncSelection): boolean { + return arePersistedMediaFilesEqual(a.files, b.files) + && areCompositionsEqual(a.compositions, b.compositions) + && areFoldersEqual(a.folders, b.folders) + && a.slotAssignments === b.slotAssignments + && a.slotClipSettings === b.slotClipSettings; +} + function registerAutoSyncDisposer(disposer: unknown): void { if (typeof disposer === 'function') { autoSyncDisposers.push(disposer as () => void); @@ -192,10 +267,11 @@ export function setupAutoSync(): void { // Subscribe to store changes and mark project dirty registerAutoSyncDisposer(useMediaStore.subscribe( - (state) => [state.files, state.compositions, state.folders, state.slotAssignments, state.slotClipSettings], + selectMediaAutoSyncState, () => { markProjectDirtyAndMaybeSave(); - } + }, + { equalityFn: isMediaAutoSyncSelectionEqual }, )); registerAutoSyncDisposer(useTimelineStore.subscribe( @@ -207,10 +283,11 @@ export function setupAutoSync(): void { state.outPoint, state.loopPlayback, state.durationLocked, - ], + ] as const, () => { markProjectDirtyAndMaybeSave(); - } + }, + { equalityFn: shallowTupleEqual }, )); registerAutoSyncDisposer(useTimelineStore.subscribe( @@ -230,17 +307,19 @@ export function setupAutoSync(): void { registerAutoSyncDisposer(useMIDIStore.subscribe((state) => state.parameterBindings, handleMIDIProjectStateChange)); registerAutoSyncDisposer(useFlashBoardStore.subscribe( - (state) => [state.boards, state.activeBoardId], + (state) => [state.boards, state.activeBoardId] as const, () => { markProjectDirtyAndMaybeSave(); - } + }, + { equalityFn: shallowTupleEqual }, )); registerAutoSyncDisposer(useExportStore.subscribe( - (state) => [state.settings, state.presets, state.selectedPresetId], + (state) => [state.settings, state.presets, state.selectedPresetId] as const, () => { markProjectDirtyAndMaybeSave(); - } + }, + { equalityFn: shallowTupleEqual }, )); // Subscribe to YouTube store changes diff --git a/src/services/project/projectLoad.ts b/src/services/project/projectLoad.ts index d9c52ea1..59289b2e 100644 --- a/src/services/project/projectLoad.ts +++ b/src/services/project/projectLoad.ts @@ -4,7 +4,6 @@ import { Logger } from '../logger'; import { engine } from '../../engine/WebGPUEngine'; import { useMediaStore, type MediaFile, type Composition, type MediaFolder, type ProjectLoadProgress } from '../../stores/mediaStore'; import { getMediaInfo } from '../../stores/mediaStore/helpers/mediaInfoHelpers'; -import { createThumbnail } from '../../stores/mediaStore/helpers/thumbnailHelpers'; import { getExpectedProxyFrameCount, getExpectedProxyFps, @@ -785,6 +784,76 @@ function convertProjectFolderToStore(projectFolders: ProjectFolder[]): MediaFold })); } +type StoreItemWithParent = { + id: string; + name?: string; + parentId: string | null; +}; + +function normalizeFolderParents(folders: MediaFolder[]): MediaFolder[] { + if (folders.length === 0) return folders; + + const foldersById = new Map(folders.map((folder) => [folder.id, folder])); + let repairedCount = 0; + + const hasBrokenParent = (folder: MediaFolder): boolean => { + if (!folder.parentId) return false; + if (folder.parentId === folder.id || !foldersById.has(folder.parentId)) return true; + + const seen = new Set([folder.id]); + let nextParentId: string | null = folder.parentId; + while (nextParentId) { + if (seen.has(nextParentId)) return true; + seen.add(nextParentId); + nextParentId = foldersById.get(nextParentId)?.parentId ?? null; + } + return false; + }; + + const normalized = folders.map((folder) => { + if (!hasBrokenParent(folder)) return folder; + repairedCount += 1; + return { ...folder, parentId: null }; + }); + + if (repairedCount > 0) { + log.warn('Recovered folders with invalid parent references', { + repairedCount, + total: folders.length, + }); + } + + return repairedCount > 0 ? normalized : folders; +} + +function normalizeItemFolderParents( + items: T[], + validFolderIds: ReadonlySet, + itemKind: string, +): T[] { + if (items.length === 0 || validFolderIds.size === 0) { + const needsRootRepair = items.some((item) => Boolean(item.parentId)); + if (!needsRootRepair) return items; + } + + let repairedCount = 0; + const normalized = items.map((item) => { + if (!item.parentId || validFolderIds.has(item.parentId)) return item; + repairedCount += 1; + return { ...item, parentId: null }; + }); + + if (repairedCount > 0) { + log.warn('Recovered media panel items with missing folder parents', { + itemKind, + repairedCount, + total: items.length, + }); + } + + return repairedCount > 0 ? normalized : items; +} + function hydrateFlashBoardFromProject(data: ProjectFlashBoardState): void { const boards: FlashBoard[] = data.boards.map((board) => { const nodes: FlashBoardNode[] = board.nodes.map((node) => { @@ -880,7 +949,7 @@ export async function loadProjectToStores(): Promise { itemsTotal: projectData.media.length, blocking: true, }); - const files = await convertProjectMediaToStore(projectData.media, { + const loadedFiles = await convertProjectMediaToStore(projectData.media, { hydrateFiles, deferCacheChecks: true, onProgress: (done, total, name) => { @@ -896,28 +965,34 @@ export async function loadProjectToStores(): Promise { }); }, }); + const folders = normalizeFolderParents(convertProjectFolderToStore(projectData.folders)); + const validFolderIds = new Set(folders.map((folder) => folder.id)); + const files = normalizeItemFolderParents(loadedFiles, validFolderIds, 'files'); setProjectLoadProgress({ phase: 'timeline', percent: 40, message: 'Restoring timeline', blocking: true, }); - const compositions = convertProjectCompositionToStore( - projectData.compositions, - projectData.uiState?.compositionViewState + const compositions = normalizeItemFolderParents( + convertProjectCompositionToStore( + projectData.compositions, + projectData.uiState?.compositionViewState + ), + validFolderIds, + 'compositions', ); - const folders = convertProjectFolderToStore(projectData.folders); // Clear timeline first const timelineStore = useTimelineStore.getState(); timelineStore.clearTimeline(); // Restore generated media items - const textItems = projectData.textItems || []; - const solidItems = projectData.solidItems || []; - const meshItems = projectData.meshItems || []; - const cameraItems = projectData.cameraItems || []; - const splatEffectorItems = projectData.splatEffectorItems || []; + const textItems = normalizeItemFolderParents(projectData.textItems || [], validFolderIds, 'text items'); + const solidItems = normalizeItemFolderParents(projectData.solidItems || [], validFolderIds, 'solid items'); + const meshItems = normalizeItemFolderParents(projectData.meshItems || [], validFolderIds, 'mesh items'); + const cameraItems = normalizeItemFolderParents(projectData.cameraItems || [], validFolderIds, 'camera items'); + const splatEffectorItems = normalizeItemFolderParents(projectData.splatEffectorItems || [], validFolderIds, 'splat effector items'); // Update media store useMediaStore.setState({ @@ -1109,6 +1184,12 @@ async function applyProjectRestoreMediaUpdate( async function runPostLoadRestoration(projectData: ProjectFile, hydrateFiles: boolean): Promise { try { + if (!hydrateFiles) { + log.info('Skipping eager post-load restoration for native backend; media details are restored lazily'); + completeProjectLoadProgress('Project ready'); + return; + } + if (hydrateFiles) { setProjectLoadProgress({ phase: 'relink', @@ -1117,70 +1198,58 @@ async function runPostLoadRestoration(projectData: ProjectFile, hydrateFiles: bo blocking: false, }); await autoRelinkFromRawFolder(); - } else { - log.info('Skipping eager auto-relink for native backend during initial project load'); } await yieldToBrowser(); - setProjectLoadProgress({ - phase: 'thumbnails', - percent: 78, - message: 'Restoring thumbnails', - blocking: false, - }); - await restoreMediaThumbnails((done, total, name) => { - const ratio = total > 0 ? done / total : 1; - setProjectLoadProgress({ - phase: 'thumbnails', - percent: 78 + ratio * 8, - message: 'Restoring thumbnails', - detail: name, - itemsDone: done, - itemsTotal: total, - blocking: false, - }); - }); + log.info('Skipping eager thumbnail restoration; media panel restores visible thumbnails lazily'); - setProjectLoadProgress({ - phase: 'metadata', - percent: 86, - message: 'Refreshing media metadata', - blocking: false, - }); - await refreshMediaMetadata((done, total, name) => { - const ratio = total > 0 ? done / total : 1; + const eagerMetadataLimit = 120; + if (projectData.media.length <= eagerMetadataLimit) { setProjectLoadProgress({ phase: 'metadata', - percent: 86 + ratio * 6, + percent: 86, message: 'Refreshing media metadata', - detail: name, - itemsDone: done, - itemsTotal: total, blocking: false, }); - }); + await refreshMediaMetadata((done, total, name) => { + const ratio = total > 0 ? done / total : 1; + setProjectLoadProgress({ + phase: 'metadata', + percent: 86 + ratio * 6, + message: 'Refreshing media metadata', + detail: name, + itemsDone: done, + itemsTotal: total, + blocking: false, + }); + }); - setProjectLoadProgress({ - phase: 'caches', - percent: 92, - message: 'Checking project caches', - itemsDone: 0, - itemsTotal: projectData.media.length, - blocking: false, - }); - await restoreDeferredMediaCacheState(projectData.media, (done, total, name, itemProgress) => { - const ratio = total > 0 ? (done + (itemProgress ?? 0)) / total : 1; setProjectLoadProgress({ phase: 'caches', - percent: 92 + ratio * 7, + percent: 92, message: 'Checking project caches', - detail: name, - itemsDone: done, - itemsTotal: total, + itemsDone: 0, + itemsTotal: projectData.media.length, blocking: false, }); - }); + await restoreDeferredMediaCacheState(projectData.media, (done, total, name, itemProgress) => { + const ratio = total > 0 ? (done + (itemProgress ?? 0)) / total : 1; + setProjectLoadProgress({ + phase: 'caches', + percent: 92 + ratio * 7, + message: 'Checking project caches', + detail: name, + itemsDone: done, + itemsTotal: total, + blocking: false, + }); + }); + } else { + log.info('Skipping eager metadata/cache restoration for large project', { + mediaCount: projectData.media.length, + }); + } completeProjectLoadProgress('Project ready'); } catch (error) { @@ -1267,77 +1336,6 @@ async function refreshMediaMetadata( log.info('Media metadata refresh complete'); } -/** - * Restore thumbnails for media files after project load. - * Checks project folder first, then regenerates from file if needed. - */ -async function restoreMediaThumbnails( - onProgress?: (done: number, total: number, name: string) => void, -): Promise { - const mediaState = useMediaStore.getState(); - // Find files that need thumbnails (video/image files without thumbnailUrl) - const filesToRestore = mediaState.files.filter(f => - f.file && !f.thumbnailUrl && (f.type === 'video' || f.type === 'image') - ); - - if (filesToRestore.length === 0) { - log.debug('No thumbnails need restoration'); - return; - } - - log.info(`Restoring thumbnails for ${filesToRestore.length} files...`); - - // Process in batches to avoid overwhelming browser - const batchSize = 5; - let completed = 0; - for (let i = 0; i < filesToRestore.length; i += batchSize) { - const batch = filesToRestore.slice(i, i + batchSize); - - await Promise.all(batch.map(async (mediaFile) => { - if (!mediaFile.file) { - completed++; - onProgress?.(completed, filesToRestore.length, mediaFile.name); - return; - } - - try { - let thumbnailUrl: string | undefined; - - // First try to get from project folder if we have a hash - if (mediaFile.fileHash && projectFileService.isProjectOpen()) { - const existingBlob = await projectFileService.getThumbnail(mediaFile.fileHash); - if (existingBlob && existingBlob.size > 0) { - thumbnailUrl = URL.createObjectURL(existingBlob); - log.debug(`Restored thumbnail from project: ${mediaFile.name}`); - } - } - - // If not found in project, regenerate from file - if (!thumbnailUrl) { - thumbnailUrl = await createThumbnail(mediaFile.file, mediaFile.type as 'video' | 'image'); - log.debug(`Regenerated thumbnail: ${mediaFile.name}`); - } - - if (thumbnailUrl) { - await applyProjectRestoreMediaUpdate((state) => ({ - files: state.files.map((f) => - f.id === mediaFile.id ? { ...f, thumbnailUrl } : f - ), - })); - } - } catch (e) { - log.warn(`Failed to restore thumbnail for: ${mediaFile.name}`, e); - } finally { - completed++; - onProgress?.(completed, filesToRestore.length, mediaFile.name); - } - })); - await yieldToBrowser(); - } - - log.info('Thumbnail restoration complete'); -} - async function restoreDeferredMediaCacheState( projectMedia: ProjectMediaFile[], onProgress?: (done: number, total: number, name: string, itemProgress?: number) => void, diff --git a/src/services/project/projectSave.ts b/src/services/project/projectSave.ts index 6900672c..0fd6d411 100644 --- a/src/services/project/projectSave.ts +++ b/src/services/project/projectSave.ts @@ -19,6 +19,7 @@ import type { } from '../../stores/flashboardStore/types'; import { projectFileService, + type ProjectFile, type ProjectMediaFile, type ProjectComposition, type ProjectTrack, @@ -363,6 +364,35 @@ function readMediaPanelViewMode(): 'classic' | 'icons' | 'board' | undefined { return undefined; } +type MediaStoreSnapshot = ReturnType; + +function countParentedProjectMedia(media: ProjectMediaFile[]): number { + return media.reduce((count, file) => count + (file.folderId ? 1 : 0), 0); +} + +function countParentedStoreMedia(files: MediaFile[]): number { + return files.reduce((count, file) => count + (file.parentId ? 1 : 0), 0); +} + +function looksLikeDefaultStoreComposition(state: MediaStoreSnapshot): boolean { + if (state.compositions.length !== 1) return false; + const [composition] = state.compositions; + return composition?.id === 'comp-1' && composition.name === 'Comp 1'; +} + +function shouldBlockDestructiveStoreSync(projectData: ProjectFile, state: MediaStoreSnapshot): boolean { + const projectMediaCount = projectData.media.length; + if (projectMediaCount < 50 || state.files.length !== projectMediaCount) return false; + + const projectParentedMedia = countParentedProjectMedia(projectData.media); + const storeParentedMedia = countParentedStoreMedia(state.files); + const lostMediaParents = projectParentedMedia >= 20 && storeParentedMedia <= Math.max(1, Math.floor(projectParentedMedia * 0.05)); + const lostMostFolders = projectData.folders.length >= 5 && state.folders.length <= Math.max(1, Math.floor(projectData.folders.length * 0.1)); + const collapsedCompositions = projectData.compositions.length > 1 && looksLikeDefaultStoreComposition(state); + + return lostMediaParents && lostMostFolders && collapsedCompositions; +} + // ============================================ // SYNC & SAVE // ============================================ @@ -387,6 +417,19 @@ export async function syncStoresToProject(): Promise { // Get fresh state after update const freshState = useMediaStore.getState(); + const projectData = projectFileService.getProjectData(); + + if (projectData && shouldBlockDestructiveStoreSync(projectData, freshState)) { + log.warn('Skipped destructive project sync from stale store state', { + projectMediaCount: projectData.media.length, + storeMediaCount: freshState.files.length, + projectFolderCount: projectData.folders.length, + storeFolderCount: freshState.folders.length, + projectCompositionCount: projectData.compositions.length, + storeCompositionCount: freshState.compositions.length, + }); + return; + } // Update project file data projectFileService.updateMedia(convertMediaFiles(freshState.files)); @@ -394,7 +437,6 @@ export async function syncStoresToProject(): Promise { projectFileService.updateFolders(convertFolders(freshState.folders)); // Update active state - const projectData = projectFileService.getProjectData(); if (projectData) { projectData.activeCompositionId = freshState.activeCompositionId; projectData.openCompositionIds = freshState.openCompositionIds; diff --git a/src/stores/mediaStore/slices/fileManageSlice.ts b/src/stores/mediaStore/slices/fileManageSlice.ts index 9e1d25b4..507cc973 100644 --- a/src/stores/mediaStore/slices/fileManageSlice.ts +++ b/src/stores/mediaStore/slices/fileManageSlice.ts @@ -12,15 +12,17 @@ import { engine } from '../../../engine/WebGPUEngine'; import { thumbnailCacheService } from '../../../services/thumbnailCacheService'; import { lottieRuntimeManager } from '../../../services/vectorAnimation/LottieRuntimeManager'; import { readLottieMetadata } from '../../../services/vectorAnimation/lottieMetadata'; -import { createThumbnail } from '../helpers/thumbnailHelpers'; +import { createThumbnail, handleThumbnailDedup } from '../helpers/thumbnailHelpers'; import { resolveGaussianSplatSequenceData } from '../../../utils/gaussianSplatSequence'; const log = Logger.create('Reload'); const isBlobUrl = (value?: string): value is string => typeof value === 'string' && value.startsWith('blob:'); +const activeThumbnailRequests = new Map>(); export interface FileManageActions { removeFile: (id: string) => void; renameFile: (id: string, name: string) => void; + ensureFileThumbnail: (id: string) => Promise; refreshFileUrls: (id: string, options?: { refreshThumbnail?: boolean }) => Promise; reloadFile: (id: string) => Promise; reloadAllFiles: () => Promise; @@ -44,6 +46,78 @@ export const createFileManageSlice: MediaSliceCreator = (set, })); }, + ensureFileThumbnail: async (id: string) => { + const existingRequest = activeThumbnailRequests.get(id); + if (existingRequest) return existingRequest; + + const request = (async () => { + const mediaFile = get().files.find((f) => f.id === id); + if (!mediaFile?.id || mediaFile.thumbnailUrl || mediaFile.isImporting) { + return Boolean(mediaFile?.thumbnailUrl); + } + if (mediaFile.type !== 'image' && mediaFile.type !== 'video') { + return false; + } + + let thumbnailUrl: string | undefined; + + try { + if (mediaFile.fileHash && projectFileService.isProjectOpen()) { + const existingBlob = await projectFileService.getThumbnail(mediaFile.fileHash); + if (existingBlob && existingBlob.size > 0) { + thumbnailUrl = URL.createObjectURL(existingBlob); + } + } + + let sourceFile = mediaFile.file; + if (!thumbnailUrl && !sourceFile && mediaFile.projectPath && projectFileService.isProjectOpen()) { + const result = await projectFileService.getFileFromRaw(mediaFile.projectPath); + sourceFile = result?.file; + } + + if (!thumbnailUrl && sourceFile) { + const generatedThumbnail = await createThumbnail(sourceFile, mediaFile.type); + thumbnailUrl = await handleThumbnailDedup(mediaFile.fileHash, generatedThumbnail); + } + + if (!thumbnailUrl) { + return false; + } + + let applied = false; + set((state) => ({ + files: state.files.map((file) => { + if (file.id !== id) return file; + if (file.thumbnailUrl) return file; + applied = true; + return { ...file, thumbnailUrl }; + }), + })); + + if (!applied && isBlobUrl(thumbnailUrl)) { + URL.revokeObjectURL(thumbnailUrl); + } + + return applied || Boolean(get().files.find((file) => file.id === id)?.thumbnailUrl); + } catch (error) { + log.warn('Failed to ensure media thumbnail', { + id, + name: mediaFile.name, + error, + }); + if (thumbnailUrl && isBlobUrl(thumbnailUrl)) { + URL.revokeObjectURL(thumbnailUrl); + } + return false; + } + })().finally(() => { + activeThumbnailRequests.delete(id); + }); + + activeThumbnailRequests.set(id, request); + return request; + }, + refreshFileUrls: async (id: string, options) => { const mediaFile = get().files.find((f) => f.id === id); if (!mediaFile) return false; diff --git a/tests/stores/mediaStore/fileManageSlice.test.ts b/tests/stores/mediaStore/fileManageSlice.test.ts index f47a7cd5..989a652b 100644 --- a/tests/stores/mediaStore/fileManageSlice.test.ts +++ b/tests/stores/mediaStore/fileManageSlice.test.ts @@ -11,10 +11,12 @@ import type { MediaState, MediaFile, MediaFolder, TextItem, SolidItem, Compositi const thumbnailMocks = vi.hoisted(() => ({ createThumbnail: vi.fn(async () => 'blob:http://localhost/refreshed-thumb'), + handleThumbnailDedup: vi.fn(async (_fileHash: string | undefined, thumbnailUrl: string | undefined) => thumbnailUrl), })); vi.mock('../../../src/stores/mediaStore/helpers/thumbnailHelpers', () => ({ createThumbnail: thumbnailMocks.createThumbnail, + handleThumbnailDedup: thumbnailMocks.handleThumbnailDedup, })); import { createFileManageSlice, type FileManageActions } from '../../../src/stores/mediaStore/slices/fileManageSlice'; @@ -226,6 +228,8 @@ describe('MediaStore - File Management', () => { store = createTestStore(); thumbnailMocks.createThumbnail.mockClear(); thumbnailMocks.createThumbnail.mockResolvedValue('blob:http://localhost/refreshed-thumb'); + thumbnailMocks.handleThumbnailDedup.mockClear(); + thumbnailMocks.handleThumbnailDedup.mockImplementation(async (_fileHash: string | undefined, thumbnailUrl: string | undefined) => thumbnailUrl); }); // --- Adding media files --- diff --git a/tests/unit/mediaPanelSourceMonitor.test.tsx b/tests/unit/mediaPanelSourceMonitor.test.tsx index b49ab130..6bfe291e 100644 --- a/tests/unit/mediaPanelSourceMonitor.test.tsx +++ b/tests/unit/mediaPanelSourceMonitor.test.tsx @@ -111,6 +111,7 @@ function createMediaState(): MockMediaState { setLabelColor: vi.fn(), importGaussianSplat: vi.fn(), refreshFileUrls: vi.fn(), + ensureFileThumbnail: vi.fn(async () => false), setSourceMonitorFile: vi.fn((id: string | null) => { state.sourceMonitorFileId = id; if (id !== null) { diff --git a/tests/unit/motionDesignRendering.test.ts b/tests/unit/motionDesignRendering.test.ts index ef2ba992..1c77c7f0 100644 --- a/tests/unit/motionDesignRendering.test.ts +++ b/tests/unit/motionDesignRendering.test.ts @@ -4,7 +4,7 @@ import { createDefaultMotionLayerDefinition, createStrokeAppearance, } from '../../src/types/motionDesign'; -import { createMotionUniformArray } from '../../src/engine/motion/MotionBuffers'; +import { createMotionInstanceArray, createMotionUniformArray } from '../../src/engine/motion/MotionBuffers'; import { getMotionRenderSize } from '../../src/engine/motion/MotionTypes'; import { getInterpolatedMotionLayer } from '../../src/utils/motionInterpolation'; import { createTestTimelineStore } from '../helpers/storeFactory'; @@ -49,7 +49,7 @@ describe('motion design rendering helpers', () => { alignment: 'outside', }); - expect(getMotionRenderSize(motion)).toEqual({ + expect(getMotionRenderSize(motion)).toMatchObject({ width: 124, height: 74, strokePadding: 12, @@ -76,6 +76,42 @@ describe('motion design rendering helpers', () => { expect(Array.from(uniforms.slice(16, 19))).toEqual([8, 1, 0]); }); + it('sizes and packs grid replicator instances for motion shapes', () => { + const motion = createDefaultMotionLayerDefinition('shape', { + size: { w: 100, h: 50 }, + }); + if (motion.replicator?.layout.mode === 'grid') { + motion.replicator.enabled = true; + motion.replicator.layout.count = { x: 3, y: 2 }; + motion.replicator.layout.spacing = { x: 50, y: 80 }; + motion.replicator.offset.opacity = 0.75; + } + + const size = getMotionRenderSize(motion); + const instances = createMotionInstanceArray(size); + + expect(size).toMatchObject({ + width: 200, + height: 130, + replicator: { + enabled: true, + countX: 3, + countY: 2, + spacingX: 50, + spacingY: 80, + instanceCount: 6, + }, + }); + expect(Array.from(instances)).toEqual([ + -50, -40, 1, 0, + 0, -40, 0.75, 0, + 50, -40, 0.5625, 0, + -50, 40, 0.421875, 0, + 0, 40, 0.31640625, 0, + 50, 40, 0.2373046875, 0, + ]); + }); + it('interpolates numeric motion properties through the property registry', () => { const clip = makeMotionClip(createDefaultMotionLayerDefinition('shape', { size: { w: 100, h: 50 }, @@ -126,4 +162,37 @@ describe('motion design rendering helpers', () => { expect(fill.color.b).toBeCloseTo(0.6, 3); } }); + + it('adds rectangle and ellipse motion shape clips only on video tracks', () => { + const store = createTestTimelineStore(); + const rectangleId = store.getState().addMotionShapeClip('video-1', 1, { + primitive: 'rectangle', + duration: 3, + size: { w: 320, h: 180 }, + name: 'Motion Rectangle', + }); + const ellipseId = store.getState().addMotionShapeClip('video-1', 4, { + primitive: 'ellipse', + duration: 2, + name: 'Motion Ellipse', + }); + const invalidId = store.getState().addMotionShapeClip('audio-1', 0, { + primitive: 'rectangle', + }); + + const rectangle = store.getState().clips.find((clip) => clip.id === rectangleId); + const ellipse = store.getState().clips.find((clip) => clip.id === ellipseId); + + expect(rectangleId).toBeTruthy(); + expect(ellipseId).toBeTruthy(); + expect(invalidId).toBeNull(); + expect(rectangle?.source?.type).toBe('motion-shape'); + expect(rectangle?.motion?.shape?.primitive).toBe('rectangle'); + expect(rectangle?.motion?.shape?.size).toEqual({ w: 320, h: 180 }); + expect(rectangle?.startTime).toBe(1); + expect(rectangle?.duration).toBe(3); + expect(ellipse?.name).toBe('Motion Ellipse'); + expect(ellipse?.motion?.shape?.primitive).toBe('ellipse'); + expect(store.getState().clips).toHaveLength(2); + }); }); diff --git a/tests/unit/performanceStats.test.ts b/tests/unit/performanceStats.test.ts index baf85332..b3cbb273 100644 --- a/tests/unit/performanceStats.test.ts +++ b/tests/unit/performanceStats.test.ts @@ -20,4 +20,18 @@ describe('PerformanceStats', () => { const snapshot = stats.getStats(true); expect(snapshot.fps).toBe(0); }); + + it('does not report stale per-second drops while idle', () => { + const stats = new PerformanceStats(); + stats.recordRafGap(100); + stats.resetPerSecondCounters(); + + const activeSnapshot = stats.getStats(false); + expect(activeSnapshot.drops.lastSecond).toBeGreaterThan(0); + + const idleSnapshot = stats.getStats(true); + expect(idleSnapshot.drops.count).toBeGreaterThan(0); + expect(idleSnapshot.drops.lastSecond).toBe(0); + expect(idleSnapshot.drops.reason).toBe('none'); + }); }); diff --git a/tests/unit/projectMediaPersistence.test.ts b/tests/unit/projectMediaPersistence.test.ts index 229d87e0..dacba7c7 100644 --- a/tests/unit/projectMediaPersistence.test.ts +++ b/tests/unit/projectMediaPersistence.test.ts @@ -150,6 +150,7 @@ vi.mock('../../src/stores/mediaStore/helpers/mediaInfoHelpers', () => ({ vi.mock('../../src/stores/mediaStore/helpers/thumbnailHelpers', () => ({ createThumbnail: vi.fn(async () => undefined), + handleThumbnailDedup: vi.fn(async (_fileHash: string | undefined, thumbnailUrl: string | undefined) => thumbnailUrl), })); vi.mock('../../src/engine/WebGPUEngine', () => ({ @@ -1004,6 +1005,181 @@ describe('project media persistence', () => { })); }); + it('moves media panel items with missing folder parents back to root on load', async () => { + mocks.getProjectData.mockReturnValue({ + media: [{ + id: 'media-orphan', + name: 'orphan.png', + type: 'image', + sourcePath: 'E:/project/Raw/orphan.png', + projectPath: 'Raw/orphan.png', + duration: 0, + width: 1024, + height: 768, + hasProxy: false, + folderId: 'missing-folder', + importedAt: new Date(1).toISOString(), + }], + compositions: [{ + id: 'comp-orphan', + name: 'Comp Orphan', + width: 1920, + height: 1080, + frameRate: 30, + duration: 60, + backgroundColor: '#000000', + folderId: 'missing-folder', + tracks: [], + clips: [], + markers: [], + }], + folders: [{ + id: 'folder-cycle', + name: 'Broken Folder', + parentId: 'folder-cycle', + }], + textItems: [{ + id: 'text-orphan', + name: 'Text Orphan', + type: 'text', + parentId: 'missing-folder', + createdAt: 1, + text: 'hello', + fontFamily: 'Arial', + fontSize: 64, + color: '#ffffff', + duration: 5, + }], + settings: { width: 1920, height: 1080, frameRate: 30 }, + activeCompositionId: null, + openCompositionIds: [], + expandedFolderIds: [], + slotAssignments: {}, + uiState: {}, + }); + + const { loadProjectToStores } = await import('../../src/services/project/projectLoad'); + await loadProjectToStores(); + + expect(mocks.mediaSetState).toHaveBeenCalledWith(expect.objectContaining({ + files: [ + expect.objectContaining({ + id: 'media-orphan', + parentId: null, + }), + ], + compositions: [ + expect.objectContaining({ + id: 'comp-orphan', + parentId: null, + }), + ], + folders: [ + expect.objectContaining({ + id: 'folder-cycle', + parentId: null, + }), + ], + textItems: [ + expect.objectContaining({ + id: 'text-orphan', + parentId: null, + }), + ], + })); + }); + + it('skips destructive project sync from a stale default store after loading a large project', async () => { + const persistedMedia = Array.from({ length: 60 }, (_, index) => ({ + id: `media-${index}`, + name: `media-${index}.png`, + type: 'image' as const, + sourcePath: `E:/project/Raw/media-${index}.png`, + projectPath: `Raw/media-${index}.png`, + width: 1024, + height: 768, + hasProxy: false, + folderId: `folder-${index % 10}`, + importedAt: new Date(index + 1).toISOString(), + })); + + mocks.mediaState.files = persistedMedia.map((file) => ({ + id: file.id, + name: file.name, + type: 'image', + parentId: null, + createdAt: 1, + url: '', + projectPath: file.projectPath, + })); + mocks.mediaState.folders = []; + mocks.mediaState.compositions = [{ + id: 'comp-1', + name: 'Comp 1', + type: 'composition', + parentId: null, + createdAt: 1, + width: 1920, + height: 1080, + frameRate: 30, + duration: 60, + backgroundColor: '#000000', + timelineData: { tracks: [], clips: [] }, + }]; + mocks.mediaState.activeCompositionId = 'comp-1'; + mocks.mediaState.openCompositionIds = ['comp-1']; + + mocks.getProjectData.mockReturnValue({ + media: persistedMedia, + compositions: [ + { + id: 'comp-old-1', + name: 'Comp 1', + width: 1920, + height: 1080, + frameRate: 30, + duration: 60, + backgroundColor: '#000000', + folderId: null, + tracks: [], + clips: [], + markers: [], + }, + { + id: 'comp-old-2', + name: 'Comp 2', + width: 1920, + height: 1080, + frameRate: 30, + duration: 60, + backgroundColor: '#000000', + folderId: null, + tracks: [], + clips: [], + markers: [], + }, + ], + folders: Array.from({ length: 10 }, (_, index) => ({ + id: `folder-${index}`, + name: `Folder ${index}`, + parentId: null, + })), + settings: { width: 1920, height: 1080, frameRate: 30 }, + activeCompositionId: 'comp-old-1', + openCompositionIds: ['comp-old-1', 'comp-old-2'], + expandedFolderIds: Array.from({ length: 10 }, (_, index) => `folder-${index}`), + slotAssignments: {}, + uiState: {}, + }); + + const { syncStoresToProject } = await import('../../src/services/project/projectSave'); + await syncStoresToProject(); + + expect(mocks.updateMedia).not.toHaveBeenCalled(); + expect(mocks.updateCompositions).not.toHaveBeenCalled(); + expect(mocks.updateFolders).not.toHaveBeenCalled(); + }); + it('restores transport MIDI bindings from project uiState', async () => { mocks.getProjectData.mockReturnValue({ media: [], From 94242343101e0d4fce2d6efd101207ae133ef1a2 Mon Sep 17 00:00:00 2001 From: Sportinger Date: Wed, 6 May 2026 12:18:45 +0200 Subject: [PATCH 3/3] Improve playback diagnostics and scrub feedback --- src/changelog-data.json | 21 +++ src/components/preview/StatsOverlay.tsx | 123 +++++++++++++++++- src/engine/render/LayerCollector.ts | 59 +++++++-- src/engine/render/RenderDispatcher.ts | 51 ++++++-- src/engine/render/RenderLoop.ts | 4 +- src/services/aiTools/definitions/playback.ts | 4 + src/services/aiTools/handlers/playback.ts | 11 ++ src/services/layerBuilder/VideoSyncManager.ts | 75 +++++++++-- src/version.ts | 6 +- tests/unit/layerCollector.test.ts | 12 +- tests/unit/renderDispatcher.test.ts | 6 +- 11 files changed, 323 insertions(+), 49 deletions(-) diff --git a/src/changelog-data.json b/src/changelog-data.json index 5621a1fc..d470b6f8 100644 --- a/src/changelog-data.json +++ b/src/changelog-data.json @@ -1,4 +1,25 @@ [ + { + "date": "2026-05-06", + "type": "improve", + "title": "Effective Playback FPS Diagnostics", + "description": "The stats overlay now reports effective FPS from the weakest active playback link, alongside render, preview, and decoder inputs so bottlenecks are visible during playback and scrubbing.", + "section": "Playback / Diagnostics" + }, + { + "date": "2026-05-06", + "type": "improve", + "title": "HTML Video Playback and Scrub Feedback", + "description": "HTML-video playback and no-proxy scrubbing now use faster render cadence, tighter drag fallback drift, improved seeked-frame caching, and empty-frame hold behavior to reduce black flashes while keeping WebCodecs disabled.", + "section": "Playback / Scrubbing" + }, + { + "date": "2026-05-06", + "type": "improve", + "title": "Scripted Scrub Diagnostics", + "description": "The AI playback tools can reset and return per-run diagnostics for scripted scrub runs, making long mixed-speed scrub tests easier to compare.", + "section": "Developer Tools" + }, { "date": "2026-05-02", "type": "improve", diff --git a/src/components/preview/StatsOverlay.tsx b/src/components/preview/StatsOverlay.tsx index 78efed27..8dc30bd1 100644 --- a/src/components/preview/StatsOverlay.tsx +++ b/src/components/preview/StatsOverlay.tsx @@ -11,11 +11,81 @@ interface StatsOverlayProps { onToggle: () => void; } +type EffectiveFpsCandidate = { + label: string; + value: number; +}; + +function normalizeFps(value: number | undefined): number | null { + if (typeof value !== 'number' || !Number.isFinite(value)) { + return null; + } + return Math.max(0, Math.round(value)); +} + +function getFpsColor(fps: number, targetFps: number): string { + const target = Math.max(1, targetFps); + if (fps >= target * 0.9) return '#4f4'; + if (fps >= target * 0.5) return '#ff4'; + return '#f44'; +} + +function getEffectiveFps(stats: EngineStats): { + value: number; + weakestLink: string; + candidates: EffectiveFpsCandidate[]; +} { + if (stats.isIdle) { + return { value: 0, weakestLink: 'Idle', candidates: [] }; + } + + const targetFps = normalizeFps(stats.targetFps) ?? 60; + const candidates: EffectiveFpsCandidate[] = [ + { label: 'Target', value: targetFps }, + ]; + + const renderFps = normalizeFps(stats.fps); + if (renderFps !== null) { + candidates.push({ label: 'Render', value: renderFps }); + } + + const playback = stats.playback; + if (playback && playback.previewFrames > 0) { + const previewRenderFps = normalizeFps(playback.previewRenderFps); + const previewUpdateFps = normalizeFps(playback.previewUpdateFps); + if (previewRenderFps !== null) { + candidates.push({ label: 'Preview render', value: previewRenderFps }); + } + if (previewUpdateFps !== null) { + candidates.push({ label: 'Preview update', value: previewUpdateFps }); + } + } + + if (playback && playback.frameEvents > 0) { + const decoderFps = normalizeFps(playback.cadenceFps); + if (decoderFps !== null) { + candidates.push({ label: 'Decoder', value: decoderFps }); + } + } + + const weakest = candidates.reduce((lowest, candidate) => ( + candidate.value < lowest.value ? candidate : lowest + )); + + return { + value: weakest.value, + weakestLink: weakest.label, + candidates, + }; +} + export function StatsOverlay({ stats, resolution, expanded, onToggle }: StatsOverlayProps) { const gpuInfo = useEngineStore(s => s.gpuInfo); const splatSequence = stats.renderDispatcher?.splatSequence; const splatVisualFps = splatSequence?.visualFrameChangesLastSecond; - const fpsColor = stats.fps >= 55 ? '#4f4' : stats.fps >= 30 ? '#ff4' : '#f44'; + const effectiveFps = useMemo(() => getEffectiveFps(stats), [stats]); + const effectiveFpsColor = getFpsColor(effectiveFps.value, stats.targetFps); + const fpsColor = getFpsColor(stats.fps, stats.targetFps); const splatFpsColor = splatVisualFps === undefined ? '#888' @@ -71,8 +141,14 @@ export function StatsOverlay({ stats, resolution, expanded, onToggle }: StatsOve > {!stats.isIdle && ( <> - {stats.fps} - FPS + {effectiveFps.value} + Eff + + R {stats.fps} + )} {stats.isIdle && ( @@ -120,8 +196,13 @@ export function StatsOverlay({ stats, resolution, expanded, onToggle }: StatsOve return (
- {stats.fps} - / {stats.targetFps} FPS + {effectiveFps.value} + effective FPS + {!stats.isIdle && ( + + Render {stats.fps} / {stats.targetFps} + + )} {splatSequence && ( Splat {splatVisualFps} FPS @@ -136,6 +217,38 @@ export function StatsOverlay({ stats, resolution, expanded, onToggle }: StatsOve
+
+ Effective FPS + + {effectiveFps.value} + +
+
+ Weakest Link + + {effectiveFps.weakestLink} + +
+
+ Render FPS + {stats.fps} +
+ {stats.playback && stats.playback.previewFrames > 0 && ( +
+ Preview FPS + + update {stats.playback.previewUpdateFps} / render {stats.playback.previewRenderFps} + +
+ )} + {effectiveFps.candidates.length > 0 && ( +
+ FPS Inputs + + {effectiveFps.candidates.map((candidate) => `${candidate.label} ${candidate.value}`).join(' | ')} + +
+ )}
Frame Gap 20 ? '#ff4' : '#aaa' }}> diff --git a/src/engine/render/LayerCollector.ts b/src/engine/render/LayerCollector.ts index 68a3daf4..d199fd1d 100644 --- a/src/engine/render/LayerCollector.ts +++ b/src/engine/render/LayerCollector.ts @@ -49,9 +49,9 @@ export class LayerCollector { // to the WebCodecs path (which may not have the correct frame yet). private scrubGraceUntil = 0; private static readonly SCRUB_GRACE_MS = 150; // ~9 frames at 60fps - private static readonly HTML_HOLD_RECOVERY_MS = 120; - private static readonly MAX_DRAG_FALLBACK_DRIFT_SECONDS = 1.2; - private static readonly MAX_DRAG_LIVE_IMPORT_DRIFT_SECONDS = 0.9; + private static readonly HTML_HOLD_RECOVERY_MS = 80; + private static readonly MAX_DRAG_FALLBACK_DRIFT_SECONDS = 0.35; + private static readonly MAX_DRAG_LIVE_IMPORT_DRIFT_SECONDS = 0.35; private isPendingWebCodecsFrameStable( provider: NonNullable['webCodecsPlayer'] | undefined @@ -861,13 +861,14 @@ export class LayerCollector { const presentedDriftSeconds = hasConfirmedPresentedFrame ? Math.abs(lastPresentedTime - targetTime) : undefined; + const currentTimeDriftSeconds = Math.abs(currentTime - targetTime); const awaitingPausedTargetFrame = hasPresentedOwnerMismatch || !deps.isPlaying && !isDragging && (!isSettling && (!hasConfirmedPresentedFrame || Math.abs(lastPresentedTime - targetTime) > 0.05)); - const cacheSearchDistanceFrames = isDragging ? 12 : 6; + const cacheSearchDistanceFrames = isDragging ? 10 : 6; const lastSameClipFrame = this.getDragHoldFrame(layer, video, deps); const dragHoldFrame = isDragging ? this.isFrameNearTarget( @@ -883,7 +884,15 @@ export class LayerCollector { const emergencyHoldFrame = dragHoldFrame; const sameClipHoldFrame = (isDragging || isSettling || awaitingPausedTargetFrame || video.seeking) - ? lastSameClipFrame + ? this.isFrameNearTarget( + lastSameClipFrame, + targetTime, + isDragging + ? LayerCollector.MAX_DRAG_FALLBACK_DRIFT_SECONDS + : 0.35 + ) + ? lastSameClipFrame + : null : null; const safeFallback = this.getSafeLastFrameFallback(layer, video, deps, targetTime) ?? dragHoldFrame; const shouldPreferStableHold = this.shouldPreferHtmlHold(layerReuseKey, { @@ -896,10 +905,8 @@ export class LayerCollector { const allowDragLiveVideoImport = !shouldPreferStableHold && !video.seeking && - ( - !hasConfirmedPresentedFrame || - (presentedDriftSeconds ?? 0) <= LayerCollector.MAX_DRAG_LIVE_IMPORT_DRIFT_SECONDS - ); + (presentedDriftSeconds ?? currentTimeDriftSeconds) <= + LayerCollector.MAX_DRAG_LIVE_IMPORT_DRIFT_SECONDS; const allowLiveVideoImport = !shouldPreferStableHold && !hasPresentedOwnerMismatch && @@ -1119,6 +1126,28 @@ export class LayerCollector { }; } + if (isDragging && !allowLiveVideoImport) { + const cachedFrame = + deps.scrubbingCache?.getCachedFrameEntry(video.src, targetTime) ?? + deps.scrubbingCache?.getNearestCachedFrameEntry(video.src, targetTime, cacheSearchDistanceFrames); + if (cachedFrame) { + this.armHtmlHold(layerReuseKey); + this.traceScrubPath(layer, 'scrub-cache', video, targetTime, lastPresentedTime); + this.currentDecoder = 'HTMLVideo(scrub-cache)'; + return { + layer, + isVideo: false, + externalTexture: null, + textureView: cachedFrame.view, + sourceWidth: video.videoWidth, + sourceHeight: video.videoHeight, + displayedMediaTime: cachedFrame.mediaTime, + targetMediaTime: targetTime, + previewPath: 'scrub-cache', + }; + } + } + // Fallback to cache if (safeFallback) { this.armHtmlHold(layerReuseKey); @@ -1193,7 +1222,7 @@ export class LayerCollector { // Video not ready - try cache const targetTime = this.getTargetVideoTime(layer, video); const isDragging = useTimelineStore.getState().isDraggingPlayhead; - const cacheSearchDistanceFrames = isDragging ? 12 : 6; + const cacheSearchDistanceFrames = isDragging ? 10 : 6; const isSettling = scrubSettleState.isPending(layer.sourceClipId); const lastSameClipFrame = this.getDragHoldFrame(layer, video, deps); const dragHoldFrame = isSettling @@ -1222,7 +1251,15 @@ export class LayerCollector { : dragHoldFrame; const sameClipHoldFrame = (isDragging || isSettling || video.seeking || video.readyState < 2) - ? lastSameClipFrame + ? this.isFrameNearTarget( + lastSameClipFrame, + targetTime, + isDragging + ? LayerCollector.MAX_DRAG_FALLBACK_DRIFT_SECONDS + : 0.35 + ) + ? lastSameClipFrame + : null : null; const safeFallback = this.getSafeLastFrameFallback(layer, video, deps, targetTime) ?? dragHoldFrame; const cachedFrame = diff --git a/src/engine/render/RenderDispatcher.ts b/src/engine/render/RenderDispatcher.ts index 9843349e..bfd8d0ea 100644 --- a/src/engine/render/RenderDispatcher.ts +++ b/src/engine/render/RenderDispatcher.ts @@ -220,8 +220,9 @@ export class RenderDispatcher { private renderTimeOverride: number | null = null; private lastPreviewSignature = ''; private lastPreviewTargetTimeMs?: number; - private static readonly MAX_DRAG_FALLBACK_DRIFT_SECONDS = 1.2; - private static readonly MAX_DRAG_LIVE_IMPORT_DRIFT_SECONDS = 0.9; + private lastPreviewDisplayedTimeMs?: number; + private static readonly MAX_DRAG_FALLBACK_DRIFT_SECONDS = 0.35; + private static readonly MAX_DRAG_LIVE_IMPORT_DRIFT_SECONDS = 0.35; private sceneRendererInitializing = false; // Native Gaussian Splat rendering state (new WebGPU path) private splatLoadingClips = new Set(); @@ -827,6 +828,7 @@ export class RenderDispatcher { this.lastPreviewSignature = signature; this.lastPreviewTargetTimeMs = targetTimeMs; + this.lastPreviewDisplayedTimeMs = displayedTimeMs; } // === MAIN RENDER === @@ -911,10 +913,32 @@ export class RenderDispatcher { // instead of flashing black. This handles transient decoder stalls on // Windows/Linux where readyState drops briefly. const isPlaying = d.renderLoop?.getIsPlaying() ?? false; - if (isPlaying && this.shouldHoldLastFrameOnEmptyPlayback(previewFallback.targetTimeMs)) { + const isDragging = useTimelineStore.getState().isDraggingPlayhead; + const emptyScrubHoldDriftMs = + typeof previewFallback.targetTimeMs === 'number' && + typeof this.lastPreviewDisplayedTimeMs === 'number' + ? Math.abs(previewFallback.targetTimeMs - this.lastPreviewDisplayedTimeMs) + : undefined; + const shouldHoldEmptyFrame = + (isPlaying && this.shouldHoldLastFrameOnEmptyPlayback(previewFallback.targetTimeMs)) || + (isDragging && this.lastRenderHadContent); + + if (shouldHoldEmptyFrame) { // Don't render anything — canvas retains previous frame automatically. // Log once so the stall is visible in telemetry. - log.debug('Holding last frame during playback stall (empty layerData)'); + log.debug('Holding last frame during empty preview frame', { + isPlaying, + isDragging, + driftMs: emptyScrubHoldDriftMs, + }); + this.recordMainPreviewFrame( + isDragging ? 'empty-hold' : 'playback-stall-hold', + undefined, + { + ...previewFallback, + displayedTimeMs: this.lastPreviewDisplayedTimeMs, + } + ); d.performanceStats.setLayerCount(0); return; } @@ -2019,13 +2043,14 @@ export class RenderDispatcher { const presentedDriftSeconds = hasConfirmedPresentedFrame ? Math.abs(lastPresentedTime - targetTime) : undefined; + const currentTimeDriftSeconds = Math.abs(video.currentTime - targetTime); const awaitingPausedTargetFrame = hasPresentedOwnerMismatch || !(d.renderLoop?.getIsPlaying() ?? false) && !isDragging && (!isSettling && (!hasConfirmedPresentedFrame || Math.abs(lastPresentedTime - targetTime) > 0.05)); - const cacheSearchDistanceFrames = isDragging ? 12 : 6; + const cacheSearchDistanceFrames = isDragging ? 10 : 6; const lastSameClipFrame = layer.sourceClipId ? scrubbingCache?.getLastFrame(video, layer.sourceClipId) ?? null : null; @@ -2044,15 +2069,21 @@ export class RenderDispatcher { const sameClipHoldFrame = !(d.renderLoop?.getIsPlaying() ?? false) && (isDragging || isSettling || awaitingPausedTargetFrame || video.seeking) - ? lastSameClipFrame + ? this.isFrameNearTarget( + lastSameClipFrame, + targetTime, + isDragging + ? RenderDispatcher.MAX_DRAG_FALLBACK_DRIFT_SECONDS + : 0.35 + ) + ? lastSameClipFrame + : null : null; const safeFallback = this.getSafePreviewFallback(layer, video) ?? dragHoldFrame; const allowDragLiveVideoImport = !video.seeking && - ( - !hasConfirmedPresentedFrame || - (presentedDriftSeconds ?? 0) <= RenderDispatcher.MAX_DRAG_LIVE_IMPORT_DRIFT_SECONDS - ); + (presentedDriftSeconds ?? currentTimeDriftSeconds) <= + RenderDispatcher.MAX_DRAG_LIVE_IMPORT_DRIFT_SECONDS; const allowLiveVideoImport = !hasPresentedOwnerMismatch && (isPausedSettle ? hasFreshPresentedFrame : !awaitingPausedTargetFrame && diff --git a/src/engine/render/RenderLoop.ts b/src/engine/render/RenderLoop.ts index 461481fb..7fad7acc 100644 --- a/src/engine/render/RenderLoop.ts +++ b/src/engine/render/RenderLoop.ts @@ -45,8 +45,8 @@ export class RenderLoop { private readonly IDLE_TIMEOUT = 1000; // 1s before idle private readonly IDLE_SUPPRESSION_TIMEOUT = 3000; // bounded reload warmup - private readonly VIDEO_FRAME_TIME = 16.67; // ~60fps target - private readonly SCRUB_FRAME_TIME = 33; // ~30fps during scrubbing (avoids wasted renders while video seeks) + private readonly VIDEO_FRAME_TIME = 15; // ~60fps target with tolerance for 16.6ms display ticks + private readonly SCRUB_FRAME_TIME = 15; // ~60fps during scrubbing with tolerance for 16.6ms display ticks private readonly WATCHDOG_INTERVAL = 2000; // Check every 2s private readonly WATCHDOG_STALL_THRESHOLD = 3000; // 3s without render = stalled diff --git a/src/services/aiTools/definitions/playback.ts b/src/services/aiTools/definitions/playback.ts index 9af8415f..7c6f5426 100644 --- a/src/services/aiTools/definitions/playback.ts +++ b/src/services/aiTools/definitions/playback.ts @@ -72,6 +72,10 @@ export const playbackToolDefinitions: ToolDefinition[] = [ type: 'number', description: 'Optional deterministic seed for random scrubs.', }, + resetDiagnostics: { + type: 'boolean', + description: 'Reset playback diagnostics before the scrub run. Defaults to true.', + }, }, required: [], }, diff --git a/src/services/aiTools/handlers/playback.ts b/src/services/aiTools/handlers/playback.ts index cec39bf8..a5876519 100644 --- a/src/services/aiTools/handlers/playback.ts +++ b/src/services/aiTools/handlers/playback.ts @@ -522,17 +522,26 @@ export async function handleSimulateScrub( timelineStore: TimelineStore ): Promise { const wasPlaying = timelineStore.isPlaying; + const resetDiagnostics = args.resetDiagnostics !== false; if (wasPlaying) { timelineStore.pause(); } const plan = createScrubPlan(args, timelineStore.playheadPosition, timelineStore.duration); + if (resetDiagnostics) { + wcPipelineMonitor.reset(); + vfPipelineMonitor.reset(); + playbackHealthMonitor.reset(); + } + const startedAt = performance.now(); const scrubResult = await runScrubMotion( timelineStore, plan.totalDurationMs, (elapsedMs) => sampleScrubPlan(plan, elapsedMs), false ); + const endedAt = performance.now(); + const runDiagnostics = collectPlaybackRunDiagnostics(startedAt, endedAt); return { success: true, @@ -561,6 +570,8 @@ export async function handleSimulateScrub( pixelDistance: scrubResult.pixelDistance, released: true, seed: plan.seed, + resetDiagnostics, + runDiagnostics, }, }; } diff --git a/src/services/layerBuilder/VideoSyncManager.ts b/src/services/layerBuilder/VideoSyncManager.ts index 80de930a..1203a583 100644 --- a/src/services/layerBuilder/VideoSyncManager.ts +++ b/src/services/layerBuilder/VideoSyncManager.ts @@ -1042,6 +1042,23 @@ export class VideoSyncManager { this.seekedFlushArmed.add(clipId); video.addEventListener('seeked', () => { this.seekedFlushArmed.delete(clipId); + const presentedTime = video.currentTime; + engine.markVideoFramePresented(video, presentedTime, clipId); + engine.captureVideoFrameAtTime(video, presentedTime, clipId); + engine.cacheFrameAtTime(video, presentedTime); + engine.requestNewFrameRender(); + + const isDragging = useTimelineStore.getState().isDraggingPlayhead; + if (isDragging && this.queuedSeekTargets[clipId] !== undefined) { + const flush = () => this.flushQueuedSeekTarget(clipId, video, 'seeked'); + if (typeof requestAnimationFrame === 'function') { + requestAnimationFrame(flush); + } else { + setTimeout(flush, 16); + } + return; + } + this.flushQueuedSeekTarget(clipId, video, 'seeked'); }, { once: true }); } @@ -2016,15 +2033,20 @@ export class VideoSyncManager { return; } - // Skip sync during GPU surface warmup — the video is playing briefly - // to activate Chrome's GPU decoder. Don't pause or seek it. + // Skip sync during GPU surface warmup - the video is playing briefly + // to activate Chrome's GPU decoder. During drag, the active scrub target + // must keep seeking instead of waiting on/retargeting warmup. if (this.warmingUpVideos.has(video)) { - this.maybeRetargetActiveWarmup(clip.id, video, timeInfo.clipTime, ctx.now, { - isPlaying: ctx.isPlaying, - isDragging: ctx.isDraggingPlayhead, - requestRender: true, - }); - return; + if (ctx.isDraggingPlayhead) { + this.clearWarmupState(video); + } else { + this.maybeRetargetActiveWarmup(clip.id, video, timeInfo.clipTime, ctx.now, { + isPlaying: ctx.isPlaying, + isDragging: ctx.isDraggingPlayhead, + requestRender: true, + }); + return; + } } // Warmup: after page reload, video GPU surfaces are empty. @@ -2035,7 +2057,7 @@ export class VideoSyncManager { const hasSrc = !!(video.src || video.currentSrc); const warmupCooldown = this.warmupRetryCooldown.get(video); const cooldownOk = !warmupCooldown || performance.now() - warmupCooldown > 2000; - if (!ctx.isPlaying && !video.seeking && hasSrc && cooldownOk && + if (!ctx.isDraggingPlayhead && !ctx.isPlaying && !video.seeking && hasSrc && cooldownOk && (video.played?.length ?? 0) === 0 && !this.warmingUpVideos.has(video)) { vfPipelineMonitor.record('vf_gpu_cold', { clipId: clip.id }); this.startTargetedWarmup(clip.id, video, timeInfo.clipTime, { @@ -2344,6 +2366,32 @@ export class VideoSyncManager { if ((video.seeking || this.rvfcHandles[clipId] !== undefined) && this.pendingSeekTargets[clipId] !== undefined) { const allowInFlightRetarget = ctx.isDraggingPlayhead && supportsFastSeek; if (ctx.isDraggingPlayhead && !allowInFlightRetarget) { + const pendingTarget = this.pendingSeekTargets[clipId]; + const pendingAge = ctx.now - (this.pendingSeekStartedAt[clipId] ?? ctx.now); + const pendingTargetDrift = + typeof pendingTarget === 'number' + ? Math.abs(pendingTarget - time) + : 0; + if ( + displayedDriftSeconds >= 1 && + pendingAge >= 45 && + pendingTargetDrift >= 0.35 + ) { + this.pendingSeekTargets[clipId] = time; + this.pendingSeekStartedAt[clipId] = ctx.now; + this.latestSeekTargets[clipId] = time; + video.currentTime = this.safeSeekTime(video, time); + this.armSeekedFlush(clipId, video); + vfPipelineMonitor.record('vf_seek_precise', { + clipId, + target: Math.round(time * 1000) / 1000, + retarget: 'true', + followup: 'drag-force-retarget', + }); + this.registerRVFC(clipId, video); + this.lastSeekRef[clipId] = ctx.now; + return; + } this.queuedSeekTargets[clipId] = time; this.latestSeekTargets[clipId] = time; this.armSeekedFlush(clipId, video); @@ -2499,13 +2547,18 @@ export class VideoSyncManager { if (prevHandle !== undefined) { video.cancelVideoFrameCallback(prevHandle); } - this.rvfcHandles[clipId] = video.requestVideoFrameCallback(() => { - const presentedTime = video.currentTime; + this.rvfcHandles[clipId] = video.requestVideoFrameCallback((_now, metadata) => { + const metadataTime = metadata?.mediaTime; + const presentedTime = + typeof metadataTime === 'number' && Number.isFinite(metadataTime) + ? metadataTime + : video.currentTime; delete this.rvfcHandles[clipId]; delete this.pendingSeekTargets[clipId]; delete this.pendingSeekStartedAt[clipId]; engine.markVideoFramePresented(video, presentedTime, clipId); engine.captureVideoFrameAtTime(video, presentedTime, clipId); + engine.cacheFrameAtTime(video, presentedTime); scrubSettleState.resolve(clipId); vfPipelineMonitor.record('vf_seek_done', { clipId }); this.flushQueuedSeekTarget(clipId, video, 'rvfc'); diff --git a/src/version.ts b/src/version.ts index 88f5b6f4..c15c03c7 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1,6 +1,6 @@ // App version // Format: MAJOR.MINOR.PATCH -export const APP_VERSION = '1.7.4'; +export const APP_VERSION = '1.7.5'; export interface ChangelogNotice { type: 'info' | 'warning' | 'success' | 'danger'; @@ -36,8 +36,8 @@ export const FEATURED_VIDEO: { // Build/Platform notice shown at top of changelog (set to null to hide) export const BUILD_NOTICE: ChangelogNotice | null = { type: 'success', - title: 'World-space camera controls', - message: 'Camera Position X/Y/Z now represents the real camera eye, with lens FOV/mm and resolution kept as independent keyframable camera settings.', + title: 'Playback diagnostics and scrub polish', + message: 'The preview overlay now reports effective FPS across the render, preview, and decoder path, with HTML-video playback and scrubbing tuned for smoother feedback.', animated: true, }; diff --git a/tests/unit/layerCollector.test.ts b/tests/unit/layerCollector.test.ts index 5fec407f..c8ba1212 100644 --- a/tests/unit/layerCollector.test.ts +++ b/tests/unit/layerCollector.test.ts @@ -1341,7 +1341,7 @@ describe('LayerCollector', () => { expect(collector.hasActiveVideo()).toBe(true); }); - it('keeps the last same-clip frame during hard scrubs instead of dropping to black', () => { + it('keeps the last near same-clip frame during hard scrubs instead of dropping to black', () => { flags.useFullWebCodecsPlayback = false; useTimelineStore.setState({ isDraggingPlayhead: true }); @@ -1359,7 +1359,7 @@ describe('LayerCollector', () => { view: { label: 'held-same-clip-frame' }, width: 1920, height: 1080, - mediaTime: 2.5, + mediaTime: 18, }; const textureManager = { importVideoTexture: vi.fn(() => null), @@ -1413,7 +1413,7 @@ describe('LayerCollector', () => { sourceHeight: heldFrame.height, displayedMediaTime: heldFrame.mediaTime, targetMediaTime: 18, - previewPath: 'same-clip-hold', + previewPath: 'emergency-hold', }); }); @@ -1570,7 +1570,7 @@ describe('LayerCollector', () => { }); }); - it('uses a same-clip hold as the last fallback when a seeked HTML frame cannot import', () => { + it('uses a near seeking cache fallback when a seeked HTML frame cannot import', () => { flags.useFullWebCodecsPlayback = false; useTimelineStore.setState({ isDraggingPlayhead: true }); @@ -1588,7 +1588,7 @@ describe('LayerCollector', () => { view: { label: 'held-seeking-frame' }, width: 1920, height: 1080, - mediaTime: 7.25, + mediaTime: 25, }; const textureManager = { importVideoTexture: vi.fn(() => null), @@ -1643,7 +1643,7 @@ describe('LayerCollector', () => { sourceHeight: heldFrame.height, displayedMediaTime: heldFrame.mediaTime, targetMediaTime: 25, - previewPath: 'same-clip-hold', + previewPath: 'seeking-cache', }); }); diff --git a/tests/unit/renderDispatcher.test.ts b/tests/unit/renderDispatcher.test.ts index 07b69072..f0382fe1 100644 --- a/tests/unit/renderDispatcher.test.ts +++ b/tests/unit/renderDispatcher.test.ts @@ -191,7 +191,11 @@ describe('RenderDispatcher empty playback hold', () => { } as unknown as Layer]); expect(renderEmptyFrame).not.toHaveBeenCalled(); - expect(recordMainPreviewFrame).not.toHaveBeenCalled(); + expect(recordMainPreviewFrame).toHaveBeenCalledWith('playback-stall-hold', undefined, { + clipId: 'clip-1', + targetTimeMs: 17_750, + displayedTimeMs: undefined, + }); expect(deps.performanceStats.setLayerCount).toHaveBeenCalledWith(0); expect(dispatcher.lastRenderHadContent).toBe(true); });