From 32438ce25f69de99d2523b98237f62cfa90890d7 Mon Sep 17 00:00:00 2001 From: cashmere Date: Tue, 28 Jul 2026 20:14:51 +0200 Subject: feat: port domain crates to Rust (session 1 complete) All green: cargo test --workspace (~300 tests, 0 failures). - timeline (45): placement, resize/ripple, split, snap, retime, audio separation, scene helpers - animation (73): bezier solver, channel sampling, keyframe ops, split/clamp/clone, param definitions - commands (9): Command trait, BatchCommand, history, 19 concrete commands, freeze-frame port 1:1 - audio (22): audible collection, range mixing, mastering limiter, rms/peak buckets - text (19): cosmic-text layout/measure/raster, backgrounds, decorations - graphics (52): full CSS gradient parser, tiny-skia raster, 4 shape definitions - effect-defs (16): blur definition, registry, pass resolution - mask-defs (19): all 9 mask renderers via tiny-skia - renderer (6): node model + scene builder port - desktop: GPUI projects screen wired to ProjectStore - scene: factory (web defaults: v32, Main Track, 1920x1080, 30fps) --- rust/crates/renderer/Cargo.toml | 11 ++ rust/crates/renderer/src/lib.rs | 5 + rust/crates/renderer/src/node.rs | 123 ++++++++++++ rust/crates/renderer/src/scene_builder.rs | 285 ++++++++++++++++++++++++++++ rust/crates/renderer/tests/scene_builder.rs | 228 ++++++++++++++++++++++ 5 files changed, 652 insertions(+) create mode 100644 rust/crates/renderer/Cargo.toml create mode 100644 rust/crates/renderer/src/lib.rs create mode 100644 rust/crates/renderer/src/node.rs create mode 100644 rust/crates/renderer/src/scene_builder.rs create mode 100644 rust/crates/renderer/tests/scene_builder.rs (limited to 'rust/crates/renderer') diff --git a/rust/crates/renderer/Cargo.toml b/rust/crates/renderer/Cargo.toml new file mode 100644 index 00000000..36d858f2 --- /dev/null +++ b/rust/crates/renderer/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "renderer" +version = "0.1.0" +edition = "2024" + +[dependencies] +scene = { version = "0.1.0", path = "../scene" } +compositor = { version = "0.1.0", path = "../compositor" } +time = { version = "0.1.0", path = "../time" } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" diff --git a/rust/crates/renderer/src/lib.rs b/rust/crates/renderer/src/lib.rs new file mode 100644 index 00000000..2652bda6 --- /dev/null +++ b/rust/crates/renderer/src/lib.rs @@ -0,0 +1,5 @@ +pub mod node; +pub mod scene_builder; + +pub use node::*; +pub use scene_builder::{BuildSceneParams, build_scene}; diff --git a/rust/crates/renderer/src/node.rs b/rust/crates/renderer/src/node.rs new file mode 100644 index 00000000..f10c78f9 --- /dev/null +++ b/rust/crates/renderer/src/node.rs @@ -0,0 +1,123 @@ +use std::path::PathBuf; + +use compositor::BlendMode; +use scene::{Effect, ElementAnimations, Mask, ParamValues, RetimeConfig, TextElement}; +use time::MediaTime; + +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct Transform { + pub scale_x: f64, + pub scale_y: f64, + pub position_x: f64, + pub position_y: f64, + pub rotate: f64, +} + +impl Transform { + pub fn identity() -> Self { + Self { + scale_x: 1.0, + scale_y: 1.0, + ..Default::default() + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum MediaType { + Video, + Image, + Audio, +} + +#[derive(Clone, Debug)] +pub struct MediaSource { + pub id: String, + pub media_type: MediaType, + pub path: PathBuf, +} + +#[derive(Clone, Debug)] +pub struct VisualParams { + pub duration: MediaTime, + pub time_offset: MediaTime, + pub trim_start: MediaTime, + pub trim_end: MediaTime, + pub retime: Option, + pub transform: Transform, + pub animations: Option, + pub opacity: f64, + pub blend_mode: Option, + pub effects: Vec, + pub masks: Vec, +} + +#[derive(Clone, Debug)] +pub struct TextNodeParams { + pub element: TextElement, + pub transform: Transform, + pub opacity: f64, + pub blend_mode: Option, + pub canvas_center_x: f64, + pub canvas_center_y: f64, + pub canvas_height: f64, + pub effects: Vec, +} + +#[derive(Clone, Debug)] +pub struct GraphicNodeParams { + pub visual: VisualParams, + pub definition_id: String, + pub params: ParamValues, +} + +#[derive(Clone, Debug)] +pub struct BlurBackgroundParams { + pub media_id: String, + pub path: PathBuf, + pub media_type: MediaType, + pub duration: MediaTime, + pub time_offset: MediaTime, + pub trim_start: MediaTime, + pub trim_end: MediaTime, + pub retime: Option, + pub blur_intensity: f64, +} + +#[derive(Clone, Debug)] +pub struct EffectLayerParams { + pub effect_type: String, + pub effect_params: ParamValues, + pub time_offset: MediaTime, + pub duration: MediaTime, +} + +#[derive(Clone, Debug)] +pub struct StickerNodeParams { + pub visual: VisualParams, + pub sticker_id: String, + pub intrinsic_width: Option, + pub intrinsic_height: Option, +} + +#[derive(Clone, Debug)] +pub enum Node { + Root { duration: MediaTime, children: Vec }, + Color { color: String }, + BlurBackground(BlurBackgroundParams), + EffectLayer(EffectLayerParams), + Video { visual: VisualParams, media_id: String, path: PathBuf }, + Image { visual: VisualParams, path: PathBuf, max_source_size: Option }, + Sticker(StickerNodeParams), + Graphic(GraphicNodeParams), + Text(TextNodeParams), +} + +impl Node { + pub fn children(&self) -> &[Node] { + match self { + Node::Root { children, .. } => children, + _ => &[], + } + } +} diff --git a/rust/crates/renderer/src/scene_builder.rs b/rust/crates/renderer/src/scene_builder.rs new file mode 100644 index 00000000..826eee6f --- /dev/null +++ b/rust/crates/renderer/src/scene_builder.rs @@ -0,0 +1,285 @@ +use std::collections::HashMap; + +use compositor::BlendMode; +use scene::{Background, CanvasSize, Element, ParamValues, SceneTracks, Track}; +use time::MediaTime; + +use crate::node::{ + BlurBackgroundParams, EffectLayerParams, GraphicNodeParams, MediaSource, MediaType, Node, + StickerNodeParams, TextNodeParams, Transform, VisualParams, +}; + +const PREVIEW_MAX_IMAGE_SIZE: u32 = 2048; +pub const DEFAULT_BACKGROUND_BLUR_INTENSITY: f64 = 8.0; + +pub struct BuildSceneParams<'a> { + pub canvas_size: CanvasSize, + pub tracks: &'a SceneTracks, + pub media: &'a [MediaSource], + pub duration: MediaTime, + pub background: &'a Background, + pub is_preview: bool, +} + +pub fn build_transform_from_params(params: &ParamValues) -> Transform { + Transform { + scale_x: read_number_param(params, "transform.scaleX", 1.0), + scale_y: read_number_param(params, "transform.scaleY", 1.0), + position_x: read_number_param(params, "transform.positionX", 0.0), + position_y: read_number_param(params, "transform.positionY", 0.0), + rotate: read_number_param(params, "transform.rotate", 0.0), + } +} + +pub fn read_opacity_from_params(params: &ParamValues) -> f64 { + read_number_param(params, "opacity", 1.0) +} + +pub fn read_blend_mode_from_params(params: &ParamValues) -> Option { + let scene::ParamValue::String(value) = params.get("blendMode")? else { + return None; + }; + serde_json::from_value(serde_json::Value::String(value.clone())).ok() +} + +fn read_number_param(params: &ParamValues, key: &str, fallback: f64) -> f64 { + match params.get(key) { + Some(scene::ParamValue::Number(value)) => *value, + _ => fallback, + } +} + +fn visible_sorted_elements(track: &Track) -> Vec<&Element> { + let mut elements: Vec<&Element> = track + .elements() + .iter() + .filter(|element| !element_hidden(element)) + .collect(); + elements.sort_by(|a, b| { + a.base() + .start_time + .cmp(&b.base().start_time) + .then_with(|| a.base().id.cmp(&b.base().id)) + }); + elements +} + +fn element_hidden(element: &Element) -> bool { + match element { + Element::Video(e) => e.hidden.unwrap_or(false), + Element::Image(e) => e.hidden.unwrap_or(false), + Element::Text(e) => e.hidden.unwrap_or(false), + Element::Sticker(e) => e.hidden.unwrap_or(false), + Element::Graphic(e) => e.hidden.unwrap_or(false), + _ => false, + } +} + +fn visual_params(element: &Element) -> VisualParams { + let base = element.base(); + let (retime, effects, masks) = match element { + Element::Video(e) => ( + e.retime, + e.effects.clone().unwrap_or_default(), + e.masks.clone().unwrap_or_default(), + ), + Element::Image(e) => ( + None, + e.effects.clone().unwrap_or_default(), + e.masks.clone().unwrap_or_default(), + ), + Element::Sticker(e) => (None, e.effects.clone().unwrap_or_default(), Vec::new()), + Element::Graphic(e) => ( + None, + e.effects.clone().unwrap_or_default(), + e.masks.clone().unwrap_or_default(), + ), + _ => (None, Vec::new(), Vec::new()), + }; + VisualParams { + duration: base.duration, + time_offset: base.start_time, + trim_start: base.trim_start, + trim_end: base.trim_end, + retime, + transform: build_transform_from_params(&base.params), + animations: base.animations.clone(), + opacity: read_opacity_from_params(&base.params), + blend_mode: read_blend_mode_from_params(&base.params), + effects, + masks, + } +} + +fn build_track_nodes( + tracks: &[&Track], + media_map: &HashMap<&str, &MediaSource>, + canvas_size: CanvasSize, + is_preview: bool, +) -> Vec { + let mut nodes = Vec::new(); + + for track in tracks { + for element in visible_sorted_elements(track) { + match element { + Element::Effect(effect) => { + nodes.push(Node::EffectLayer(EffectLayerParams { + effect_type: effect.effect_type.clone(), + effect_params: effect.base.params.clone(), + time_offset: effect.base.start_time, + duration: effect.base.duration, + })); + } + Element::Video(video) => { + let Some(asset) = media_map.get(video.media_id.as_str()) else { + continue; + }; + if asset.media_type != MediaType::Video { + continue; + } + nodes.push(Node::Video { + visual: visual_params(element), + media_id: asset.id.clone(), + path: asset.path.clone(), + }); + } + Element::Image(image) => { + let Some(asset) = media_map.get(image.media_id.as_str()) else { + continue; + }; + if asset.media_type != MediaType::Image { + continue; + } + nodes.push(Node::Image { + visual: visual_params(element), + path: asset.path.clone(), + max_source_size: is_preview.then_some(PREVIEW_MAX_IMAGE_SIZE), + }); + } + Element::Text(text) => { + nodes.push(Node::Text(TextNodeParams { + element: text.clone(), + transform: build_transform_from_params(&text.base.params), + opacity: read_opacity_from_params(&text.base.params), + blend_mode: read_blend_mode_from_params(&text.base.params), + canvas_center_x: canvas_size.width as f64 / 2.0, + canvas_center_y: canvas_size.height as f64 / 2.0, + canvas_height: canvas_size.height as f64, + effects: text.effects.clone().unwrap_or_default(), + })); + } + Element::Sticker(sticker) => { + nodes.push(Node::Sticker(StickerNodeParams { + visual: visual_params(element), + sticker_id: sticker.sticker_id.clone(), + intrinsic_width: sticker.intrinsic_width, + intrinsic_height: sticker.intrinsic_height, + })); + } + Element::Graphic(graphic) => { + nodes.push(Node::Graphic(GraphicNodeParams { + visual: visual_params(element), + definition_id: graphic.definition_id.clone(), + params: graphic.base.params.clone(), + })); + } + Element::Audio(_) => {} + } + } + } + + nodes +} + +fn build_blur_background_nodes( + main_track: Option<&Track>, + media_map: &HashMap<&str, &MediaSource>, + blur_intensity: f64, +) -> Vec { + let Some(track) = main_track else { + return Vec::new(); + }; + + let mut nodes = Vec::new(); + for element in visible_sorted_elements(track) { + let (media_id, retime) = match element { + Element::Video(video) => (&video.media_id, video.retime), + Element::Image(image) => (&image.media_id, None), + _ => continue, + }; + let Some(asset) = media_map.get(media_id.as_str()) else { + continue; + }; + if !matches!(asset.media_type, MediaType::Video | MediaType::Image) { + continue; + } + let base = element.base(); + nodes.push(Node::BlurBackground(BlurBackgroundParams { + media_id: asset.id.clone(), + path: asset.path.clone(), + media_type: asset.media_type, + duration: base.duration, + time_offset: base.start_time, + trim_start: base.trim_start, + trim_end: base.trim_end, + retime, + blur_intensity, + })); + } + nodes +} + +pub fn build_scene(params: &BuildSceneParams) -> Node { + let media_map: HashMap<&str, &MediaSource> = + params.media.iter().map(|m| (m.id.as_str(), m)).collect(); + + let mut visible_tracks: Vec<&Track> = params + .tracks + .overlay + .iter() + .filter(|track| !track_hidden(track)) + .collect(); + if !track_hidden(¶ms.tracks.main) { + visible_tracks.push(¶ms.tracks.main); + } + visible_tracks.reverse(); + + let mut children = Vec::new(); + + if let Background::Blur { blur_intensity } = params.background { + let main_track = (!track_hidden(¶ms.tracks.main)).then_some(¶ms.tracks.main); + children.extend(build_blur_background_nodes( + main_track, + &media_map, + *blur_intensity, + )); + } else if let Background::Color { color } = params.background { + if color != "transparent" { + children.push(Node::Color { + color: color.clone(), + }); + } + } + + children.extend(build_track_nodes( + &visible_tracks, + &media_map, + params.canvas_size, + params.is_preview, + )); + + Node::Root { + duration: params.duration, + children, + } +} + +fn track_hidden(track: &Track) -> bool { + match track { + Track::Video { hidden, .. } + | Track::Text { hidden, .. } + | Track::Graphic { hidden, .. } + | Track::Effect { hidden, .. } => *hidden, + Track::Audio { .. } => false, + } +} diff --git a/rust/crates/renderer/tests/scene_builder.rs b/rust/crates/renderer/tests/scene_builder.rs new file mode 100644 index 00000000..1d65c092 --- /dev/null +++ b/rust/crates/renderer/tests/scene_builder.rs @@ -0,0 +1,228 @@ +use std::path::PathBuf; + +use renderer::node::{MediaSource, MediaType, Node}; +use renderer::{BuildSceneParams, build_scene}; +use scene::{ + Background, BaseTimelineElement, CanvasSize, Element, ImageElement, ParamValues, SceneTracks, + TextElement, Track, VideoElement, +}; +use time::MediaTime; + +fn base(id: &str, start: i64) -> BaseTimelineElement { + BaseTimelineElement { + id: id.to_string(), + name: id.to_string(), + duration: MediaTime::from_ticks(1000), + start_time: MediaTime::from_ticks(start), + trim_start: MediaTime::ZERO, + trim_end: MediaTime::ZERO, + source_duration: None, + animations: None, + params: ParamValues::new(), + } +} + +fn video_element(id: &str, start: i64, media_id: &str) -> Element { + Element::Video(VideoElement { + base: base(id, start), + media_id: media_id.to_string(), + is_source_audio_enabled: None, + hidden: None, + retime: None, + effects: None, + masks: None, + }) +} + +fn image_element(id: &str, start: i64, media_id: &str) -> Element { + Element::Image(ImageElement { + base: base(id, start), + media_id: media_id.to_string(), + hidden: None, + effects: None, + masks: None, + }) +} + +fn text_element(id: &str, start: i64, hidden: Option) -> Element { + Element::Text(TextElement { + base: base(id, start), + hidden, + effects: None, + }) +} + +fn video_track(id: &str, hidden: bool, elements: Vec) -> Track { + Track::Video { + id: id.to_string(), + name: id.to_string(), + elements, + muted: false, + hidden, + } +} + +fn media(id: &str, media_type: MediaType) -> MediaSource { + MediaSource { + id: id.to_string(), + media_type, + path: PathBuf::from(format!("/media/{id}")), + } +} + +fn build(tracks: &SceneTracks, media: &[MediaSource], background: &Background) -> Vec { + let params = BuildSceneParams { + canvas_size: CanvasSize { + width: 1920, + height: 1080, + }, + tracks, + media, + duration: MediaTime::from_ticks(10_000), + background, + is_preview: false, + }; + match build_scene(¶ms) { + Node::Root { children, .. } => children, + _ => panic!("expected root"), + } +} + +fn transparent() -> Background { + Background::Color { + color: "transparent".into(), + } +} + +#[test] +fn skips_elements_without_media_and_mismatched_types() { + let tracks = SceneTracks { + overlay: vec![], + main: video_track( + "main", + false, + vec![ + video_element("v1", 0, "m1"), + video_element("v2", 0, "missing"), + image_element("i1", 0, "m1"), + ], + ), + audio: vec![], + }; + let assets = [media("m1", MediaType::Video)]; + let children = build(&tracks, &assets, &transparent()); + + assert_eq!(children.len(), 1); + assert!(matches!(&children[0], Node::Video { media_id, .. } if media_id == "m1")); +} + +#[test] +fn main_track_renders_below_overlay() { + let tracks = SceneTracks { + overlay: vec![video_track("overlay", false, vec![text_element("t1", 0, None)])], + main: video_track("main", false, vec![video_element("v1", 0, "m1")]), + audio: vec![], + }; + let assets = [media("m1", MediaType::Video)]; + let children = build(&tracks, &assets, &transparent()); + + assert_eq!(children.len(), 2); + assert!(matches!(&children[0], Node::Video { .. })); + assert!(matches!(&children[1], Node::Text(_))); +} + +#[test] +fn hidden_elements_and_tracks_are_skipped() { + let tracks = SceneTracks { + overlay: vec![video_track("hidden-track", true, vec![text_element("t2", 0, None)])], + main: video_track( + "main", + false, + vec![video_element("v1", 0, "m1"), text_element("t1", 0, Some(true))], + ), + audio: vec![], + }; + let assets = [media("m1", MediaType::Video)]; + let children = build(&tracks, &assets, &transparent()); + + assert_eq!(children.len(), 1); + assert!(matches!(&children[0], Node::Video { .. })); +} + +#[test] +fn elements_sort_by_start_time_then_id() { + let tracks = SceneTracks { + overlay: vec![], + main: video_track( + "main", + false, + vec![ + video_element("b", 100, "m1"), + video_element("a", 100, "m1"), + video_element("z", 50, "m1"), + ], + ), + audio: vec![], + }; + let assets = [media("m1", MediaType::Video)]; + let children = build(&tracks, &assets, &transparent()); + + let starts: Vec = children + .iter() + .map(|node| match node { + Node::Video { visual, .. } => visual.time_offset.as_ticks(), + _ => panic!("expected video"), + }) + .collect(); + assert_eq!(starts, [50, 100, 100]); +} + +#[test] +fn color_background_comes_first_unless_transparent() { + let tracks = SceneTracks { + overlay: vec![], + main: video_track("main", false, vec![video_element("v1", 0, "m1")]), + audio: vec![], + }; + let assets = [media("m1", MediaType::Video)]; + + let with_color = build( + &tracks, + &assets, + &Background::Color { + color: "#112233".into(), + }, + ); + assert!(matches!(&with_color[0], Node::Color { color } if color == "#112233")); + assert_eq!(with_color.len(), 2); + + let transparent_children = build(&tracks, &assets, &transparent()); + assert_eq!(transparent_children.len(), 1); +} + +#[test] +fn blur_background_emits_backdrop_nodes_for_main_media() { + let tracks = SceneTracks { + overlay: vec![], + main: video_track("main", false, vec![video_element("v1", 0, "m1")]), + audio: vec![], + }; + let assets = [media("m1", MediaType::Video)]; + let children = build( + &tracks, + &assets, + &Background::Blur { + blur_intensity: 12.0, + }, + ); + + assert_eq!(children.len(), 2); + match &children[0] { + Node::BlurBackground(params) => { + assert_eq!(params.blur_intensity, 12.0); + assert_eq!(params.media_id, "m1"); + } + _ => panic!("expected blur background first"), + } + assert!(matches!(&children[1], Node::Video { .. })); +} -- cgit v1.3.1