From 9bc8f8aab2ad5843652e6e6ba5af7a1f68481a23 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Thu, 30 Jul 2026 15:58:05 +0200 Subject: [PATCH 01/46] feat(compositor): re-express the Linux port on the v1.8.0 cross-backend architecture release/v1.8.0's macOS/Metal port unified the cross-backend layer differently from the original Linux port: per-platform modules aliased to generic names (compositor / d3d / text / pipeline / cpu_frames), a shared frame_geometry::plan_frame (the geometry half of compose_frame, iso-render across backends), and shared live.rs + compositor-view-napi. This re-expresses the Linux port on that architecture instead of the old parallel CompBackend trait. New per-platform Linux modules (cfg(target_os = "linux"), aliased in lib.rs): - d3d_linux (Gpu/Backend, wgpu/Vulkan) from vk.rs - linux_frames (CpuFrames, NV12-split upload) from vk_frames.rs - text_linux (cosmic-text rasterizer) from text_cosmic.rs - compositor_linux (renders FrameGeometry via WGSL) from vk_render + layer.wgsl - pipeline_linux (Decoder; export is a WP6 stub) from pipeline_lin + vk_decode plus linux_decode (SwDecoder helper) and vk_shaders/{layer,blur}.wgsl. build.rs gains a Linux branch (wrapper_linux.h; ffmpeg without D3D11VA/VideoToolbox); compositor/Cargo.toml gains cfg(linux) deps (wgpu / pollster / cosmic-text); poc-d3d (the Win32 bench POC) is cfg(windows)-gated so the workspace builds on Linux. Verified on Linux (dzn/AMD): `cargo check --workspace` is green, and the compose_linux test renders a real 960x540 frame through the shared plan_frame geometry (mean_R = 202.6). compose_frame draws the core today (solid background + screen cover-fit with rounded corners); webcam PiP, cursor, annotations (WGSL mode 11 text), background blur and 3D tilt reuse the same draw primitives and land incrementally -- now sharing the Metal port's cross-backend plumbing instead of a parallel Linux one. --- crates/Cargo.lock | 1045 ++++++++++++++++++- crates/Cargo.toml | 5 + crates/compositor/Cargo.toml | 7 + crates/compositor/build.rs | 6 +- crates/compositor/src/compositor_linux.rs | 535 ++++++++++ crates/compositor/src/d3d_linux.rs | 163 +++ crates/compositor/src/lib.rs | 26 + crates/compositor/src/linux_decode.rs | 327 ++++++ crates/compositor/src/linux_frames.rs | 350 +++++++ crates/compositor/src/pipeline_linux.rs | 160 +++ crates/compositor/src/text_linux.rs | 218 ++++ crates/compositor/src/vk_shaders/blur.wgsl | 85 ++ crates/compositor/src/vk_shaders/layer.wgsl | 126 +++ crates/compositor/tests/compose_linux.rs | 54 + crates/compositor/wrapper_linux.h | 14 + crates/poc-d3d/Cargo.toml | 3 + crates/poc-d3d/src/main.rs | 10 + 17 files changed, 3131 insertions(+), 3 deletions(-) create mode 100644 crates/compositor/src/compositor_linux.rs create mode 100644 crates/compositor/src/d3d_linux.rs create mode 100644 crates/compositor/src/linux_decode.rs create mode 100644 crates/compositor/src/linux_frames.rs create mode 100644 crates/compositor/src/pipeline_linux.rs create mode 100644 crates/compositor/src/text_linux.rs create mode 100644 crates/compositor/src/vk_shaders/blur.wgsl create mode 100644 crates/compositor/src/vk_shaders/layer.wgsl create mode 100644 crates/compositor/tests/compose_linux.rs create mode 100644 crates/compositor/wrapper_linux.h diff --git a/crates/Cargo.lock b/crates/Cargo.lock index df0c787e9..dbe2f253a 100644 --- a/crates/Cargo.lock +++ b/crates/Cargo.lock @@ -17,12 +17,36 @@ dependencies = [ "memchr", ] +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anyhow" version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "ash" +version = "0.38.0+1.3.281" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb44936d800fea8f016d7f2311c6a4f97aebd5dc86f09906139ec848cf3a46f" +dependencies = [ + "libloading", +] + [[package]] name = "autocfg" version = "1.5.1" @@ -44,11 +68,26 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash", + "rustc-hash 1.1.0", "shlex 1.3.0", "syn 2.0.119", ] +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bitflags" version = "1.3.2" @@ -60,6 +99,9 @@ name = "bitflags" version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +dependencies = [ + "serde_core", +] [[package]] name = "block" @@ -67,11 +109,31 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a" +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + [[package]] name = "bytemuck" version = "1.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6aedf8ae72766347502cf3cb4f41cf5e9cc37d28bee90f1fdaaae15f9cf9424" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f65693059b6b9c588b9f62fed1cedbf0a8b805631457ea162d68f0de186f3de5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] [[package]] name = "byteorder-lite" @@ -104,6 +166,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + [[package]] name = "clang-sys" version = "1.8.1" @@ -115,6 +183,16 @@ dependencies = [ "libloading", ] +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + [[package]] name = "compositor-view-napi" version = "0.0.0" @@ -163,6 +241,39 @@ dependencies = [ "libc", ] +[[package]] +name = "core_maths" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77745e017f5edba1a9c1d854f6f3a52dac8a12dd5af5d2f54aecf61e43d80d30" +dependencies = [ + "libm", +] + +[[package]] +name = "cosmic-text" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be17b688510d934ce13f48a2beba700e11583e281e0fda99c22bb256a14eda73" +dependencies = [ + "bitflags 2.13.1", + "fontdb", + "harfrust", + "linebender_resource_handle", + "log", + "rangemap", + "rustc-hash 2.1.3", + "self_cell", + "skrifa 0.40.0", + "smol_str", + "swash", + "sys-locale", + "unicode-bidi", + "unicode-linebreak", + "unicode-script", + "unicode-segmentation", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -182,12 +293,27 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + [[package]] name = "either" version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + [[package]] name = "fdeflate" version = "0.3.7" @@ -213,6 +339,53 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "font-types" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b38ad915f6dadd993ced50848a8291a543bd41ca62bc10740d5e64e2ab4cfd7" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "font-types" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a7299a780854a6d391be2ae1c8521c9368471b559dbfd6a8dbd9f407eaff100" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "fontconfig-parser" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbc773e24e02d4ddd8395fd30dc147524273a83e54e0f312d986ea30de5f5646" +dependencies = [ + "roxmltree", +] + +[[package]] +name = "fontdb" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "457e789b3d1202543297a350643cf459f836cade38934e7a4cf6a39e7cde2905" +dependencies = [ + "fontconfig-parser", + "log", + "memmap2", + "slotmap", + "tinyvec", + "ttf-parser", +] + [[package]] name = "foreign-types" version = "0.5.0" @@ -240,12 +413,159 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "gl_generator" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a95dfc23a2b4a9a2f5ab41d194f8bfda3cabec42af4e39f08c339eb2a0c124d" +dependencies = [ + "khronos_api", + "log", + "xml-rs", +] + [[package]] name = "glob" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "glow" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5e5ea60d70410161c8bf5da3fdfeaa1c72ed2c15f8bbb9d19fe3a4fad085f08" +dependencies = [ + "js-sys", + "slotmap", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "glutin_wgl_sys" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c4ee00b289aba7a9e5306d57c2d05499b2e5dc427f84ac708bd2c090212cf3e" +dependencies = [ + "gl_generator", +] + +[[package]] +name = "gpu-alloc" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45cf04b2726f02df5508c6de726acdc90cdf97ac771a9a0ffd8ba10a6e696bf9" +dependencies = [ + "bitflags 2.13.1", + "gpu-alloc-types", +] + +[[package]] +name = "gpu-alloc-types" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2bbed164dd10ed526c2e4fe3e721ca4a71c61730e5aafac6844b417b3227058" +dependencies = [ + "bitflags 2.13.1", +] + +[[package]] +name = "gpu-allocator" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c151a2a5ef800297b4e79efa4f4bec035c5f51d5ae587287c9b952bdf734cacd" +dependencies = [ + "log", + "presser", + "thiserror 1.0.69", + "windows", +] + +[[package]] +name = "gpu-descriptor" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b89c83349105e3732062a895becfc71a8f921bb71ecbbdd8ff99263e3b53a0ca" +dependencies = [ + "bitflags 2.13.1", + "gpu-descriptor-types", + "hashbrown 0.15.5", +] + +[[package]] +name = "gpu-descriptor-types" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdf242682df893b86f33a73828fb09ca4b2d3bb6cc95249707fc684d27484b91" +dependencies = [ + "bitflags 2.13.1", +] + +[[package]] +name = "harfrust" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9da2e5ae821f6e96664977bf974d6d6a2d6682f9ccee23e62ec1d134246845f9" +dependencies = [ + "bitflags 2.13.1", + "bytemuck", + "core_maths", + "read-fonts 0.37.0", + "smallvec", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hexf-parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" + [[package]] name = "image" version = "0.25.10" @@ -261,6 +581,16 @@ dependencies = [ "zune-jpeg", ] +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + [[package]] name = "itertools" version = "0.13.0" @@ -276,6 +606,62 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "khronos-egl" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aae1df220ece3c0ada96b8153459b67eebe9ae9212258bb0134ae60416fdf76" +dependencies = [ + "libc", + "libloading", + "pkg-config", +] + +[[package]] +name = "khronos_api" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc" + [[package]] name = "libc" version = "0.2.186" @@ -292,6 +678,33 @@ dependencies = [ "windows-link", ] +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "linebender_resource_handle" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4a5ff6bcca6c4867b1c4fd4ef63e4db7436ef363e0ad7531d1558856bae64f4" + +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.33" @@ -313,6 +726,15 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + [[package]] name = "metal" version = "0.29.0" @@ -328,6 +750,21 @@ dependencies = [ "paste", ] +[[package]] +name = "metal" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f569fb946490b5743ad69813cb19629130ce9374034abe31614a36402d18f99e" +dependencies = [ + "bitflags 2.13.1", + "block", + "core-graphics-types", + "foreign-types", + "log", + "objc", + "paste", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -354,6 +791,28 @@ dependencies = [ "pxfm", ] +[[package]] +name = "naga" +version = "24.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e380993072e52eef724eddfcde0ed013b0c023c3f0417336ed041aa9f076994e" +dependencies = [ + "arrayvec", + "bit-set", + "bitflags 2.13.1", + "cfg_aliases", + "codespan-reporting", + "hexf-parse", + "indexmap", + "log", + "rustc-hash 1.1.0", + "spirv", + "strum", + "termcolor", + "thiserror 2.0.19", + "unicode-xid", +] + [[package]] name = "napi" version = "2.16.17" @@ -411,6 +870,15 @@ dependencies = [ "libloading", ] +[[package]] +name = "ndk-sys" +version = "0.5.0+25.2.9519653" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c196769dd60fd4f363e11d948139556a344e79d451aeb2fa2fd040738ef7691" +dependencies = [ + "jni-sys 0.3.1", +] + [[package]] name = "nom" version = "7.1.3" @@ -454,20 +922,67 @@ dependencies = [ "block", "cc", "core-foundation", + "cosmic-text", "image", - "metal", + "metal 0.29.0", "objc", + "pollster", "serde", "serde_json", + "wgpu", "windows", ] +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + [[package]] name = "paste" version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "png" version = "0.18.1" @@ -490,6 +1005,18 @@ dependencies = [ "windows", ] +[[package]] +name = "pollster" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3a9f18d041e6d0e102a0a46750538147e5e8992d3b4873aaafee2520b00ce3" + +[[package]] +name = "presser" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf8e6a8aa66ce33f63993ffc4ea4271eb5b0530a9002db8455ea6050c77bfa" + [[package]] name = "prettyplease" version = "0.2.37" @@ -509,6 +1036,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "profiling" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d595e54a326bc53c1c197b32d295e14b169e3cfeaa8dc82b529f947fba6bcf5" + [[package]] name = "pxfm" version = "0.1.30" @@ -524,6 +1057,55 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "range-alloc" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca45419789ae5a7899559e9512e58ca889e41f04f1f2445e9f4b290ceccd1d08" + +[[package]] +name = "rangemap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" + +[[package]] +name = "raw-window-handle" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" + +[[package]] +name = "read-fonts" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b634fabf032fab15307ffd272149b622260f55974d9fad689292a5d33df02e5" +dependencies = [ + "bytemuck", + "core_maths", + "font-types 0.11.3", +] + +[[package]] +name = "read-fonts" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046a7d674daf459825b32f5062056d6882db0d2f5a479fbd76ccfc870ac18709" +dependencies = [ + "bytemuck", + "font-types 0.12.2", + "once_cell", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.1", +] + [[package]] name = "regex" version = "1.13.1" @@ -553,12 +1135,48 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "renderdoc-sys" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b30a45b0cd0bcca8037f3d0dc3421eaf95327a17cad11964fb8179b4fc4832" + +[[package]] +name = "roxmltree" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97" + [[package]] name = "rustc-hash" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "self_cell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab42ca02749e120097e328d91d415325bdf43b1c72c4c8badf37375fe40a813" + [[package]] name = "semver" version = "1.0.28" @@ -626,6 +1244,101 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" +[[package]] +name = "skrifa" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fbdfe3d2475fbd7ddd1f3e5cf8288a30eb3e5f95832829570cd88115a7434ac" +dependencies = [ + "bytemuck", + "read-fonts 0.37.0", +] + +[[package]] +name = "skrifa" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819ab7d62b1d3e72d9d9dea5650bac30424f9111364bb94928dbf5ecad1baa68" +dependencies = [ + "bytemuck", + "read-fonts 0.41.0", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "slotmap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038" +dependencies = [ + "version_check", +] + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "smol_str" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa7368fcf4852a4c2dd92df0cace6a71f2091ca0a23391ce7f3a31833f1523" + +[[package]] +name = "spirv" +version = "0.3.0+sdk-1.3.268.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eda41003dc44290527a59b13432d4a0379379fa074b70174882adfbdfd917844" +dependencies = [ + "bitflags 2.13.1", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.119", +] + +[[package]] +name = "swash" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c2499c2d826531388872b2268718aed907a39bd785ab0dcfe57fab26283f92e" +dependencies = [ + "skrifa 0.44.0", + "yazi", + "zeno", +] + [[package]] name = "syn" version = "2.0.119" @@ -648,18 +1361,319 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sys-locale" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eab9a99a024a169fe8a903cf9d4a3b3601109bcc13bd9e3c6fff259138626c4" +dependencies = [ + "libc", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +dependencies = [ + "thiserror-impl 2.0.19", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "ttf-parser" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" +dependencies = [ + "core_maths", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-linebreak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" + +[[package]] +name = "unicode-script" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "383ad40bb927465ec0ce7720e033cb4ca06912855fc35db31b5755d0de75b1ee" + [[package]] name = "unicode-segmentation" version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wgpu" +version = "24.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b0b3436f0729f6cdf2e6e9201f3d39dc95813fad61d826c1ed07918b4539353" +dependencies = [ + "arrayvec", + "bitflags 2.13.1", + "cfg_aliases", + "document-features", + "js-sys", + "log", + "naga", + "parking_lot", + "profiling", + "raw-window-handle", + "smallvec", + "static_assertions", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "wgpu-core", + "wgpu-hal", + "wgpu-types", +] + +[[package]] +name = "wgpu-core" +version = "24.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f0aa306497a238d169b9dc70659105b4a096859a34894544ca81719242e1499" +dependencies = [ + "arrayvec", + "bit-vec", + "bitflags 2.13.1", + "cfg_aliases", + "document-features", + "indexmap", + "log", + "naga", + "once_cell", + "parking_lot", + "profiling", + "raw-window-handle", + "rustc-hash 1.1.0", + "smallvec", + "thiserror 2.0.19", + "wgpu-hal", + "wgpu-types", +] + +[[package]] +name = "wgpu-hal" +version = "24.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f112f464674ca69f3533248508ee30cb84c67cf06c25ff6800685f5e0294e259" +dependencies = [ + "android_system_properties", + "arrayvec", + "ash", + "bit-set", + "bitflags 2.13.1", + "block", + "bytemuck", + "cfg_aliases", + "core-graphics-types", + "glow", + "glutin_wgl_sys", + "gpu-alloc", + "gpu-allocator", + "gpu-descriptor", + "js-sys", + "khronos-egl", + "libc", + "libloading", + "log", + "metal 0.31.0", + "naga", + "ndk-sys", + "objc", + "once_cell", + "ordered-float", + "parking_lot", + "profiling", + "range-alloc", + "raw-window-handle", + "renderdoc-sys", + "rustc-hash 1.1.0", + "smallvec", + "thiserror 2.0.19", + "wasm-bindgen", + "web-sys", + "wgpu-types", + "windows", + "windows-core", +] + +[[package]] +name = "wgpu-types" +version = "24.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50ac044c0e76c03a0378e7786ac505d010a873665e2d51383dcff8dd227dc69c" +dependencies = [ + "bitflags 2.13.1", + "js-sys", + "log", + "web-sys", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + [[package]] name = "windows" version = "0.58.0" @@ -730,6 +1744,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -794,6 +1817,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "xml-rs" +version = "0.8.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae8337f8a065cfc972643663ea4279e04e7256de865aa66fe25cec5fb912d3f" + +[[package]] +name = "yazi" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01738255b5a16e78bbb83e7fbba0a1e7dd506905cfc53f4622d89015a03fbb5" + +[[package]] +name = "zeno" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6df3dc4292935e51816d896edcd52aa30bc297907c26167fec31e2b0c6a32524" + [[package]] name = "zmij" version = "1.0.23" diff --git a/crates/Cargo.toml b/crates/Cargo.toml index ecb97d3c9..208f376cc 100644 --- a/crates/Cargo.toml +++ b/crates/Cargo.toml @@ -29,6 +29,11 @@ serde_json = "1" # Décodage des wallpapers image (jpg/png) pour le fond natif. default-features off → # on ne tire que les décodeurs nécessaires (build plus léger). image = { version = "0.25", default-features = false, features = ["jpeg", "png"] } +# Port Linux du compositor (PR #183) : wgpu (Vulkan) + pollster (block_on) + +# cosmic-text (rastérisation texte, remplace DirectWrite/CoreText). +wgpu = { version = "24", features = ["wgsl"] } +pollster = "0.4" +cosmic-text = "0.19" [workspace.dependencies.windows] version = "0.58" diff --git a/crates/compositor/Cargo.toml b/crates/compositor/Cargo.toml index 1a4692bfd..f968b5497 100644 --- a/crates/compositor/Cargo.toml +++ b/crates/compositor/Cargo.toml @@ -44,3 +44,10 @@ metal = "0.29" objc = "0.2" block = "0.1" core-foundation = "0.9" + +# Linux : wgpu (Vulkan) pour le rendu, pollster pour block_on les ops wgpu, +# cosmic-text pour la rastérisation du texte (remplace DirectWrite/CoreText). +[target.'cfg(target_os = "linux")'.dependencies] +wgpu.workspace = true +pollster.workspace = true +cosmic-text.workspace = true diff --git a/crates/compositor/build.rs b/crates/compositor/build.rs index 7c4e9d478..362ebd8a9 100644 --- a/crates/compositor/build.rs +++ b/crates/compositor/build.rs @@ -76,9 +76,13 @@ fn main() { // Le wrapper.h à binder dépend de la plateforme cible : // - Windows : D3D11VA (ID3D11VA*), - // - macOS : VideoToolbox (AVVideotoolboxContext). + // - macOS : VideoToolbox (AVVideotoolboxContext), + // - Linux : software/VAAPI, aucun hwcontext propriétaire (le d3d11.h / + // les headers VT n'existent pas → wrapper dédié). let wrapper = if target_is_macos { "wrapper_macos.h" + } else if target_os == "linux" { + "wrapper_linux.h" } else { "wrapper_windows.h" }; diff --git a/crates/compositor/src/compositor_linux.rs b/crates/compositor/src/compositor_linux.rs new file mode 100644 index 000000000..f42b83990 --- /dev/null +++ b/crates/compositor/src/compositor_linux.rs @@ -0,0 +1,535 @@ +//! Moteur de composition Linux -- wgpu / WGSL. +//! +//! Equivalent Linux de `compositor_windows.rs` / `compositor_macos.rs` : meme +//! surface publique (`Compositor::{new, new_sized, normalize_render_size, +//! render_size, set_scene, set_live_params, set_cursor, set_cursor_time, +//! set_timeline_time, clear_cursor, scene_snapshot, clear_srv_cache, +//! compose_frame, readback_direct}`) pour que `live.rs` et `compositor-view-napi` +//! (cfg-re-exportes via `crate::compositor`) l'utilisent sans connaitre la +//! plateforme. +//! +//! **Iso-render.** La GEOMETRIE (placement de chaque calque) vient de +//! `frame_geometry::plan_frame` -- la MEME fonction que Windows/macOS, au pixel +//! pres. Ce module ne fait que RENDRE le `FrameGeometry` via wgpu/WGSL +//! (`vk_shaders/layer.wgsl`), la ou macOS le rend via Metal/MSL. +//! +//! **Portee actuelle.** `compose_frame` rend le coeur : fond uni + calque ecran +//! cover-fit (coins arrondis). Les calques riches (webcam PiP, curseur, +//! annotations texte mode 11, blur de fond, tilt 3D, motion blur) sont dessines +//! par les memes primitives (`draw_layer`) et arrivent par iterations, comme le +//! port Metal les a ajoutes -- chacun reutilise `layer.wgsl` (modes deja portes) +//! ou une passe dediee (`blur.wgsl`). + +use std::cell::RefCell; + +use anyhow::Result; +use wgpu::util::DeviceExt; + +use crate::config::Cfg; +use crate::d3d::Gpu; +use crate::ffi::AVFrame; +// Re-exports que le code partage (live.rs, compositor-view-napi) consomme via +// `crate::compositor::…`, a l'identique de `compositor_macos`. +pub use crate::frame_geometry::{ + live_params_from_scene, webcam_shape_code, FIXTURE_FRAMES, LayerCB, LiveParams, OUT_H, OUT_W, +}; +use crate::frame_geometry::{parse_hex, plan_frame, FrameGeometryInput}; +use crate::scene::{Scene, SceneBackground}; + +const LAYER_WGSL: &str = include_str!("vk_shaders/layer.wgsl"); + +/// `&LayerCB` -> `&[u8; 128]`. `LayerCB` est `#[repr(C, align(16))]`, son layout +/// EST le buffer uniforme WGSL (16 vec4 + 1 vec2 + 2 f32 = 128 octets). +fn layer_bytes(cb: &LayerCB) -> &[u8] { + unsafe { std::slice::from_raw_parts(cb as *const LayerCB as *const u8, 128) } +} + +pub struct Compositor { + gpu: Gpu, + render_w: u32, + render_h: u32, + + // Pipeline de calque (VS + FS `layer.wgsl`), sampler lineaire, bind group + // layout (uniform + 2 textures + sampler). Immuables apres `new_sized`. + pipeline: wgpu::RenderPipeline, + bind_group_layout: wgpu::BindGroupLayout, + sampler: wgpu::Sampler, + + // Render target offscreen + buffer de readback (recree au resize). + rt: wgpu::Texture, + rt_view: wgpu::TextureView, + readback_buf: wgpu::Buffer, + /// `bytes_per_row` padde a 256 (contrainte wgpu de copy_texture_to_buffer). + readback_bpr: u32, + + // Etat pilote par live.rs (interior mutability : les methodes sont `&self`). + live_params: RefCell, + scene: RefCell>, + cursor: RefCell>, + cursor_time: RefCell>, + timeline_time: RefCell>, + + /// Rasterizer de texte (annotations mode 11). `None` si l'init cosmic-text + /// echoue -- le rendu continue sans texte plutot que de tout casser. + #[allow(dead_code)] + text_raster: Option, +} + +impl Compositor { + pub fn new(gpu: &Gpu) -> Result { + Self::new_sized(gpu, OUT_W, OUT_H) + } + + pub fn new_sized(gpu: &Gpu, w: u32, h: u32) -> Result { + let (w, h) = Self::normalize_render_size(w, h); + let gpu = Gpu { + device: gpu.device.clone(), + context: gpu.context.clone(), + backend: gpu.backend, + feature_level: gpu.feature_level, + }; + + let module = gpu.device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("layer.wgsl"), + source: wgpu::ShaderSource::Wgsl(LAYER_WGSL.into()), + }); + let sampler = gpu.device.create_sampler(&wgpu::SamplerDescriptor { + label: Some("layer"), + address_mode_u: wgpu::AddressMode::ClampToEdge, + address_mode_v: wgpu::AddressMode::ClampToEdge, + address_mode_w: wgpu::AddressMode::ClampToEdge, + mag_filter: wgpu::FilterMode::Linear, + min_filter: wgpu::FilterMode::Linear, + mipmap_filter: wgpu::FilterMode::Nearest, + ..Default::default() + }); + let tex_entry = |binding: u32| wgpu::BindGroupLayoutEntry { + binding, + visibility: wgpu::ShaderStages::VERTEX | wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }; + let bind_group_layout = + gpu.device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("layer"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::VERTEX | wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: wgpu::BufferSize::new(128), + }, + count: None, + }, + tex_entry(1), + tex_entry(2), + wgpu::BindGroupLayoutEntry { + binding: 3, + visibility: wgpu::ShaderStages::VERTEX | wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + ], + }); + let pipeline_layout = gpu.device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("layer"), + bind_group_layouts: &[&bind_group_layout], + push_constant_ranges: &[], + }); + let pipeline = gpu.device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("layer"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &module, + entry_point: Some("vs_main"), + compilation_options: wgpu::PipelineCompilationOptions::default(), + buffers: &[], + }, + fragment: Some(wgpu::FragmentState { + module: &module, + entry_point: Some("fs_main"), + compilation_options: wgpu::PipelineCompilationOptions::default(), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Rgba8Unorm, + blend: Some(wgpu::BlendState::PREMULTIPLIED_ALPHA_BLENDING), + write_mask: wgpu::ColorWrites::ALL, + })], + }), + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleStrip, + ..Default::default() + }, + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview: None, + cache: None, + }); + + let (rt, rt_view, readback_buf, readback_bpr) = Self::make_targets(&gpu, w, h); + + Ok(Compositor { + gpu, + render_w: w, + render_h: h, + pipeline, + bind_group_layout, + sampler, + rt, + rt_view, + readback_buf, + readback_bpr, + live_params: RefCell::new(LiveParams::default()), + scene: RefCell::new(None), + cursor: RefCell::new(None), + cursor_time: RefCell::new(None), + timeline_time: RefCell::new(None), + text_raster: crate::text::TextRasterizer::new().ok(), + }) + } + + /// RT RGBA8 + buffer de readback (bytes_per_row padde a 256). + fn make_targets( + gpu: &Gpu, + w: u32, + h: u32, + ) -> (wgpu::Texture, wgpu::TextureView, wgpu::Buffer, u32) { + let rt = gpu.device.create_texture(&wgpu::TextureDescriptor { + label: Some("rt"), + size: wgpu::Extent3d { + width: w, + height: h, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }); + let rt_view = rt.create_view(&wgpu::TextureViewDescriptor::default()); + let bpr = (w * 4).div_ceil(256) * 256; + let readback_buf = gpu.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("readback"), + size: u64::from(bpr) * u64::from(h), + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }); + (rt, rt_view, readback_buf, bpr) + } + + /// Dimensions paires (NV12 4:2:0), min 2x2. Symetrie avec les autres backends. + pub fn normalize_render_size(w: u32, h: u32) -> (u32, u32) { + ((w.max(2) + 1) & !1, (h.max(2) + 1) & !1) + } + + pub fn render_size(&self) -> (u32, u32) { + (self.render_w, self.render_h) + } + + pub fn set_live_params(&self, p: LiveParams) { + *self.live_params.borrow_mut() = p; + } + + pub fn set_scene(&self, s: Option) { + *self.scene.borrow_mut() = s; + } + + pub fn set_cursor(&self, track: crate::cursor::CursorTrack) { + *self.cursor.borrow_mut() = Some(track); + } + + pub fn set_cursor_time(&self, t: Option) { + *self.cursor_time.borrow_mut() = t; + } + + pub fn set_timeline_time(&self, t: Option) { + *self.timeline_time.borrow_mut() = t; + } + + pub fn clear_cursor(&self) { + *self.cursor.borrow_mut() = None; + } + + pub fn scene_snapshot(&self) -> Option { + self.scene.borrow().clone() + } + + /// Pas de cache de SRV cote wgpu (les `TextureView`s sont recreees a chaque + /// draw depuis le carrier) -- no-op conserve pour la symetrie d'API. + pub fn clear_srv_cache(&self) {} + + // -- seam frame (lit le carrier `data[0]`) -- + + fn pixel_buffer_of(frame: *const AVFrame) -> Option<()> { + if frame.is_null() || unsafe { (*frame).data[0] }.is_null() { + None + } else { + Some(()) + } + } + + unsafe fn nv12_srvs( + &self, + frame: *const AVFrame, + ) -> Result<(wgpu::TextureView, wgpu::TextureView)> { + crate::linux_frames::nv12_planes(frame) + } + + unsafe fn tex_dims(&self, frame: *const AVFrame) -> (u32, u32) { + if frame.is_null() || (*frame).data[0].is_null() { + return (1, 1); + } + crate::linux_frames::carrier_dims(frame) + } + + // -- rendu -- + + /// Prepare un draw de calque : buffer uniforme init a `cb` + bind group + /// (uniform + deux textures + sampler). Cree AVANT la render pass pour que + /// les ressources vivent pendant tout le pass. Un buffer PAR draw : + /// `write_buffer` entre draws d'une meme pass ne s'entrelace pas. + fn make_bind( + &self, + cb: &LayerCB, + planes: Option<(&wgpu::TextureView, &wgpu::TextureView)>, + dummy: &wgpu::TextureView, + ) -> (wgpu::Buffer, wgpu::BindGroup) { + let uniform = self.gpu.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("layer-uniform"), + contents: layer_bytes(cb), + usage: wgpu::BufferUsages::UNIFORM, + }); + let (y, uv) = planes.unwrap_or((dummy, dummy)); + let bind = self.gpu.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("layer"), + layout: &self.bind_group_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: uniform.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::TextureView(y), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: wgpu::BindingResource::TextureView(uv), + }, + wgpu::BindGroupEntry { + binding: 3, + resource: wgpu::BindingResource::Sampler(&self.sampler), + }, + ], + }); + (uniform, bind) + } + + /// Rend une frame dans le RT interne. Le screen `screen`/`webcam` sont des + /// carriers `linux_frames` ; la geometrie vient de `plan_frame`. Coeur : + /// fond uni + ecran cover-fit. `readback_direct` lit ensuite le RT. + pub unsafe fn compose_frame( + &self, + screen: *const AVFrame, + webcam: *const AVFrame, + frame: f32, + cfg: &Cfg, + ) -> Result<()> { + if Self::pixel_buffer_of(screen).is_none() { + return self.clear_rt(); + } + let (sy, suv) = self.nv12_srvs(screen)?; + let (stw, sth) = self.tex_dims(screen); + let (wtw, wth) = self.tex_dims(webcam); + let (scw, sch) = ((*screen).width as f32, (*screen).height as f32); + let (wcw, wch) = if webcam.is_null() { + (1.0, 1.0) + } else { + ((*webcam).width as f32, (*webcam).height as f32) + }; + let u_max = scw / (stw.max(1)) as f32; + let v_max = sch / (sth.max(1)) as f32; + let (rw, rh) = (self.render_w as f32, self.render_h as f32); + + let scene_ref = self.scene.borrow(); + let cursor_ref = self.cursor.borrow(); + let lp = *self.live_params.borrow(); + let g = plan_frame(&FrameGeometryInput { + render_px: [rw, rh], + screen_tex_px: [stw as f32, sth as f32], + screen_visible_px: [scw, sch], + webcam_visible_px: [wcw, wch], + u_max, + v_max, + frame, + cfg, + live: lp, + scene: scene_ref.as_ref(), + cursor: cursor_ref.as_ref(), + timeline_t_override: *self.timeline_time.borrow(), + }); + let _ = (wtw, wth); // webcam PiP : calque a venir. + + // Couleur de fond : scene Color -> sa couleur, sinon live_params.bg_color. + // (Gradient/Image : calques a venir ; repli couleur ici.) + let bg = match scene_ref.as_ref().map(|s| s.background.clone()) { + Some(SceneBackground::Color { color }) => parse_hex(&color).unwrap_or(lp.bg_color), + _ => lp.bg_color, + }; + + // Calque ecran (mode 0 : NV12 -> RGB), place par plan_frame (cover-fit + + // coins arrondis). `src = g.cut` (crop utilisateur + zoom en UV texture). + let screen_layer = LayerCB { + dst: g.s_dst, + src: g.cut, + quad_px: [g.s_dst[2] * rw, g.s_dst[3] * rh], + radius_px: g.s_radius, + mode: 0.0, + color: [1.0, 1.0, 1.0, 1.0], + ..Default::default() + }; + // Bind group construit AVANT le pass (doit vivre pendant tout le pass) ; + // `_screen_uniform` garde le buffer uniforme en vie (reference par le bind). + let dummy = self.dummy_view(); + let (_screen_uniform, screen_bind) = + self.make_bind(&screen_layer, Some((&sy, &suv)), &dummy); + + let mut encoder = self.gpu.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("compose"), + }); + { + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("compose-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.rt_view, + resolve_target: None, + ops: wgpu::Operations { + // Le fond uni EST la couleur de clear (gradient/image + // ajouteront un draw ici). + load: wgpu::LoadOp::Clear(wgpu::Color { + r: bg[0] as f64, + g: bg[1] as f64, + b: bg[2] as f64, + a: bg[3] as f64, + }), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + rpass.set_pipeline(&self.pipeline); + rpass.set_bind_group(0, &screen_bind, &[]); + rpass.draw(0..4, 0..1); + } + self.gpu.context.submit(std::iter::once(encoder.finish())); + Ok(()) + } + + /// Clear le RT a la couleur de fond (ecran absent). + fn clear_rt(&self) -> Result<()> { + let bg = self.live_params.borrow().bg_color; + let mut encoder = self.gpu.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("clear"), + }); + encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("clear-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.rt_view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color { + r: bg[0] as f64, + g: bg[1] as f64, + b: bg[2] as f64, + a: bg[3] as f64, + }), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + self.gpu.context.submit(std::iter::once(encoder.finish())); + Ok(()) + } + + fn dummy_view(&self) -> wgpu::TextureView { + let t = self.gpu.device.create_texture(&wgpu::TextureDescriptor { + label: Some("dummy"), + size: wgpu::Extent3d { + width: 1, + height: 1, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::R8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING, + view_formats: &[], + }); + t.create_view(&wgpu::TextureViewDescriptor::default()) + } + + /// Lit le RT en RGBA8 tightly-packed `(render_w * render_h * 4)`. Depadde le + /// `bytes_per_row` aligne a 256 exige par wgpu. + pub unsafe fn readback_direct(&self) -> Result<(u32, u32, Vec)> { + let (w, h) = (self.render_w, self.render_h); + let mut encoder = self.gpu.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("readback"), + }); + encoder.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { + texture: &self.rt, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyBufferInfo { + buffer: &self.readback_buf, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(self.readback_bpr), + rows_per_image: Some(h), + }, + }, + wgpu::Extent3d { + width: w, + height: h, + depth_or_array_layers: 1, + }, + ); + self.gpu.context.submit(std::iter::once(encoder.finish())); + + let slice = self.readback_buf.slice(..); + let (tx, rx) = std::sync::mpsc::channel(); + slice.map_async(wgpu::MapMode::Read, move |r| { + let _ = tx.send(r); + }); + self.gpu.device.poll(wgpu::Maintain::Wait); + rx.recv() + .map_err(|_| anyhow::anyhow!("map_async channel"))? + .map_err(|e| anyhow::anyhow!("map_async: {e:?}"))?; + let mapped = slice.get_mapped_range(); + + let row = (w * 4) as usize; + let bpr = self.readback_bpr as usize; + let mut out = vec![0u8; row * h as usize]; + for y in 0..h as usize { + out[y * row..(y + 1) * row].copy_from_slice(&mapped[y * bpr..y * bpr + row]); + } + drop(mapped); + self.readback_buf.unmap(); + Ok((w, h, out)) + } +} diff --git a/crates/compositor/src/d3d_linux.rs b/crates/compositor/src/d3d_linux.rs new file mode 100644 index 000000000..67a701c18 --- /dev/null +++ b/crates/compositor/src/d3d_linux.rs @@ -0,0 +1,163 @@ +//! Backend GPU Linux -- wgpu (Vulkan). +//! +//! Equivalent Linux de `d3d_windows.rs` / `d3d_macos.rs` : meme surface publique +//! (`Backend`, `Gpu`, `create`, `create_backend`, `create_auto`, `probe`, +//! `diagnose`) pour que `pipeline`, `live.rs` et `compositor-view-napi` +//! l'utilisent sans connaitre la plateforme (cf. `lib.rs`, qui re-exporte +//! `crate::d3d` vers `d3d_linux` sous `cfg(target_os = "linux")`). +//! +//! # `Backend::Cpu` sur Linux +//! +//! Contrairement a macOS (ou Metal n'a pas de rasteriseur logiciel), Linux EN A +//! un : Mesa **lavapipe** (`llvmpipe`), le pendant Vulkan de WARP. `probe()` le +//! classe donc en `Backend::Cpu` (le meme repli que WARP cote Windows : notice +//! dans la preview, warning a l'export), et un vrai GPU (RADV, dzn, NVK...) en +//! `Backend::Hardware`. + +use anyhow::{Context, Result}; +use std::sync::OnceLock; + +/// Qui execute le pipeline (symetrie d'API avec `d3d_windows::Backend`). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Backend { + /// Vrai GPU (RADV / dzn / NVK / ...) via Vulkan. + Hardware, + /// Mesa lavapipe (`llvmpipe`), rasteriseur logiciel Vulkan. + Cpu, +} + +/// Handle GPU Linux : `wgpu::Device` + `wgpu::Queue` (Arc internes cote wgpu, +/// `.clone()` bon marche). Les champs `device`/`context`/`backend`/ +/// `feature_level` sont alignes sur `d3d_windows::Gpu` / `d3d_macos::Gpu` pour +/// que `live.rs::Player` copie la struct champ par champ sans cfg-fendre le +/// constructeur. +pub struct Gpu { + pub device: wgpu::Device, + /// Pendant de `ID3D11DeviceContext` (D3D11) / `MTLCommandQueue` (Metal) : + /// la file de soumission wgpu. + pub context: wgpu::Queue, + pub backend: Backend, + /// Pas d'equivalent `D3D_FEATURE_LEVEL` en wgpu ; conserve a 0 pour la + /// symetrie d'API (les diagnostics futurs pourront le renseigner). + pub feature_level: u64, +} + +/// `probe()` -- propriete de la machine, mise en cache (la preview et la modale +/// d'export en ont besoin toutes les deux). `None` si aucun adaptateur wgpu +/// (headless sans lavapipe, kernel sans DRM ni ICD logiciel). +static PROBE: OnceLock> = OnceLock::new(); + +pub fn probe() -> Option { + *PROBE.get_or_init(|| create_backend(Backend::Hardware).ok().map(|g| g.backend)) +} + +/// Cree un device wgpu (Vulkan). `_backend` est indicatif : on prend le meilleur +/// adaptateur disponible (HighPerformance) et on reporte son type REEL via +/// `classify` (lavapipe -> `Cpu`, sinon `Hardware`) -- pas de chemin de rendu +/// distinct entre les deux cote Linux, seul le libelle change. +pub fn create_backend(_backend: Backend) -> Result { + pollster::block_on(create_async()) +} + +async fn create_async() -> Result { + let instance = wgpu::Instance::new(&wgpu::InstanceDescriptor { + backends: wgpu::Backends::all(), + ..Default::default() + }); + let adapter = instance + .request_adapter(&wgpu::RequestAdapterOptions { + power_preference: wgpu::PowerPreference::HighPerformance, + ..Default::default() + }) + .await + .context("aucun adaptateur graphique compatible")?; + let info = adapter.get_info(); + let backend = classify(&info); + let (device, queue) = adapter + .request_device( + &wgpu::DeviceDescriptor { + label: Some("openscreen-linux"), + required_features: wgpu::Features::empty(), + required_limits: wgpu::Limits::default(), + memory_hints: wgpu::MemoryHints::default(), + }, + None, + ) + .await + .context("request_device a echoue")?; + Ok(Gpu { + device, + context: queue, + backend, + feature_level: 0, + }) +} + +/// lavapipe expose "llvmpipe" dans le nom d'adaptateur -- c'est l'equivalent +/// Vulkan de WARP, a ranger sous `Cpu`. +fn classify(info: &wgpu::AdapterInfo) -> Backend { + let n = info.name.to_ascii_lowercase(); + if n.contains("llvmpipe") || n.contains("lavapipe") { + Backend::Cpu + } else { + Backend::Hardware + } +} + +impl Gpu { + /// Chemin de production. Symetrie d'API avec `d3d_windows::Gpu::create_auto` ; + /// `_debug` est le pendant de la couche de debug D3D11 (rien a faire ici, + /// wgpu a `WGPU_VALIDATION` en variable d'env). + pub fn create_auto(_debug: bool) -> Result { + create_backend(Backend::Hardware) + } + + /// Creation hardware-strict (tests et goldens). + pub fn create(_debug: bool) -> Result { + create_backend(Backend::Hardware) + } + + /// Le backend de cette machine, mis en cache. Expose comme fonction ASSOCIEE + /// (`Gpu::probe()`) parce que `compositor-view-napi` l'appelle ainsi. + pub fn probe() -> Option { + probe() + } +} + +/// Message d'echec actionnable (symetrie d'API avec `d3d_windows::diagnose`). +pub fn diagnose(err: &anyhow::Error) -> String { + format!("{err:#}") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn classify_cpu_pour_lavapipe() { + let info = wgpu::AdapterInfo { + name: "llvmpipe (LLVM 21.1.8, 256 bits)".into(), + vendor: 0x10005, + device: 0, + device_type: wgpu::DeviceType::Cpu, + driver: "llvmpipe".into(), + driver_info: String::new(), + backend: wgpu::Backend::Vulkan, + }; + assert_eq!(classify(&info), Backend::Cpu); + } + + #[test] + fn classify_hardware_pour_gpu_reel() { + let info = wgpu::AdapterInfo { + name: "Microsoft Direct3D12 (AMD Radeon(TM) Graphics)".into(), + vendor: 0x1002, + device: 0, + device_type: wgpu::DeviceType::IntegratedGpu, + driver: "Dozen".into(), + driver_info: String::new(), + backend: wgpu::Backend::Vulkan, + }; + assert_eq!(classify(&info), Backend::Hardware); + } +} diff --git a/crates/compositor/src/lib.rs b/crates/compositor/src/lib.rs index 79ea40c73..1bbd293d8 100644 --- a/crates/compositor/src/lib.rs +++ b/crates/compositor/src/lib.rs @@ -52,6 +52,11 @@ pub mod d3d_macos; #[cfg(target_os = "macos")] pub use d3d_macos as d3d; +#[cfg(target_os = "linux")] +pub mod d3d_linux; +#[cfg(target_os = "linux")] +pub use d3d_linux as d3d; + // Source de frames du backend « CPU-like » : Windows → cpu_frames_windows (WARP + swscale), // macOS → mac_frames (logiciel → CVPixelBuffer). Ré-exporté sous le nom `cpu_frames` (privé). #[cfg(windows)] @@ -64,6 +69,13 @@ mod mac_frames; #[cfg(target_os = "macos")] use mac_frames as cpu_frames; +#[cfg(target_os = "linux")] +mod linux_frames; +#[cfg(target_os = "linux")] +use linux_frames as cpu_frames; +#[cfg(target_os = "linux")] +mod linux_decode; + // Moteur de composition + rastériseur de texte + pipeline : un fichier par plateforme. // Le pipeline est gardé séparé (pas de fusion comme live) parce que la ffmpeg-side // diffère entre D3D11VA et VideoToolbox : les types `AVD3D11VADeviceContext` vs @@ -84,6 +96,13 @@ pub mod pipeline_macos; #[cfg(target_os = "macos")] pub mod text_macos; +#[cfg(target_os = "linux")] +pub mod text_linux; +#[cfg(target_os = "linux")] +pub mod compositor_linux; +#[cfg(target_os = "linux")] +pub mod pipeline_linux; + #[cfg(windows)] pub use compositor_windows as compositor; #[cfg(windows)] @@ -98,6 +117,13 @@ pub use pipeline_macos as pipeline; #[cfg(target_os = "macos")] pub use text_macos as text; +#[cfg(target_os = "linux")] +pub use text_linux as text; +#[cfg(target_os = "linux")] +pub use compositor_linux as compositor; +#[cfg(target_os = "linux")] +pub use pipeline_linux as pipeline; + // `live.rs` est resté un fichier unique parce que sa machinerie principale (Player, // LiveView, render_thread) est entièrement cross-platform : elle ne touche qu'au // Compositor (cfg-ré-exporté) et au ffmpeg `Decoder` (portable). Seules les diff --git a/crates/compositor/src/linux_decode.rs b/crates/compositor/src/linux_decode.rs new file mode 100644 index 000000000..3d098d2c6 --- /dev/null +++ b/crates/compositor/src/linux_decode.rs @@ -0,0 +1,327 @@ +//! D├®codeur logiciel ffmpeg ÔÇö utilis├® par la tranche verticale `vk_render` +//! pour ouvrir un MP4 fixture et en extraire la `n`-i├¿me frame en m├®moire +//! syst├¿me, sans aucune d├®pendance ├á `D3D11VA`. C├┤t├® production, ce sera +//! `pipeline::Decoder::open` c├┤t├® Windows (qui route par D3D11VA quand FL 11_1 +//! + vid├®o disponible, par `vk_frames::VkFrames` sinon) ; ici on isole le +//! chemin ┬½ software decode + `vk_frames::present` ┬╗ pour le d├®montrer sans +//! toucher `pipeline.rs` (cf. spec ┬º3.4 ÔÇö `pipeline.rs` est dans WP6). +//! +//! **S├®curit├® lifetime.** L'`AVFrame` retourn├® est allou├® par `av_frame_alloc` +//! et lib├®r├® par `av_frame_free` ÔÇö le caller doit soit appeler `free_frame()` +//! soit (mieux) laisser `vk_frames::VkFrames::present` la consommer puis +//! r├®├®crire la prochaine. Garder une frame au-del├á du prochain `decode_n` +//! lib├¿re l'ancienne, exactement comme `cpu_frames::present` c├┤t├® #162. + +use anyhow::{bail, Context, Result}; +use std::ffi::CString; +use std::ptr; + +use crate::ffi::{ + av_frame_alloc, av_frame_free, av_frame_move_ref, av_frame_unref, av_packet_alloc, + av_packet_free, av_read_frame, av_seek_frame, avcodec_alloc_context3, avcodec_find_decoder, + avcodec_flush_buffers, avcodec_free_context, avcodec_open2, avcodec_parameters_to_context, + avcodec_receive_frame, avcodec_send_packet, avformat_close_input, avformat_find_stream_info, + avformat_open_input, AVCodecContext, AVFormatContext, AVFrame, AVMediaType, AVStream, +}; + +/// `sn_fmt_stream` est d├®fini dans `crates/compositor/shim.c` ÔÇö bindgen ne le voit pas +/// (shim.c est compil├® s├®par├®ment par `cc::Build`). On le d├®clare ici en `extern "C"` +/// comme `pipeline.rs` le fait. La m├¬me convention appara├«t ├á plusieurs endroits du +/// crate pour tous les accesseurs du shim. +extern "C" { + fn sn_fmt_stream(s: *mut AVFormatContext, i: i32) -> *mut AVStream; +} + +/// `SEEK_SET` constant ÔÇö la position de seek `av_seek_frame` interpr├¿te +/// `timestamp` comme un timestamp absolu (AV_TIME_BASE = microsecondes). +const SEEK_SET: i32 = 0; + +/// Cherche la vid├®o du fichier, ouvre le d├®codeur, et rend un ├®tat pr├¬t ├á +/// d├®coder. La struct expose `decode_at(frame_idx)` qui seek + d├®code jusqu'├á +/// la frame `frame_idx` (0-index├®e depuis le d├®but du flux). +/// +/// Pub (pas `pub(crate)`) parce que `crates/compositor/tests/vk_cross_golden.rs` +/// est un crate externe vis-├á-vis de la lib ; le test pilote la tranche. +pub struct SwDecoder { + fmt: *mut AVFormatContext, + dec: *mut AVCodecContext, + stream_idx: i32, + /// Timebase du flux vid├®o (en secondes par tick). Permet de convertir un + /// `frame_idx` en timestamp de seek. + stream_timebase: f64, + /// Cadence reelle du flux (avg_frame_rate), PAS 1/time_base. + fps: f64, +} + +/// Lib├¿re toutes les ressources ffmpeg. `Drop` ne peut pas faillir ; on +/// panique sur une erreur double-free improbable (les handles sont nullifi├®s +/// apr├¿s lib├®ration, un deuxi├¿me `Drop` les trouve ├á null et n'agit pas). +impl Drop for SwDecoder { + fn drop(&mut self) { + unsafe { + if !self.dec.is_null() { + avcodec_free_context(&mut self.dec); + } + if !self.fmt.is_null() { + avformat_close_input(&mut self.fmt); + } + } + } +} + +impl SwDecoder { + pub fn open(path: &str) -> Result { + unsafe { Self::open_inner(path) } + } + + unsafe fn open_inner(path: &str) -> Result { + let path_c = CString::new(path).context("chemin NUL inattendu")?; + let mut fmt: *mut AVFormatContext = ptr::null_mut(); + let r = avformat_open_input(&mut fmt, path_c.as_ptr(), ptr::null(), ptr::null_mut()); + if r < 0 { + bail!("avformat_open_input({path}) a ├®chou├®: {r}"); + } + if fmt.is_null() { + bail!("avformat_open_input({path}) a rendu un fmt null"); + } + let r = avformat_find_stream_info(fmt, ptr::null_mut()); + if r < 0 { + avformat_close_input(&mut fmt); + bail!("avformat_find_stream_info({path}) a ├®chou├®: {r}"); + } + // Trouver le premier flux vid├®o. `av_find_best_stream` fait ├ºa 1.0. + let stream_idx = crate::ffi::av_find_best_stream( + fmt, + AVMediaType::AVMEDIA_TYPE_VIDEO, + -1, + -1, + ptr::null_mut(), + 0, + ); + if stream_idx < 0 { + avformat_close_input(&mut fmt); + bail!("av_find_best_stream n'a pas trouv├® de flux vid├®o dans {path}: {stream_idx}"); + } + // Codec params ÔåÆ context ÔåÆ open. AVFormatContext est opaque : `sn_fmt_stream` + // (du `shim.c`) extrait `streams[i]` ; AVStream ne l'est pas, on lit son + // `codecpar` directement. Cf. `pipeline.rs` pour la convention. + let stream = sn_fmt_stream(fmt, stream_idx); + let mut dec = avcodec_alloc_context3(ptr::null()); + if dec.is_null() { + avformat_close_input(&mut fmt); + bail!("avcodec_alloc_context3 a ├®chou├®"); + } + let par = (*stream).codecpar; + let r = avcodec_parameters_to_context(dec, par); + if r < 0 { + avcodec_free_context(&mut dec); + avformat_close_input(&mut fmt); + bail!("avcodec_parameters_to_context: {r}"); + } + let codec = avcodec_find_decoder((*par).codec_id); + if codec.is_null() { + avcodec_free_context(&mut dec); + avformat_close_input(&mut fmt); + bail!( + "avcodec_find_decoder n'a pas trouv├® de d├®codeur pour codec_id {}", + (*par).codec_id + ); + } + let r = avcodec_open2(dec, codec, ptr::null_mut()); + if r < 0 { + avcodec_free_context(&mut dec); + avformat_close_input(&mut fmt); + bail!("avcodec_open2: {r}"); + } + // Timebase du flux vid├®o ÔÇö `AVRational { num, den }`. ffmpeg utilise `num` ticks + // par `den` secondes. Le wrapper bindgen expose les deux champs en i32. + let stream_timebase = { + let num = (*stream).time_base.num as f64; + let den = (*stream).time_base.den as f64; + if den == 0.0 { + 1.0 / 60.0 // fallback : suppose 60 fps + } else { + num / den + } + }; + // fps reel du flux : avg_frame_rate d'abord, r_frame_rate en secours, + // 60 en dernier recours. PAS 1/time_base (le time_base est le timescale + // du conteneur, souvent 15360, pas la cadence). + let fps = { + let a = (*stream).avg_frame_rate; + let r = (*stream).r_frame_rate; + if a.num > 0 && a.den > 0 { + a.num as f64 / a.den as f64 + } else if r.num > 0 && r.den > 0 { + r.num as f64 / r.den as f64 + } else { + 60.0 + } + }; + Ok(SwDecoder { + fmt, + dec, + stream_idx, + stream_timebase, + fps, + }) + } + + /// Seek vers la keyframe la plus proche AVANT `frame_idx`, puis d├®code + /// jusqu'├á atteindre la frame demand├®e. Le seek est r├®solu par + /// `av_seek_frame` avec `SEEK_SET | BACKWARD` (cherche le keyframe + /// pr├®c├®dent le timestamp demand├®). Renvoie une `AVFrame` allou├®e par + /// `av_frame_alloc` que le caller doit lib├®rer via `free_frame` ÔÇö + /// ou laisser `vk_frames::VkFrames::present` consommer (qui r├®├®crit + /// `present` avec son carrier, l'ancienne frame devient inaccessible). + /// + /// **Robustesse.** Pour la tranche verticale (`crates/fixture/screen.mp4` + /// qui est un `-c copy` d'un fragment de recording), `av_seek_frame` peut + /// renvoyer un packet dont la premi├¿re lecture NAL est mal align├®e (le + /// moov de la source est en queue, le parser fait de son mieux mais le + /// premier packet apr├¿s un BACKWARD seek contient parfois un NAL + /// fragment├®). On skippe ces packets avec `send_packet` qui renvoie + /// `AVERROR_INVALIDDATA` plut├┤t que de paniquer : la prochaine it├®ration + /// lira le packet complet suivant. + pub unsafe fn decode_at(&mut self, frame_idx: u32) -> Result<*mut AVFrame> { + let fps = self.fps; + let target_ts = (frame_idx as f64 / fps) * 1_000_000.0; // AV_TIME_BASE = ┬Ás + // BACKWARD = 4 (chercher la keyframe pr├®c├®dente). Cf. ffmpeg `av_seek_flag`. + let seek_flags = SEEK_SET | 4; + let r = av_seek_frame(self.fmt, -1, target_ts as i64, seek_flags); + if r < 0 { + bail!("av_seek_frame(ts={target_ts:.0} ┬Ás) a ├®chou├®: {r}"); + } + // Flush le d├®codeur ÔÇö sans ├ºa, le seek laisse l'├®tat interne avec les + // frames de l'ancien GOP, et la premi├¿re `receive_frame` peut ├¬tre + // une frame d'avant le seek. + avcodec_flush_buffers(self.dec); + + let mut pkt: *mut crate::ffi::AVPacket = ptr::null_mut(); + let mut frame: *mut AVFrame = ptr::null_mut(); + let mut found: *mut AVFrame = ptr::null_mut(); + + let target_ts_seconds = target_ts / 1_000_000.0; + let mut invalid_skips = 0u32; + 'outer: loop { + pkt = av_packet_alloc(); + if pkt.is_null() { + bail!("av_packet_alloc en boucle"); + } + let r = av_read_frame(self.fmt, pkt); + if r < 0 { + // EOF ou erreur : on a ├®puis├® le fichier sans atteindre la cible. + av_packet_free(&mut pkt); + break 'outer; + } + if (*pkt).stream_index != self.stream_idx { + // Pas un packet vid├®o ÔÇö on le jette et on continue. + av_packet_free(&mut pkt); + continue; + } + let send_r = avcodec_send_packet(self.dec, pkt); + av_packet_free(&mut pkt); + if send_r == -0x2A2A2A2A { + // AVERROR_INVALIDDATA ÔÇö packet mal align├® apr├¿s un seek. On le + // saute et on continue ; le decodeur attendra un packet propre. + // Valeur ffmpeg = -1094995529 (0xBEEBBEEB), ici ├®crite comme + // un nombre n├®gatif litt├®ral pour ├®viter la d├®pendance `ffi::`. + invalid_skips += 1; + if invalid_skips > 8 { + bail!("plus de 8 packets invalides apr├¿s seek ÔÇö fichier ou codec cass├®"); + } + continue; + } + if send_r < 0 && send_r != -11 { + bail!("avcodec_send_packet: {send_r}"); + } + frame = av_frame_alloc(); + if frame.is_null() { + bail!("av_frame_alloc en boucle"); + } + loop { + let recv_r = avcodec_receive_frame(self.dec, frame); + if recv_r == 0 { + if found.is_null() { + found = av_frame_alloc(); + if found.is_null() { + bail!("av_frame_alloc pour resultat"); + } + } + av_frame_move_ref(found, frame); + av_frame_unref(frame); + if (*found).best_effort_timestamp as f64 * self.stream_timebase >= target_ts_seconds { + break 'outer; + } + } else if recv_r == -11 { + break; + } else if recv_r == -541478725 { + // AVERROR_EOF + break 'outer; + } else if recv_r < 0 { + bail!("avcodec_receive_frame: {recv_r}"); + } else { + break; + } + } + av_frame_free(&mut frame); + } + if !frame.is_null() { + av_frame_free(&mut frame); + } + if !pkt.is_null() { + av_packet_free(&mut pkt); + } + if found.is_null() { + bail!("decode_at(frame_idx={frame_idx}) : aucune frame re├ºue"); + } + Ok(found) + } + + /// Cadence reelle du flux (images/s). Sert a convertir un temps en secondes + /// vers un index de frame pour le seek du preview Linux. + pub fn fps(&self) -> f64 { + self.fps + } + + /// Duree du flux video en secondes (`stream.duration * time_base`), lue via + /// le shim `sn_fmt_stream` (AVFormatContext opaque en bindgen). `None` si + /// indisponible. Pendant Linux de `pipeline_macos::Decoder::available_duration_sec`. + pub fn duration_sec(&self) -> Option { + unsafe { + let stream = sn_fmt_stream(self.fmt, self.stream_idx); + if stream.is_null() { + return None; + } + let duration = (*stream).duration; + if duration > 0 && self.stream_timebase > 0.0 { + let s = duration as f64 * self.stream_timebase; + if s.is_finite() && s > 0.0 { + return Some(s); + } + } + None + } + } + + /// Lib├¿re une frame renvoy├®e par `decode_at`. + pub unsafe fn free_frame(mut frame: *mut AVFrame) { + av_frame_free(&mut frame); + } +} + +// ---------- tests ---------- + +#[cfg(test)] +mod tests { + use super::*; + + /// Le d├®codeur ne paniquera pas si le fichier n'existe pas ÔÇö il renvoie + /// `Err`. C'est ce que le test d'int├®gration attend pour skipper proprement + /// quand `crates/fixture/screen.mp4` est absent. + #[test] + fn open_sur_chemin_inexistant_renvoie_err() { + let r = SwDecoder::open("Z:/does/not/exist.mp4"); + assert!(r.is_err()); + } +} diff --git a/crates/compositor/src/linux_frames.rs b/crates/compositor/src/linux_frames.rs new file mode 100644 index 000000000..dcb99bedb --- /dev/null +++ b/crates/compositor/src/linux_frames.rs @@ -0,0 +1,350 @@ +//! L'axe DECODAGE du backend « CPU-like » Linux : une frame libavcodec en +//! memoire systeme devient deux textures wgpu NV12-split (Y `R8Unorm`, UV +//! entrelacee `Rg8Unorm`), presentees exactement comme si un decodeur materiel +//! les avait produites. +//! +//! Equivalent Linux de `cpu_frames_windows.rs` / `mac_frames.rs`. Meme contrat +//! de « frame seam » (cf. `mac_frames.rs:12-19`) : `compositor::nv12_srvs()` +//! lit quatre champs de l'AVFrame presentee — +//! - `data[0]` : un carrier `Box` (les deux textures wgpu), +//! opaque cote Rust, relu par `compositor_linux::nv12_srvs`, +//! - `data[1]` : 0 (pas d'array), +//! - `width`/`height` : dimensions visibles. +//! `format` est pose a `AV_PIX_FMT_D3D11` comme sur Windows/macOS : un sentinel +//! « buffer GPU natif dans data[0] », jamais inspecte par ffmpeg dans ce pipeline. +//! +//! # Difference avec D3D11/Metal +//! +//! D3D11 a un format NV12 natif (une texture, deux sous-ressources) ; macOS a +//! le CVPixelBuffer IOSurface (zero-copy via CVMetalTextureCache). Vulkan/wgpu +//! n'a pas de format NV12 portable, donc on le decompose en DEUX textures +//! (Y + UV) uploadees par `write_texture`. Le shader WGSL echantillonne les deux +//! et fait le YUV->RGB (cf. `vk_shaders/layer.wgsl`). + +use anyhow::{bail, Result}; +use std::ptr; + +use crate::d3d::Gpu; +use crate::ffi::{ + av_frame_alloc, av_frame_free, av_frame_get_buffer, av_frame_unref, sws_freeContext, + sws_getContext, sws_scale, AVFrame, AVPixelFormat, SwsContext, +}; + +/// `SWS_POINT` (plus proche voisin) : la conversion se fait a dimensions EGALES, +/// aucun reechantillonnage. Valeur figee par l'ABI de libswscale (bindgen ne +/// genere pas les `SWS_*`, ce sont des macros). +const SWS_POINT: i32 = 0x10; + +/// Une frame decodee presentee au compositor sous forme de deux textures wgpu : +/// plane Y (`R8Unorm`, `w x h`) et plane UV entrelacee (`Rg8Unorm`, +/// `(w/2) x (h/2)`). Equivalent NV12-split de la `ID3D11Texture2D` NV12 (D3D11) +/// / du CVPixelBuffer (macOS). +pub(crate) struct VkFrameTex { + pub y: wgpu::Texture, + pub uv: wgpu::Texture, + pub width: u32, + pub height: u32, +} + +#[inline] +fn pack_carrier(tex: Box) -> *mut u8 { + Box::into_raw(tex) as *mut u8 +} + +#[inline] +unsafe fn unpack_carrier<'a>(p: *const u8) -> &'a VkFrameTex { + debug_assert!(!p.is_null()); + &*(p as *const VkFrameTex) +} + +/// Source de frames du backend « CPU-like » Linux. Meme surface que +/// `mac_frames::CpuFrames` (`new` / `present` / `current`) : `pipeline` garde la +/// meme mecanique. Allocation unique de textures reecrites a chaque frame. +pub(crate) struct CpuFrames { + device: wgpu::Device, + queue: wgpu::Queue, + sws: *mut SwsContext, + /// `(w, h, format source)` du contexte swscale courant. + sws_key: (i32, i32, i32), + /// NV12 en memoire systeme : cible de swscale, source de l'upload. + nv12: *mut AVFrame, + tex: Option>, + tex_dims: (u32, u32), + /// La frame remise au compositor. Ne possede aucun pixel : `data[0]` pointe + /// le carrier `VkFrameTex`. + present: *mut AVFrame, +} + +impl CpuFrames { + pub(crate) fn new(gpu: &Gpu) -> Result { + let present = unsafe { av_frame_alloc() }; + let nv12 = unsafe { av_frame_alloc() }; + if present.is_null() || nv12.is_null() { + bail!("av_frame_alloc (linux_frames)"); + } + Ok(CpuFrames { + device: gpu.device.clone(), + queue: gpu.context.clone(), + sws: ptr::null_mut(), + sws_key: (0, 0, -1), + nv12, + tex: None, + tex_dims: (0, 0), + present, + }) + } + + /// Convertit `src` (sortie decodeur, memoire systeme) en NV12, l'uploade dans + /// les textures wgpu, et rend la frame de presentation dont `data[0]` est un + /// carrier `Box`. Le pointeur reste valide jusqu'au prochain + /// `present()` — meme contrat que `mac_frames::present`. + pub(crate) unsafe fn present(&mut self, src: *mut AVFrame) -> Result<*mut AVFrame> { + if src.is_null() { + bail!("linux_frames::present: frame source nulle"); + } + let w = (*src).width; + let h = (*src).height; + if w <= 0 || h <= 0 { + bail!("frame decodee sans dimensions ({w}x{h})"); + } + self.ensure_sws(w, h, (*src).format)?; + self.ensure_nv12(w, h)?; + self.ensure_textures(w as u32, h as u32)?; + + let converted = sws_scale( + self.sws, + (*src).data.as_ptr() as *const *const u8, + (*src).linesize.as_ptr(), + 0, + h, + (*self.nv12).data.as_ptr(), + (*self.nv12).linesize.as_ptr(), + ); + if converted <= 0 { + bail!("sws_scale a converti {converted} lignes"); + } + + self.upload()?; + self.attach_carrier(w, h)?; + // Contrat lu par le compositor : sentinel + timestamps recopies (sinon la + // timeline se croit a t=0). + (*self.present).format = AVPixelFormat::AV_PIX_FMT_D3D11 as i32; + (*self.present).pts = (*src).pts; + (*self.present).best_effort_timestamp = (*src).best_effort_timestamp; + Ok(self.present) + } + + unsafe fn ensure_sws(&mut self, w: i32, h: i32, src_fmt: i32) -> Result<()> { + let key = (w, h, src_fmt); + if self.sws_key == key && !self.sws.is_null() { + return Ok(()); + } + if !self.sws.is_null() { + sws_freeContext(self.sws); + } + self.sws = sws_getContext( + w, + h, + src_fmt as AVPixelFormat::Type, + w, + h, + AVPixelFormat::AV_PIX_FMT_NV12, + SWS_POINT, + ptr::null_mut(), + ptr::null_mut(), + ptr::null(), + ); + if self.sws.is_null() { + bail!("sws_getContext {w}x{h} fmt {src_fmt} -> NV12"); + } + self.sws_key = key; + Ok(()) + } + + unsafe fn ensure_nv12(&mut self, w: i32, h: i32) -> Result<()> { + if (*self.nv12).width == w + && (*self.nv12).height == h + && (*self.nv12).format == AVPixelFormat::AV_PIX_FMT_NV12 as i32 + { + return Ok(()); + } + av_frame_unref(self.nv12); + (*self.nv12).width = w; + (*self.nv12).height = h; + (*self.nv12).format = AVPixelFormat::AV_PIX_FMT_NV12 as i32; + if av_frame_get_buffer(self.nv12, 32) < 0 { + bail!("av_frame_get_buffer NV12 {w}x{h}"); + } + Ok(()) + } + + fn ensure_textures(&mut self, w: u32, h: u32) -> Result<()> { + // NV12 impose des dimensions paires pour le chroma : arrondi au-dessus pour + // les textures, `present.width/height` reste aux dimensions visibles (meme + // ecart texture/visible que l'alignement macrobloc D3D11VA, 1080 -> 1088). + let dims = ((w + 1) & !1, (h + 1) & !1); + if let Some(tex) = &self.tex { + if tex.width == dims.0 && tex.height == dims.1 { + return Ok(()); + } + } + let y = self.device.create_texture(&wgpu::TextureDescriptor { + label: Some("nv12-y"), + size: wgpu::Extent3d { + width: dims.0, + height: dims.1, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::R8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + let uv = self.device.create_texture(&wgpu::TextureDescriptor { + label: Some("nv12-uv"), + size: wgpu::Extent3d { + width: dims.0 / 2, + height: dims.1 / 2, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rg8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + self.tex = Some(Box::new(VkFrameTex { + y, + uv, + width: dims.0, + height: dims.1, + })); + self.tex_dims = dims; + Ok(()) + } + + /// Upload du NV12 swscale dans les deux textures wgpu. `linesize[0]/[1]` sont + /// les strides memoire (paddes SIMD par swscale), passes tels quels a + /// `bytes_per_row`. + unsafe fn upload(&mut self) -> Result<()> { + let tex = match self.tex.as_ref() { + Some(t) => t, + None => bail!("upload avant ensure_textures"), + }; + let y_stride = (*self.nv12).linesize[0] as usize; + let uv_stride = (*self.nv12).linesize[1] as usize; + let y_size = y_stride * tex.height as usize; + let uv_size = uv_stride * tex.height.div_ceil(2) as usize; + self.queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture: &tex.y, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + std::slice::from_raw_parts((*self.nv12).data[0], y_size), + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(y_stride as u32), + rows_per_image: Some(tex.height), + }, + wgpu::Extent3d { + width: tex.width, + height: tex.height, + depth_or_array_layers: 1, + }, + ); + self.queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture: &tex.uv, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + std::slice::from_raw_parts((*self.nv12).data[1], uv_size), + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(uv_stride as u32), + rows_per_image: Some(tex.height / 2), + }, + wgpu::Extent3d { + width: tex.width / 2, + height: tex.height / 2, + depth_or_array_layers: 1, + }, + ); + Ok(()) + } + + /// Attache le carrier `Box` a `present.data[0]` et fixe les + /// dimensions visibles (avant padding pair). + unsafe fn attach_carrier(&mut self, w: i32, h: i32) -> Result<()> { + let tex = match self.tex.as_ref() { + Some(t) => t, + None => bail!("attach_carrier avant ensure_textures"), + }; + // Libere un carrier precedent eventuel avant de le remplacer. + if !(*self.present).data[0].is_null() { + let _ = Box::from_raw((*self.present).data[0] as *mut VkFrameTex); + } + (*self.present).data[0] = pack_carrier(Box::new(VkFrameTex { + y: tex.y.clone(), + uv: tex.uv.clone(), + width: tex.width, + height: tex.height, + })); + (*self.present).data[1] = ptr::null_mut(); + (*self.present).width = w; + (*self.present).height = h; + Ok(()) + } + + /// La frame de presentation courante (jamais nulle) — symetrie d'API avec + /// `mac_frames::CpuFrames::current`. + pub(crate) fn current(&self) -> *mut AVFrame { + self.present + } +} + +/// Dimensions (texture, padded pair) du carrier `frame.data[0]`. `(1, 1)` si nul. +pub(crate) unsafe fn carrier_dims(frame: *const AVFrame) -> (u32, u32) { + if (*frame).data[0].is_null() { + return (1, 1); + } + let tex = unpack_carrier((*frame).data[0]); + (tex.width, tex.height) +} + +/// Equivalent Linux de `nv12_srvs` : retourne les deux `TextureView` samplables +/// depuis le carrier `frame.data[0]`. Appele par `compositor_linux`. +pub(crate) unsafe fn nv12_planes( + frame: *const AVFrame, +) -> Result<(wgpu::TextureView, wgpu::TextureView)> { + if (*frame).data[0].is_null() { + bail!("nv12_planes: carrier nul dans data[0]"); + } + let tex = unpack_carrier((*frame).data[0]); + Ok(( + tex.y.create_view(&wgpu::TextureViewDescriptor::default()), + tex.uv.create_view(&wgpu::TextureViewDescriptor::default()), + )) +} + +impl Drop for CpuFrames { + fn drop(&mut self) { + unsafe { + if !self.sws.is_null() { + sws_freeContext(self.sws); + } + if !(*self.present).data[0].is_null() { + let _ = Box::from_raw((*self.present).data[0] as *mut VkFrameTex); + (*self.present).data[0] = ptr::null_mut(); + } + av_frame_free(&mut self.present); + av_frame_free(&mut self.nv12); + } + } +} diff --git a/crates/compositor/src/pipeline_linux.rs b/crates/compositor/src/pipeline_linux.rs new file mode 100644 index 000000000..2693f6ff3 --- /dev/null +++ b/crates/compositor/src/pipeline_linux.rs @@ -0,0 +1,160 @@ +//! Pipeline Linux (PR #183) : decode software (`linux_decode::SwDecoder`) + +//! upload NV12-split (`linux_frames::CpuFrames`). +//! +//! Equivalent Linux de `pipeline_windows.rs` / `pipeline_macos.rs` : meme +//! surface publique consommee par le code partage (`Decoder`, `ClipSource`, +//! `ExportCodec`, `ExportParams`, `Stats`, `run_composited_multi`). +//! +//! **Export.** `run_composited_multi` (encode + mux MP4) est un **stub** ici, +//! comme `pipeline_macos` stube plusieurs de ses propres fonctions d'export : +//! le cablage VAAPI/libopenh264 + muxer (via le shim C pour `AVFormatContext. +//! streams`, opaque en bindgen) arrive avec WP6. La PREVIEW (le `Decoder`) +//! est, elle, complete -- c'est ce dont `live.rs` a besoin. + +use anyhow::{bail, Result}; +use std::ptr; + +use crate::config::Cfg; +use crate::d3d::Gpu; +use crate::ffi::AVFrame; +use crate::linux_decode::SwDecoder; +use crate::linux_frames::CpuFrames; + +/// Bilan d'un run d'export. Memes champs que `pipeline_macos::Stats`. +pub struct Stats { + pub frames: u64, + pub wall_s: f64, + pub fps: f64, + pub video_duration_s: f64, +} + +/// Un clip de la timeline. Memes champs que `pipeline_macos::ClipSource`. +pub struct ClipSource { + pub screen: String, + pub webcam: String, + pub source_start_sec: f64, + pub source_end_sec: f64, + pub webcam_offset_sec: f64, + pub has_audio: bool, +} + +/// Codec cible. Memes variantes que `pipeline_macos::ExportCodec`. +#[derive(Clone, Copy, Debug)] +pub enum ExportCodec { + H264, + H265, +} + +/// Params d'export. Memes champs que `pipeline_macos::ExportParams`. +pub struct ExportParams { + pub width: u32, + pub height: u32, + pub fps: Option, + pub codec: ExportCodec, +} + +impl Default for ExportParams { + fn default() -> Self { + Self { + width: 1920, + height: 1080, + fps: None, + codec: ExportCodec::H264, + } + } +} + +/// Decodeur Linux : software decode (`SwDecoder`) + upload NV12-split +/// (`CpuFrames`). Meme surface que `pipeline_macos::Decoder` +/// (`open`/`seek_to`/`next`/`cur_frame`/`cur_time_sec`/`fps`) pour que `live.rs` +/// le pilote sans connaitre la plateforme. +pub struct Decoder { + sw: SwDecoder, + frames: CpuFrames, + cur: *mut AVFrame, + /// Index de la prochaine frame a decoder (sequentiel). + next_idx: u32, + fps: f64, +} + +// SAFETY : les pointeurs FFI n'ont pas d'affinite thread ; le caller uphold la +// regle « un thread a la fois » (idem `pipeline_macos::Decoder`). +unsafe impl Send for Decoder {} + +impl Decoder { + pub fn open(path: &str, gpu: &Gpu) -> Result { + let sw = SwDecoder::open(path)?; + let fps = sw.fps(); + let frames = CpuFrames::new(gpu)?; + Ok(Decoder { + sw, + frames, + cur: ptr::null_mut(), + next_idx: 0, + fps, + }) + } + + /// Decode la frame a `seconds` (seek), la presente en carrier, la retourne. + pub unsafe fn seek_to(&mut self, seconds: f64) -> Result<*mut AVFrame> { + let idx = (seconds.max(0.0) * self.fps).round() as u32; + self.decode_present(idx) + } + + /// Decode la frame SEQUENTIELLE suivante. + // ponytail: `decode_at` par index re-seek a chaque frame -- OK pour le + // scrub/preview, a optimiser en WP si le bench de lecture l'exige. + pub unsafe fn next(&mut self) -> Result<*mut AVFrame> { + let idx = self.next_idx; + self.decode_present(idx) + } + + unsafe fn decode_present(&mut self, idx: u32) -> Result<*mut AVFrame> { + let raw = self.sw.decode_at(idx)?; + let carrier = self.frames.present(raw)?; + SwDecoder::free_frame(raw); + self.cur = carrier; + self.next_idx = idx + 1; + Ok(carrier) + } + + pub unsafe fn cur_frame(&self) -> *mut AVFrame { + self.cur + } + + /// Temps source (secondes) de la frame courante. + pub unsafe fn cur_time_sec(&self) -> f64 { + if self.next_idx == 0 || self.fps <= 0.0 { + 0.0 + } else { + (self.next_idx as f64 - 1.0) / self.fps + } + } + + pub unsafe fn fps(&self) -> f64 { + self.fps + } + + /// Duree du flux (secondes). Pendant de + /// `pipeline_macos::Decoder::available_duration_sec` ; consomme par + /// `timeline_walk` pour borner la marche d'export. + pub unsafe fn available_duration_sec(&self) -> Option { + self.sw.duration_sec() + } +} + +/// Export multiclip -- **STUB (WP6)**. L'encode (VAAPI/libopenh264) + le muxer +/// MP4 (via shim C pour `AVFormatContext.streams`) arrivent plus tard ; +/// `pipeline_macos` stube de meme son chemin d'export. On echoue lisiblement +/// plutot que d'ecrire un fichier vide. +pub fn run_composited_multi( + _clips: &[ClipSource], + _out: &str, + _gpu: &Gpu, + _comp: &crate::compositor::Compositor, + _cfg: &Cfg, + _params: &ExportParams, + _progress: &mut dyn FnMut(u64), +) -> Result { + bail!("run_composited_multi: export natif Linux pas encore implemente (WP6)") +} diff --git a/crates/compositor/src/text_linux.rs b/crates/compositor/src/text_linux.rs new file mode 100644 index 000000000..b80bf0e06 --- /dev/null +++ b/crates/compositor/src/text_linux.rs @@ -0,0 +1,218 @@ +//! Rasterisation de texte Linux (PR #183) -- `cosmic-text` (rustybuzz + swash + +//! fontdb) au lieu de DirectWrite (Windows) / CoreText (macOS). +//! +//! Equivalent Linux de `text_windows.rs` / `text_macos.rs` : meme surface +//! publique (`TextSpec` + `cache_key`, `TextRasterizer::new()`, +//! `rasterize(&self, gpu, spec)`) pour que `compositor` (cfg-re-exporte) et +//! `text_anim` (partage) l'utilisent sans connaitre la plateforme. +//! +//! **Difference de format.** macOS/Windows bakent la couleur dans une texture +//! BGRA premultipliee (CoreText/Direct2D). Ici on produit un **atlas de +//! couverture R8** (alpha) que le shader WGSL (`layer.wgsl` mode 11) teinte par +//! `layer.color` -- meme resultat visuel, et le contrat d'iso-render porte sur +//! la GEOMETRIE (`frame_geometry::plan_frame`), pas sur la rasterisation texte +//! (dont l'ecart d'antialiasing est deja exclu des goldens cross-backend). + +use anyhow::{bail, Result}; +use std::cell::RefCell; + +use cosmic_text::{Attrs, Buffer, FontSystem, Metrics, Shaping, SwashCache}; + +use crate::d3d::Gpu; + +/// Tout ce dont le rendu d'un texte depend. Meme structure et meme `cache_key` +/// que `text_windows::TextSpec` / `text_macos::TextSpec` : la cle est partagee +/// entre plateformes, donc deux specs identiques produisent la meme texture. +#[derive(Clone, PartialEq)] +pub struct TextSpec { + pub content: String, + /// RGBA 0..1 (deja parse depuis la chaine CSS cote appelant). + pub color: [f32; 4], + /// RGBA 0..1 ; alpha 0 = pas de fond (le CSS `transparent`). + pub background: [f32; 4], + pub font_size_px: f32, + pub font_family: String, + pub bold: bool, + pub italic: bool, + pub underline: bool, + /// "left" | "center" | "right". + pub align: String, + /// Taille de la boite en px de sortie. + pub box_px: [u32; 2], +} + +impl TextSpec { + /// FNV-1a sur les memes octets, dans le meme ordre, que + /// `text_macos::TextSpec::cache_key` / `text_windows` -- la policy est + /// partagee (cache cross-plateforme coherent). + pub fn cache_key(&self) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + let mut mix = |bytes: &[u8]| { + for b in bytes { + h ^= *b as u64; + h = h.wrapping_mul(0x100_0000_01b3); + } + }; + mix(self.content.as_bytes()); + mix(self.font_family.as_bytes()); + mix(&self.font_size_px.to_bits().to_le_bytes()); + for c in self.color.iter().chain(self.background.iter()) { + mix(&c.to_bits().to_le_bytes()); + } + mix(&[self.bold as u8, self.italic as u8, self.underline as u8]); + mix(self.align.as_bytes()); + mix(&self.box_px[0].to_le_bytes()); + mix(&self.box_px[1].to_le_bytes()); + h + } +} + +/// Le resultat d'une rasterisation : la texture R8 de couverture + ses dims. +pub struct RasterizedGlyphs { + pub view: wgpu::TextureView, + pub width: u32, + pub height: u32, +} + +/// Le rasterizer Linux. `fontdb` lit `/usr/share/fonts` a la construction du +/// `FontSystem`. Etat (font_system, swash_cache) en `RefCell` pour que +/// `rasterize(&self, ...)` matche la signature `&self` des autres plateformes +/// (le compositor tient un `Option` et l'appelle sur `&self`). +pub struct TextRasterizer { + font_system: RefCell, + swash_cache: RefCell, +} + +impl TextRasterizer { + pub fn new() -> Result { + Ok(TextRasterizer { + font_system: RefCell::new(FontSystem::new()), + swash_cache: RefCell::new(SwashCache::new()), + }) + } + + /// Rasterise `spec` dans une texture R8Unorm (couverture alpha) et rend sa + /// view. `gpu` fournit le device/queue wgpu (passe au rasterize comme cote + /// macOS). Le cache par `cache_key()` est gere par le caller (compositor). + pub fn rasterize(&self, gpu: &Gpu, spec: &TextSpec) -> Result { + let (w, h) = (spec.box_px[0].max(1), spec.box_px[1].max(1)); + if spec.content.is_empty() { + bail!("text_linux::rasterize: texte vide"); + } + + let mut font_system = self.font_system.borrow_mut(); + let mut swash_cache = self.swash_cache.borrow_mut(); + + let font_size = spec.font_size_px.max(1.0); + let line_height = font_size * 1.4; // heuristique standard. + let metrics = Metrics::new(font_size, line_height); + let mut buffer = Buffer::new(&mut font_system, metrics); + buffer.set_size(Some(w as f32), Some(h as f32)); + + let mut attrs = Attrs::new(); + attrs = attrs.family(cosmic_text::Family::Name(&spec.font_family)); + if spec.bold { + attrs = attrs.weight(cosmic_text::Weight::BOLD); + } + if spec.italic { + attrs = attrs.style(cosmic_text::Style::Italic); + } + if spec.underline { + attrs = attrs.underline(cosmic_text::UnderlineStyle::Single); + } + buffer.set_text(&spec.content, &attrs, Shaping::Advanced, None); + buffer.shape_until_scroll(&mut font_system, false); + + // Atlas R8 : on n'ecrit que le canal alpha (couverture). Le tint par + // `spec.color` se fait cote shader (mode 11). + let mut atlas: Vec = vec![0u8; (w * h) as usize]; + for run in buffer.layout_runs() { + for glyph in run.glyphs.iter() { + let physical = glyph.physical((0.0, 0.0), 1.0); + let img = swash_cache.get_image(&mut font_system, physical.cache_key); + let glyph_x = physical.x + run.line_top as i32; + let glyph_y = physical.y; + let Some(img) = img else { continue }; + let placement = img.placement; + let (img_w, img_h) = (placement.width, placement.height); + if img_w == 0 || img_h == 0 { + continue; + } + let stride = match img.content { + cosmic_text::SwashContent::Mask => img_w as usize, + cosmic_text::SwashContent::Color => img_w as usize * 4, + _ => continue, + }; + let alpha_offset = if matches!(img.content, cosmic_text::SwashContent::Color) { + 3 + } else { + 0 + }; + let bpp = if alpha_offset == 0 { 1 } else { 4 }; + for row in 0..img_h as i32 { + let dest_y = glyph_y + placement.top + row; + if dest_y < 0 || dest_y >= h as i32 { + continue; + } + let dest_x = glyph_x + placement.left; + if dest_x >= w as i32 { + continue; + } + let copy_len = (img_w as i32).min(w as i32 - dest_x).max(0) as usize; + if copy_len == 0 { + continue; + } + let src_row = &img.data[(row as usize) * stride..(row as usize + 1) * stride]; + let atlas_row_start = (dest_y as usize) * w as usize; + for col in 0..copy_len { + let atlas_idx = atlas_row_start + (dest_x as usize + col); + let src_idx = col * bpp + alpha_offset; + if src_idx < src_row.len() { + atlas[atlas_idx] = src_row[src_idx]; + } + } + } + } + } + + let texture = gpu.device.create_texture(&wgpu::TextureDescriptor { + label: Some("text-atlas"), + size: wgpu::Extent3d { + width: w, + height: h, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::R8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + gpu.context.write_texture( + wgpu::TexelCopyTextureInfo { + texture: &texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + &atlas, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(w), + rows_per_image: Some(h), + }, + wgpu::Extent3d { + width: w, + height: h, + depth_or_array_layers: 1, + }, + ); + let view = texture.create_view(&wgpu::TextureViewDescriptor::default()); + Ok(RasterizedGlyphs { + view, + width: w, + height: h, + }) + } +} diff --git a/crates/compositor/src/vk_shaders/blur.wgsl b/crates/compositor/src/vk_shaders/blur.wgsl new file mode 100644 index 000000000..ea9d4a018 --- /dev/null +++ b/crates/compositor/src/vk_shaders/blur.wgsl @@ -0,0 +1,85 @@ +// Tranche verticale WP4 ÔÇö Kawase blur (mode 9+10 du HLSL) port├® en WGSL. +// +// Le Kawase blur est une approximation gaussienne en 6 passes : down 3x +// (RTÔåÆ┬¢ÔåÆ┬╝ÔåÆÔàø) puis up 3x (ÔàøÔåÆ┬╝ÔåÆ┬¢ÔåÆRT). Chaque passe est un 5-tap lin├®aire +// ├á offset 2.2 px (cf. HLSL `ps_kawase_down` / `ps_kawase_up`). Le r├®sultat +// est visuellement ├®quivalent ├á un flou gaussien ~30-50 px (selon la +// taille de la pyramide) ├á un co├╗t constant 6├ù5 = 30 taps ÔÇö vs 49 taps +// pour une passe gaussienne ├®quivalente. Cf. HLSL `Compositor::blur_bg`. +// +// Bindings : la passe de down lit d'une texture RGBA8 et ├®crit dans +// une texture RGBA8 plus petite ; la passe d'up fait l'inverse. Toutes +// les passes partagent le m├¬me bind group layout, seule la constante +// `texel_offset` (dans LayerCB `fx`) change entre les passes. + +struct Layer { + dst: vec4, + src: vec4, + quad_px: vec2, + radius_px: f32, + mode: f32, + color: vec4, + fx: vec4, // .x = texel offset (2.2 pour Kawase) + src_prev: vec4, + dst_prev: vec4, + mb: vec4, +} + +@group(0) @binding(0) var layer: Layer; +@group(0) @binding(1) var tex: texture_2d; +@group(0) @binding(2) var samp: sampler; + +struct VsOut { + @builtin(position) pos: vec4, + @location(0) uv: vec2, +} + +@vertex +fn vs_main(@builtin(vertex_index) vid: u32) -> VsOut { + // Fullscreen triangle ÔÇö un seul triangle couvre tout l'├®cran, plus + // efficace qu'un quad en termes de pixels shaders ├®mis. + let pos = array, 3>( + vec2(-1.0, -1.0), + vec2( 3.0, -1.0), + vec2(-1.0, 3.0), + ); + var o: VsOut; + o.pos = vec4(pos[vid], 0.0, 1.0); + o.uv = pos[vid] * 0.5 + vec2(0.5, 0.5); + // Note : on inverse Y parce que wgpu NDC y-up mais l'image source est + // y-down (cf. le Y-flip dans le VS du layer.wgsl principal). + o.uv.y = 1.0 - o.uv.y; + return o; +} + +// Kawase down : 5-tap lin├®aire ├á offset `texel_offset` en coords source. +// `texel_offset` est 2.2 typiquement (le spread mesur├® du filtre). +@fragment +fn fs_kawase_down(i: VsOut) -> @location(0) vec4 { + let o = layer.fx.x; + let c = textureSample(tex, samp, i.uv).rgb; + let s1 = textureSample(tex, samp, i.uv + vec2( o, o) / vec2(layer.quad_px.x, layer.quad_px.y)).rgb; + let s2 = textureSample(tex, samp, i.uv + vec2(-o, o) / vec2(layer.quad_px.x, layer.quad_px.y)).rgb; + let s3 = textureSample(tex, samp, i.uv + vec2( o, -o) / vec2(layer.quad_px.x, layer.quad_px.y)).rgb; + let s4 = textureSample(tex, samp, i.uv + vec2(-o, -o) / vec2(layer.quad_px.x, layer.quad_px.y)).rgb; + return vec4((c + s1 + s2 + s3 + s4) * 0.2, layer.color.a); +} + +// Kawase up : interpolation lin├®aire entre la texture de destination +// (`tex`) et l'├®chantillon ├á offset `texel_offset` dans la m├¬me texture. +// C'est l'algorithme Kawase ┬½ up ┬╗ original ÔÇö moins connu que le down +// mais c'est ce qui donne le look "soft glow" mesur├® sur le banc. +// +// On interpole entre la valeur au centre et les 4 voisins ├á offset `o`. +@fragment +fn fs_kawase_up(i: VsOut) -> @location(0) vec4 { + let o = layer.fx.x; + let c = textureSample(tex, samp, i.uv).rgb; + let s1 = textureSample(tex, samp, i.uv + vec2( o, o) / vec2(layer.quad_px.x, layer.quad_px.y)).rgb; + let s2 = textureSample(tex, samp, i.uv + vec2(-o, o) / vec2(layer.quad_px.x, layer.quad_px.y)).rgb; + let s3 = textureSample(tex, samp, i.uv + vec2( o, -o) / vec2(layer.quad_px.x, layer.quad_px.y)).rgb; + let s4 = textureSample(tex, samp, i.uv + vec2(-o, -o) / vec2(layer.quad_px.x, layer.quad_px.y)).rgb; + // Pond├®ration (1.0 centre, 0.5 chaque voisin) ÔÇö 1+4├ù0.5 = 3.0, /3 = 1/3 par + // ├®chantillon. Le rendu Kawase up est plus doux que le down. + return vec4((c + (s1 + s2 + s3 + s4) * 0.5) / 3.0, layer.color.a); +} diff --git a/crates/compositor/src/vk_shaders/layer.wgsl b/crates/compositor/src/vk_shaders/layer.wgsl new file mode 100644 index 000000000..15d279160 --- /dev/null +++ b/crates/compositor/src/vk_shaders/layer.wgsl @@ -0,0 +1,126 @@ +// Tranche verticale WP3 ÔÇö port 1:1 des modes 0 (vid├®o NV12) et 1 (couleur pleine) +// du `ps_main` HLSL (`crates/compositor/src/shaders.hlsl`). Le mode 2 (ombre +// port├®e) partage la m├¬me SDF et le m├¬me feather que les autres modes, donc on +// l'inclut aussi pour parit├®. +// +// Les constantes YUV (BT.709 limited) sont reprises ├á l'identique du HLSL : +// Yf = (Y * 255 ÔêÆ 16) / 219 +// Cb = (UV.x * 255 ÔêÆ 128) / 224 +// Cr = (UV.y * 255 ÔêÆ 128) / 224 +// R = Yf + 1.5748 ┬À Cr +// G = Yf ÔêÆ 0.1873 ┬À Cb ÔêÆ 0.4681 ┬À Cr +// B = Yf + 1.8556 ┬À Cb +// Mesur├® en S1 (cf. doc ┬º7 E1). Une d├®viation > 0/255 entre HLSL et WGSL ici +// indiquerait une diff├®rence de pr├®cision fp32 ; IEEE-754 round-to-nearest est +// identique sur les deux backends. +// +// Le rendu est en alpha PR├ëMULTIPLI├ë (cf. commentaire HLSL), convention qu'on +// retrouve dans tous les autres modes du compositeur (texte, curseur, ombre). + +struct Layer { + dst: vec4, // x,y,w,h sortie 0..1 (origine haut-gauche) + src: vec4, // u0,v0,u1,v1 source 0..1 + quad_px: vec2, // taille du quad en px de sortie (pour la SDF isotrope) + radius_px: f32, + mode: f32, // 0 = vid├®o NV12, 1 = couleur pleine, 2 = ombre + color: vec4, + fx: vec4, // mode 2 : spread ombre en px + src_prev: vec4, + dst_prev: vec4, + mb: vec4, +} + +@group(0) @binding(0) var layer: Layer; +@group(0) @binding(1) var texY: texture_2d; // R8Unorm, sample .r +@group(0) @binding(2) var texUV: texture_2d; // Rg8Unorm, sample .rg +@group(0) @binding(3) var samp: sampler; + +struct VsOut { + @builtin(position) pos: vec4, + @location(0) uv: vec2, // UV d'├®chantillonnage source + @location(1) local: vec2, // pixel local dans le quad (SDF) + @location(2) pout: vec2, // position 0..1 sortie +}; + +@vertex +fn vs_main(@builtin(vertex_index) vid: u32) -> VsOut { + // strip 4 vertices : (0,0)(1,0)(0,1)(1,1) + let c = vec2(f32(vid & 1u), f32((vid >> 1u) & 1u)); + let p = layer.dst.xy + c * layer.dst.zw; + let ndc = vec2(p.x * 2.0 - 1.0, 1.0 - p.y * 2.0); + var o: VsOut; + o.pos = vec4(ndc, 0.0, 1.0); + o.uv = layer.src.xy + c * (layer.src.zw - layer.src.xy); + o.local = c * layer.quad_px; + o.pout = p; + return o; +} + +fn yuv709_limited(y: f32, cbcr: vec2) -> vec3 { + let Yf = (y * 255.0 - 16.0) / 219.0; + let Cb = (cbcr.x * 255.0 - 128.0) / 224.0; + let Cr = (cbcr.y * 255.0 - 128.0) / 224.0; + return clamp(vec3( + Yf + 1.5748 * Cr, + Yf - 0.1873 * Cb - 0.4681 * Cr, + Yf + 1.8556 * Cb, + ), vec3(0.0), vec3(1.0)); +} + +fn sample_yuv(uv: vec2) -> vec3 { + let y = textureSample(texY, samp, uv).r; + let cbcr = textureSample(texUV, samp, uv).rg; + return yuv709_limited(y, cbcr); +} + +// SDF rectangle ├á coins arrondis (< 0 dedans). Identique au HLSL. +fn sd_round_rect(p: vec2, halfsz: vec2, r: f32) -> f32 { + let q = abs(p) - halfsz + vec2(r); + return length(max(q, vec2(0.0))) + min(max(q.x, q.y), 0.0) - r; +} + +@fragment +fn fs_main(i: VsOut) -> @location(0) vec4 { + var rgb: vec3; + var alpha: f32; + + if layer.mode < 0.5 { + // Mode 0 ÔÇö vid├®o NV12. Le motion blur (taps) et le flou de mouvement par + // v├®locit├® sont d├®lib├®r├®ment omis de la tranche verticale (ils vivent en + // WP4, apr├¿s la validation du chemin iso). Couleur ├®chantillonn├®e une fois + // ├á l'UV interpol├®e. + rgb = sample_yuv(i.uv); + } else if layer.mode < 1.5 { + // Mode 1 ÔÇö couleur pleine. + rgb = layer.color.rgb; + } else if layer.mode > 10.5 && layer.mode < 11.5 { + // Mode 11 : texte. texY est l'atlas R8 (couverture alpha au canal .r, + // produit par text_cosmic::TextRasterizer), teinte par layer.color. + // Sortie en alpha premultiplie, comme les autres modes. + let cov = textureSample(texY, samp, i.uv).r; + let a = layer.color.a * cov; + return vec4(layer.color.rgb * a, a); + } else { + // Mode 2 ÔÇö ombre port├®e (SDF d'un quad arrondi ├®largi de `fx.x`). + let spread = layer.fx.x; + let halfsz = layer.quad_px * 0.5 - vec2(spread); + let p = i.local - layer.quad_px * 0.5; + let d = sd_round_rect(p, halfsz, layer.radius_px); + let a = layer.color.a * (1.0 - smoothstep(0.0, spread, d)); + return vec4(layer.color.rgb * a, a); + } + + alpha = layer.color.a; + + if layer.radius_px > 0.0 { + // Feather ~1.5 px sur le bord du quad ÔÇö parit├® exacte avec le HLSL + // (`smoothstep(0.0, 1.5, d)`). Le shader HLSL inclut `quad_px` en px de + // SORTIE ; on reproduit la m├¬me chose ici. + let halfsz = layer.quad_px * 0.5; + let p = i.local - layer.quad_px * 0.5; + let d = sd_round_rect(p, halfsz, layer.radius_px); + alpha *= 1.0 - smoothstep(0.0, 1.5, d); + } + + return vec4(rgb * alpha, alpha); // alpha pr├®multipli├® +} diff --git a/crates/compositor/tests/compose_linux.rs b/crates/compositor/tests/compose_linux.rs new file mode 100644 index 000000000..652cb5797 --- /dev/null +++ b/crates/compositor/tests/compose_linux.rs @@ -0,0 +1,54 @@ +//! Verifie que le port Linux reconciliie sur v1.8.0 REND une frame : +//! `d3d::Gpu` -> `compositor::Compositor` -> `pipeline::Decoder` (les modules +//! Linux, via les alias cfg) -> `compose_frame` (geometrie partagee +//! `plan_frame`) -> `readback_direct`. Bypass le render-thread de `live.rs` +//! pour isoler la chaine de rendu elle-meme. +//! +//! Opt-in (rend sur GPU) : `OPENSCREEN_LINUX_COMPOSE=1` + la fixture +//! `crates/fixture/screen.mp4`. Sinon skip (le teardown Vulkan/Mesa segfault a +//! l'exit apres le rendu -- verifier via la sortie, pas l'exit code). + +use std::path::Path; + +use openscreen_compositor::compositor::Compositor; +use openscreen_compositor::config::Cfg; +use openscreen_compositor::d3d::Gpu; +use openscreen_compositor::pipeline::Decoder; + +const FIXTURE: &str = "../fixture/screen.mp4"; +const W: u32 = 960; +const H: u32 = 540; + +#[test] +fn compose_linux_rend_une_frame() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("compose_linux: opt-in (OPENSCREEN_LINUX_COMPOSE=1 + fixture). Skip."); + return; + } + + let gpu = Gpu::create(false).expect("Gpu::create"); + let comp = Compositor::new_sized(&gpu, W, H).expect("Compositor::new_sized"); + let mut dec = Decoder::open(FIXTURE, &gpu).expect("Decoder::open"); + + let (w, h, rgba) = unsafe { + let sf = dec.seek_to(1.0).expect("Decoder::seek_to"); + let cfg = Cfg::c8(); + // webcam = screen (mon compose coeur ne dessine que l'ecran). + comp.compose_frame(sf, sf, 0.0, &cfg).expect("compose_frame"); + comp.readback_direct().expect("readback_direct") + }; + + let n = (rgba.len() / 4) as f32; + let mut sum = 0u64; + for px in rgba.chunks_exact(4) { + sum += px[0] as u64; + } + let mean_r = sum as f32 / n; + println!("compose_linux : {w}x{h} bytes={} mean_R={:.1}", rgba.len(), mean_r); + + assert_eq!(rgba.len(), (W * H * 4) as usize); + assert!( + mean_r > 5.0 && mean_r < 250.0, + "mean R={mean_r} hors plage plausible (5..250) — frame vide ?" + ); +} diff --git a/crates/compositor/wrapper_linux.h b/crates/compositor/wrapper_linux.h new file mode 100644 index 000000000..e01a31a97 --- /dev/null +++ b/crates/compositor/wrapper_linux.h @@ -0,0 +1,14 @@ +/* Linux ffmpeg wrapper for bindgen (PR #183). + * + * Software/VAAPI decode+encode only: no D3D11VA (Windows) nor VideoToolbox + * (macOS) hwcontext headers, which pull platform-specific system headers + * (d3d11.h / CoreVideo) that don't exist on Linux. The generic hwcontext.h is + * kept for AVHWDeviceContext should the VAAPI path need it later. */ +#include +#include +#include +#include +#include +#include +#include +#include diff --git a/crates/poc-d3d/Cargo.toml b/crates/poc-d3d/Cargo.toml index 34ba4e7ce..4c42f5820 100644 --- a/crates/poc-d3d/Cargo.toml +++ b/crates/poc-d3d/Cargo.toml @@ -15,4 +15,7 @@ path = "src/main.rs" [dependencies] openscreen-compositor.workspace = true anyhow.workspace = true + +# GUI Win32 : Windows-only (le POC ne se compile qu'en no-op ailleurs). +[target.'cfg(windows)'.dependencies] windows.workspace = true diff --git a/crates/poc-d3d/src/main.rs b/crates/poc-d3d/src/main.rs index 6a8a43f16..9d5365faf 100644 --- a/crates/poc-d3d/src/main.rs +++ b/crates/poc-d3d/src/main.rs @@ -1,9 +1,19 @@ //! Binaire du POC de mesure : GUI Win32 de preview/export, harnais de bench fps, mode live. //! Tout le rendu vient d'`openscreen-compositor` — ce crate ne fait que le piloter et le mesurer. +// GUI Win32 + bench encodeurs Windows : Windows-only. Sur les autres plateformes +// le POC n'a rien a piloter (le rendu/bench passe par les tests du crate compositor). +#[cfg(windows)] mod app; +#[cfg(windows)] mod bench; +#[cfg(windows)] fn main() -> anyhow::Result<()> { bench::run() } + +#[cfg(not(windows))] +fn main() { + eprintln!("poc-d3d : bench GUI Win32 (Windows-only), rien a faire sur cette plateforme."); +} From 611d5c29df57856fee7bee730b5659a6d0158d83 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Thu, 30 Jul 2026 16:12:17 +0200 Subject: [PATCH 02/46] feat(compositor): draw text annotations in compositor_linux (WGSL mode 11) compose_frame now draws the scene's text annotations over the screen layer, mirroring the "text" branch of compositor_macos: each annotation is placed relative to the screen rect (g.s_dst), its TextSpec is rasterised by text_linux (cosmic-text -> R8 coverage atlas), and drawn with layer.wgsl mode 11 tinted by the annotation colour. Bind groups are built before the render pass (one uniform buffer per draw). Differs from the Metal path only in the texture format: macOS bakes colour into a BGRA texture; here the atlas is R8 coverage and the shader tints it (cb.color) -- same visual result, and the iso-render contract is on the shared plan_frame geometry, not on text rasterisation. Image/figure/blur annotations and the text_anim reveal/opacity animation remain incremental. --- crates/compositor/src/compositor_linux.rs | 74 +++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/crates/compositor/src/compositor_linux.rs b/crates/compositor/src/compositor_linux.rs index f42b83990..c74cc55d9 100644 --- a/crates/compositor/src/compositor_linux.rs +++ b/crates/compositor/src/compositor_linux.rs @@ -401,6 +401,74 @@ impl Compositor { let (_screen_uniform, screen_bind) = self.make_bind(&screen_layer, Some((&sy, &suv)), &dummy); + // Annotations TEXTE (mode 11) -- placees relativement au rect ecran + // `g.s_dst` (les coords x/y/w/h de l'annotation sont des fractions de ce + // rect, cf. `scene.rs`). Mirroir de la branche "text" de + // `compositor_macos`, tint cote shader (atlas R8) au lieu de couleur + // bakee. (image/figure/blur + animation `text_anim` : incremental.) + struct AnnDraw { + _buf: wgpu::Buffer, + _glyphs: crate::text::RasterizedGlyphs, + bind: wgpu::BindGroup, + } + let mut ann_draws: Vec = Vec::new(); + if let (Some(scene), Some(raster)) = (scene_ref.as_ref(), self.text_raster.as_ref()) { + for a in &scene.annotations { + if a.kind != "text" { + continue; + } + let Some(text) = a.text.as_ref() else { continue }; + if text.content.trim().is_empty() { + continue; + } + let dst = [ + g.s_dst[0] + a.x * g.s_dst[2], + g.s_dst[1] + a.y * g.s_dst[3], + a.w * g.s_dst[2], + a.h * g.s_dst[3], + ]; + let quad_px = [dst[2] * rw, dst[3] * rh]; + let color = parse_hex(&text.color).unwrap_or([1.0, 1.0, 1.0, 1.0]); + let spec = crate::text::TextSpec { + content: text.content.clone(), + color, + background: parse_hex(&text.background_color).unwrap_or([0.0, 0.0, 0.0, 0.0]), + font_size_px: text.font_size_rel * (g.s_dst[3] * rh), + font_family: text.font_family.clone(), + bold: text.font_weight == "bold", + italic: text.font_style == "italic", + underline: text.text_decoration == "underline", + align: text.text_align.clone(), + box_px: [ + quad_px[0].round().max(1.0) as u32, + quad_px[1].round().max(1.0) as u32, + ], + }; + let glyphs = match raster.rasterize(&self.gpu, &spec) { + Ok(gl) => gl, + Err(e) => { + eprintln!("[annotation texte] {}: {e:#}", a.id); + continue; + } + }; + let cb = LayerCB { + dst, + src: [0.0, 0.0, 1.0, 1.0], + quad_px, + mode: 11.0, + color, + ..Default::default() + }; + // Atlas R8 au binding 1 (texY) que le mode 11 echantillonne. + let (buf, bind) = self.make_bind(&cb, Some((&glyphs.view, &glyphs.view)), &dummy); + ann_draws.push(AnnDraw { + _buf: buf, + _glyphs: glyphs, + bind, + }); + } + } + let mut encoder = self.gpu.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("compose"), }); @@ -429,6 +497,12 @@ impl Compositor { rpass.set_pipeline(&self.pipeline); rpass.set_bind_group(0, &screen_bind, &[]); rpass.draw(0..4, 0..1); + // Annotations texte par-dessus (ordre de la liste = z-index, deja + // trie cote app). + for a in &ann_draws { + rpass.set_bind_group(0, &a.bind, &[]); + rpass.draw(0..4, 0..1); + } } self.gpu.context.submit(std::iter::once(encoder.finish())); Ok(()) From 16f2ea16401fad41791114d5f57c82f4bb27ffc2 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Thu, 30 Jul 2026 17:17:57 +0200 Subject: [PATCH 03/46] feat(compositor): draw the webcam PiP in compositor_linux compose_frame Webcam layer (mode 0) placed at plan_frame g.w_dst/g.w_radius, gated by g.shape_fade>0, drawn over the screen. cover-crop UV, mirror, shadow and circle shape remain incremental; compose_linux test now writes a PPM for visual inspection. --- crates/compositor/src/compositor_linux.rs | 27 +++++++++++++++++++++++ crates/compositor/tests/compose_linux.rs | 16 ++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/crates/compositor/src/compositor_linux.rs b/crates/compositor/src/compositor_linux.rs index c74cc55d9..903ff2cd0 100644 --- a/crates/compositor/src/compositor_linux.rs +++ b/crates/compositor/src/compositor_linux.rs @@ -401,6 +401,28 @@ impl Compositor { let (_screen_uniform, screen_bind) = self.make_bind(&screen_layer, Some((&sy, &suv)), &dummy); + // Webcam PiP (mode 0) -- placee par plan_frame (`g.w_dst`, coins + // `g.w_radius`), gardee par `g.shape_fade > 0` (webcam visible). + // cover-crop UV + miroir + ombre + forme cercle = incremental (src plein + // cadre ici). `webcam_planes` garde les vues en vie pendant le pass. + let webcam_planes = if g.shape_fade > 0.0 && !webcam.is_null() { + self.nv12_srvs(webcam).ok() + } else { + None + }; + let webcam_draw = webcam_planes.as_ref().map(|(wy, wuv)| { + let cb = LayerCB { + dst: g.w_dst, + src: [0.0, 0.0, 1.0, 1.0], + quad_px: g.w_px, + radius_px: g.w_radius, + mode: 0.0, + color: [0.0, 0.0, 0.0, 1.0], + ..Default::default() + }; + self.make_bind(&cb, Some((wy, wuv)), &dummy) + }); + // Annotations TEXTE (mode 11) -- placees relativement au rect ecran // `g.s_dst` (les coords x/y/w/h de l'annotation sont des fractions de ce // rect, cf. `scene.rs`). Mirroir de la branche "text" de @@ -497,6 +519,11 @@ impl Compositor { rpass.set_pipeline(&self.pipeline); rpass.set_bind_group(0, &screen_bind, &[]); rpass.draw(0..4, 0..1); + // Webcam PiP par-dessus l'ecran. + if let Some((_buf, bind)) = &webcam_draw { + rpass.set_bind_group(0, bind, &[]); + rpass.draw(0..4, 0..1); + } // Annotations texte par-dessus (ordre de la liste = z-index, deja // trie cote app). for a in &ann_draws { diff --git a/crates/compositor/tests/compose_linux.rs b/crates/compositor/tests/compose_linux.rs index 652cb5797..1fae85a9d 100644 --- a/crates/compositor/tests/compose_linux.rs +++ b/crates/compositor/tests/compose_linux.rs @@ -46,6 +46,22 @@ fn compose_linux_rend_une_frame() { let mean_r = sum as f32 / n; println!("compose_linux : {w}x{h} bytes={} mean_R={:.1}", rgba.len(), mean_r); + // PPM P6 pour inspection visuelle. + let out = std::env::var("OPENSCREEN_VK_OUT").unwrap_or_else(|_| "target".into()); + let _ = std::fs::create_dir_all(&out); + let ppm = format!("{out}/compose_linux.ppm"); + { + use std::io::Write; + let mut f = std::fs::File::create(&ppm).expect("create ppm"); + write!(f, "P6\n{w} {h}\n255\n").unwrap(); + let mut rgb = vec![0u8; (w * h * 3) as usize]; + for (d, s) in rgb.chunks_exact_mut(3).zip(rgba.chunks_exact(4)) { + d.copy_from_slice(&s[0..3]); + } + f.write_all(&rgb).unwrap(); + } + println!("wrote {ppm}"); + assert_eq!(rgba.len(), (W * H * 4) as usize); assert!( mean_r > 5.0 && mean_r < 250.0, From ebd2ca292a05d780c11406459923f3b253cb4104 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Thu, 30 Jul 2026 18:09:19 +0200 Subject: [PATCH 04/46] feat(compositor): background blur (dual-Kawase) in compositor_linux Restructure compose_frame into bg-pass (clear + gradient mode 5) -> blur_bg (3 down / 3 up Kawase pyramid, blur.wgsl) -> fg-pass (screen + webcam + annotations, LoadOp::Load). Gated by cfg.bg_blur. Adds the WGSL mode 5 linear gradient to layer.wgsl. 1:1 port of the Windows/macOS blur_bg. compose_linux test now renders a gradient background + padding so the blurred background is visible around the inset screen. --- crates/compositor/src/compositor_linux.rs | 266 ++++++++++++++++++-- crates/compositor/src/vk_shaders/layer.wgsl | 5 + crates/compositor/tests/compose_linux.rs | 9 +- 3 files changed, 263 insertions(+), 17 deletions(-) diff --git a/crates/compositor/src/compositor_linux.rs b/crates/compositor/src/compositor_linux.rs index 903ff2cd0..0e3d3a7ef 100644 --- a/crates/compositor/src/compositor_linux.rs +++ b/crates/compositor/src/compositor_linux.rs @@ -37,6 +37,7 @@ use crate::frame_geometry::{parse_hex, plan_frame, FrameGeometryInput}; use crate::scene::{Scene, SceneBackground}; const LAYER_WGSL: &str = include_str!("vk_shaders/layer.wgsl"); +const BLUR_WGSL: &str = include_str!("vk_shaders/blur.wgsl"); /// `&LayerCB` -> `&[u8; 128]`. `LayerCB` est `#[repr(C, align(16))]`, son layout /// EST le buffer uniforme WGSL (16 vec4 + 1 vec2 + 2 f32 = 128 octets). @@ -55,6 +56,16 @@ pub struct Compositor { bind_group_layout: wgpu::BindGroupLayout, sampler: wgpu::Sampler, + // Chaine de blur Kawase du fond (`blur.wgsl`) : layout dedie (uniform + 1 + // tex + sampler), 2 pipelines (down/up), 3 textures de pyramide (1/2, 1/4, + // 1/8 de la sortie). Les `TextureView` gardent leurs textures en vie. + blur_bgl: wgpu::BindGroupLayout, + blur_down: wgpu::RenderPipeline, + blur_up: wgpu::RenderPipeline, + blur_half: wgpu::TextureView, + blur_qtr: wgpu::TextureView, + blur_oct: wgpu::TextureView, + // Render target offscreen + buffer de readback (recree au resize). rt: wgpu::Texture, rt_view: wgpu::TextureView, @@ -171,6 +182,103 @@ impl Compositor { cache: None, }); + // --- Chaine de blur Kawase du fond (`blur.wgsl`) --- + let blur_module = gpu.device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("blur.wgsl"), + source: wgpu::ShaderSource::Wgsl(BLUR_WGSL.into()), + }); + // Layout blur : 0 = uniform, 1 = texture, 2 = sampler (blur.wgsl). + let blur_bgl = gpu.device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("blur"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::VERTEX | wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: wgpu::BufferSize::new(128), + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 2, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + ], + }); + let blur_pl = gpu.device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("blur"), + bind_group_layouts: &[&blur_bgl], + push_constant_ranges: &[], + }); + let mk_blur = |entry: &str| { + gpu.device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some(entry), + layout: Some(&blur_pl), + vertex: wgpu::VertexState { + module: &blur_module, + entry_point: Some("vs_main"), + compilation_options: wgpu::PipelineCompilationOptions::default(), + buffers: &[], + }, + fragment: Some(wgpu::FragmentState { + module: &blur_module, + entry_point: Some(entry), + compilation_options: wgpu::PipelineCompilationOptions::default(), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Rgba8Unorm, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + }), + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleList, + ..Default::default() + }, + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview: None, + cache: None, + }) + }; + let blur_down = mk_blur("fs_kawase_down"); + let blur_up = mk_blur("fs_kawase_up"); + let mk_pyr = |dw: u32, dh: u32, label: &str| { + gpu.device + .create_texture(&wgpu::TextureDescriptor { + label: Some(label), + size: wgpu::Extent3d { + width: dw.max(1), + height: dh.max(1), + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT + | wgpu::TextureUsages::TEXTURE_BINDING, + view_formats: &[], + }) + .create_view(&wgpu::TextureViewDescriptor::default()) + }; + let blur_half = mk_pyr(w / 2, h / 2, "blur-half"); + let blur_qtr = mk_pyr(w / 4, h / 4, "blur-qtr"); + let blur_oct = mk_pyr(w / 8, h / 8, "blur-oct"); + let (rt, rt_view, readback_buf, readback_bpr) = Self::make_targets(&gpu, w, h); Ok(Compositor { @@ -180,6 +288,12 @@ impl Compositor { pipeline, bind_group_layout, sampler, + blur_bgl, + blur_down, + blur_up, + blur_half, + blur_qtr, + blur_oct, rt, rt_view, readback_buf, @@ -210,7 +324,9 @@ impl Compositor { sample_count: 1, dimension: wgpu::TextureDimension::D2, format: wgpu::TextureFormat::Rgba8Unorm, - usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT + | wgpu::TextureUsages::COPY_SRC + | wgpu::TextureUsages::TEXTURE_BINDING, view_formats: &[], }); let rt_view = rt.create_view(&wgpu::TextureViewDescriptor::default()); @@ -224,6 +340,80 @@ impl Compositor { (rt, rt_view, readback_buf, bpr) } + /// Une passe Kawase : lit `src`, ecrit `dst` (fullscreen triangle, 3 + /// vertices). `src_px` = dimensions de la source (pour le pas de texel). + fn blur_pass( + &self, + encoder: &mut wgpu::CommandEncoder, + pipeline: &wgpu::RenderPipeline, + src: &wgpu::TextureView, + dst: &wgpu::TextureView, + src_px: [f32; 2], + ) { + let cb = LayerCB { + quad_px: src_px, + mode: -1.0, + color: [1.0, 1.0, 1.0, 1.0], + fx: [2.0, 0.0, 0.0, 0.0], // texel offset Kawase + ..Default::default() + }; + let uniform = self.gpu.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("blur-uniform"), + contents: layer_bytes(&cb), + usage: wgpu::BufferUsages::UNIFORM, + }); + let bind = self.gpu.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("blur"), + layout: &self.blur_bgl, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: uniform.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::TextureView(src), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: wgpu::BindingResource::Sampler(&self.sampler), + }, + ], + }); + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("blur-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: dst, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + rpass.set_pipeline(pipeline); + rpass.set_bind_group(0, &bind, &[]); + rpass.draw(0..3, 0..1); + } + + /// Floute le RT (le fond deja dessine) : dual-Kawase 3 down (RT -> 1/2 -> + /// 1/4 -> 1/8) + 3 up (1/8 -> 1/4 -> 1/2 -> RT). ~gaussien a cout constant. + fn blur_bg(&self, encoder: &mut wgpu::CommandEncoder) { + let (rw, rh) = (self.render_w as f32, self.render_h as f32); + let (hw, hh) = (rw * 0.5, rh * 0.5); + let (qw, qh) = (rw * 0.25, rh * 0.25); + let (ow, oh) = (rw * 0.125, rh * 0.125); + self.blur_pass(encoder, &self.blur_down, &self.rt_view, &self.blur_half, [rw, rh]); + self.blur_pass(encoder, &self.blur_down, &self.blur_half, &self.blur_qtr, [hw, hh]); + self.blur_pass(encoder, &self.blur_down, &self.blur_qtr, &self.blur_oct, [qw, qh]); + self.blur_pass(encoder, &self.blur_up, &self.blur_oct, &self.blur_qtr, [ow, oh]); + self.blur_pass(encoder, &self.blur_up, &self.blur_qtr, &self.blur_half, [qw, qh]); + self.blur_pass(encoder, &self.blur_up, &self.blur_half, &self.rt_view, [hw, hh]); + } + /// Dimensions paires (NV12 4:2:0), min 2x2. Symetrie avec les autres backends. pub fn normalize_render_size(w: u32, h: u32) -> (u32, u32) { ((w.max(2) + 1) & !1, (h.max(2) + 1) & !1) @@ -377,11 +567,29 @@ impl Compositor { }); let _ = (wtw, wth); // webcam PiP : calque a venir. - // Couleur de fond : scene Color -> sa couleur, sinon live_params.bg_color. - // (Gradient/Image : calques a venir ; repli couleur ici.) - let bg = match scene_ref.as_ref().map(|s| s.background.clone()) { - Some(SceneBackground::Color { color }) => parse_hex(&color).unwrap_or(lp.bg_color), - _ => lp.bg_color, + // Fond : Color -> clear a la couleur ; Gradient -> clear transparent + + // draw mode 5 ; Image -> repli couleur (calque image a venir). Le blur + // (si cfg.bg_blur) floute ensuite ce fond, avant l'ecran. + let (bg_clear, gradient_cb) = match scene_ref.as_ref().map(|s| s.background.clone()) { + Some(SceneBackground::Color { color }) => { + (parse_hex(&color).unwrap_or(lp.bg_color), None) + } + Some(SceneBackground::Gradient { angle_deg, stops }) => { + let c0 = stops.first().and_then(|s| parse_hex(s)).unwrap_or(lp.bg_color); + let c1 = stops.last().and_then(|s| parse_hex(s)).unwrap_or(c0); + let a = angle_deg.to_radians(); + let cb = LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + src: [c1[0], c1[1], c1[2], c1[3]], + quad_px: [rw, rh], + mode: 5.0, + color: c0, + fx: [a.sin(), -a.cos(), 0.0, 0.0], + ..Default::default() + }; + ([0.0, 0.0, 0.0, 1.0], Some(cb)) + } + _ => (lp.bg_color, None), }; // Calque ecran (mode 0 : NV12 -> RGB), place par plan_frame (cover-fit + @@ -401,6 +609,9 @@ impl Compositor { let (_screen_uniform, screen_bind) = self.make_bind(&screen_layer, Some((&sy, &suv)), &dummy); + // Fond gradient (mode 5), dessine dans la passe de fond si present. + let gradient_bind = gradient_cb.as_ref().map(|cb| self.make_bind(cb, None, &dummy)); + // Webcam PiP (mode 0) -- placee par plan_frame (`g.w_dst`, coins // `g.w_radius`), gardee par `g.shape_fade > 0` (webcam visible). // cover-crop UV + miroir + ombre + forme cercle = incremental (src plein @@ -494,20 +705,19 @@ impl Compositor { let mut encoder = self.gpu.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("compose"), }); + // Passe 1 : fond (clear a `bg_clear` + gradient mode 5 eventuel). { let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { - label: Some("compose-pass"), + label: Some("bg-pass"), color_attachments: &[Some(wgpu::RenderPassColorAttachment { view: &self.rt_view, resolve_target: None, ops: wgpu::Operations { - // Le fond uni EST la couleur de clear (gradient/image - // ajouteront un draw ici). load: wgpu::LoadOp::Clear(wgpu::Color { - r: bg[0] as f64, - g: bg[1] as f64, - b: bg[2] as f64, - a: bg[3] as f64, + r: bg_clear[0] as f64, + g: bg_clear[1] as f64, + b: bg_clear[2] as f64, + a: bg_clear[3] as f64, }), store: wgpu::StoreOp::Store, }, @@ -516,16 +726,40 @@ impl Compositor { timestamp_writes: None, occlusion_query_set: None, }); + if let Some((_buf, bind)) = &gradient_bind { + rpass.set_pipeline(&self.pipeline); + rpass.set_bind_group(0, bind, &[]); + rpass.draw(0..4, 0..1); + } + } + // Blur du fond (avant l'ecran), si active par la scene/l'inspector. + if cfg.bg_blur { + self.blur_bg(&mut encoder); + } + // Passe 2 : avant-plan (ecran + webcam + annotations), compose par-dessus + // le fond (eventuellement floute) avec `LoadOp::Load`. + { + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("fg-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.rt_view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Load, + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); rpass.set_pipeline(&self.pipeline); rpass.set_bind_group(0, &screen_bind, &[]); rpass.draw(0..4, 0..1); - // Webcam PiP par-dessus l'ecran. if let Some((_buf, bind)) = &webcam_draw { rpass.set_bind_group(0, bind, &[]); rpass.draw(0..4, 0..1); } - // Annotations texte par-dessus (ordre de la liste = z-index, deja - // trie cote app). for a in &ann_draws { rpass.set_bind_group(0, &a.bind, &[]); rpass.draw(0..4, 0..1); diff --git a/crates/compositor/src/vk_shaders/layer.wgsl b/crates/compositor/src/vk_shaders/layer.wgsl index 15d279160..dc5f297f2 100644 --- a/crates/compositor/src/vk_shaders/layer.wgsl +++ b/crates/compositor/src/vk_shaders/layer.wgsl @@ -93,6 +93,11 @@ fn fs_main(i: VsOut) -> @location(0) vec4 { } else if layer.mode < 1.5 { // Mode 1 ÔÇö couleur pleine. rgb = layer.color.rgb; + } else if layer.mode > 4.5 && layer.mode < 5.5 { + // Mode 5 -- gradient lineaire : color (c0) -> src.rgb (c1) le long de + // la direction fx.xy (sin, -cos de l'angle). Parite avec le HLSL/MSL. + let t = clamp(dot(i.pout - vec2(0.5), layer.fx.xy) + 0.5, 0.0, 1.0); + rgb = mix(layer.color.rgb, layer.src.rgb, t); } else if layer.mode > 10.5 && layer.mode < 11.5 { // Mode 11 : texte. texY est l'atlas R8 (couverture alpha au canal .r, // produit par text_cosmic::TextRasterizer), teinte par layer.color. diff --git a/crates/compositor/tests/compose_linux.rs b/crates/compositor/tests/compose_linux.rs index 1fae85a9d..b733144bf 100644 --- a/crates/compositor/tests/compose_linux.rs +++ b/crates/compositor/tests/compose_linux.rs @@ -14,6 +14,7 @@ use openscreen_compositor::compositor::Compositor; use openscreen_compositor::config::Cfg; use openscreen_compositor::d3d::Gpu; use openscreen_compositor::pipeline::Decoder; +use openscreen_compositor::scene::Scene; const FIXTURE: &str = "../fixture/screen.mp4"; const W: u32 = 960; @@ -30,9 +31,15 @@ fn compose_linux_rend_une_frame() { let comp = Compositor::new_sized(&gpu, W, H).expect("Compositor::new_sized"); let mut dec = Decoder::open(FIXTURE, &gpu).expect("Decoder::open"); + // Scene : fond gradient + padding (l'ecran est inset -> le fond floute se + // voit tout autour) pour valider visuellement le blur du background. + let scene_json = r##"{"clips":[],"layout":{"preset":"no-webcam","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false},"effects":{"padding":0.18,"blur":true,"shadow":0,"roundnessFrac":0.05,"motionBlur":0},"background":{"kind":"gradient","angleDeg":45,"stops":["#ff3b6b","#3b6bff"]},"zoomRegions":[],"annotations":[],"cursor":{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"},"cropByClip":[],"output":{"width":1920,"height":1080,"fps":30}}"##; + comp.set_scene(Some(Scene::from_json(scene_json).expect("scene json"))); + let (w, h, rgba) = unsafe { let sf = dec.seek_to(1.0).expect("Decoder::seek_to"); - let cfg = Cfg::c8(); + let mut cfg = Cfg::c8(); + cfg.bg_blur = true; // webcam = screen (mon compose coeur ne dessine que l'ecran). comp.compose_frame(sf, sf, 0.0, &cfg).expect("compose_frame"); comp.readback_direct().expect("readback_direct") From 8639d00c1cd51ed6922a5e04f9ae04ecaddc25e3 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Thu, 30 Jul 2026 18:26:04 +0200 Subject: [PATCH 05/46] feat(compositor): themed cursor sprite (mode 7) in compositor_linux Draw the cursor as a themed RGBA sprite on top of the composite: plan_cursor -> upright placement -> load_image_texture (PNG/data-URI via the image crate, cached in img_cache) -> LayerCB mode 7. Adds WGSL mode 7 (sprite sampled on texY, Clip-to-canvas via fx). 1:1 port of the macOS draw_cur_themed upright path; tilted (mode 13) and motion-blur (taps>1) are follow-up increments. compose_linux test now also renders a green data-URI sprite at screen center and asserts the green pixels appear. --- crates/compositor/src/compositor_linux.rs | 137 +++++++++++++++++++- crates/compositor/src/vk_shaders/layer.wgsl | 12 ++ crates/compositor/tests/compose_linux.rs | 66 ++++++++++ 3 files changed, 214 insertions(+), 1 deletion(-) diff --git a/crates/compositor/src/compositor_linux.rs b/crates/compositor/src/compositor_linux.rs index 0e3d3a7ef..ef7052259 100644 --- a/crates/compositor/src/compositor_linux.rs +++ b/crates/compositor/src/compositor_linux.rs @@ -33,7 +33,10 @@ use crate::ffi::AVFrame; pub use crate::frame_geometry::{ live_params_from_scene, webcam_shape_code, FIXTURE_FRAMES, LayerCB, LiveParams, OUT_H, OUT_W, }; -use crate::frame_geometry::{parse_hex, plan_frame, FrameGeometryInput}; +use crate::frame_geometry::{ + cursor_sprite_dst, parse_hex, plan_cursor, plan_frame, CursorPlacement, CursorPlanInput, + FrameGeometryInput, +}; use crate::scene::{Scene, SceneBackground}; const LAYER_WGSL: &str = include_str!("vk_shaders/layer.wgsl"); @@ -84,6 +87,10 @@ pub struct Compositor { /// echoue -- le rendu continue sans texte plutot que de tout casser. #[allow(dead_code)] text_raster: Option, + + /// Cache des sprites curseur (PNG RGBA -> texture wgpu), par chemin. Meme + /// role que `img_cache` cote macOS : un sprite chargé une fois par session. + img_cache: RefCell>, } impl Compositor { @@ -304,6 +311,7 @@ impl Compositor { cursor_time: RefCell::new(None), timeline_time: RefCell::new(None), text_raster: crate::text::TextRasterizer::new().ok(), + img_cache: RefCell::new(std::collections::HashMap::new()), }) } @@ -522,6 +530,49 @@ impl Compositor { (uniform, bind) } + /// Charge un PNG/JPEG (chemin fichier ou data URI) en texture RGBA8. Port + /// wgpu du `load_image_texture` macOS, memes chemins (`decode_data_uri` + /// partage, crate `image`). Sert aux sprites de curseur (mode 7). + fn load_image_texture(&self, path: &str) -> Result<(wgpu::Texture, u32, u32)> { + let img = if let Some(bytes) = crate::frame_geometry::decode_data_uri(path) { + image::load_from_memory(&bytes) + .map_err(|e| anyhow::anyhow!("data URI image ({} octets) : {e}", bytes.len()))? + .to_rgba8() + } else { + image::open(path) + .map_err(|e| anyhow::anyhow!("sprite {path} : {e}"))? + .to_rgba8() + }; + let (w, h) = (img.width(), img.height()); + let pixels = img.into_raw(); + let tex = self.gpu.device.create_texture(&wgpu::TextureDescriptor { + label: Some("sprite"), + size: wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + self.gpu.context.write_texture( + wgpu::TexelCopyTextureInfo { + texture: &tex, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + &pixels, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(w * 4), + rows_per_image: Some(h), + }, + wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 }, + ); + Ok((tex, w, h)) + } + /// Rend une frame dans le RT interne. Le screen `screen`/`webcam` sont des /// carriers `linux_frames` ; la geometrie vient de `plan_frame`. Coeur : /// fond uni + ecran cover-fit. `readback_direct` lit ensuite le RT. @@ -702,6 +753,85 @@ impl Compositor { } } + // Curseur thematise (mode 7, sprite RGBA). Placement UPRIGHT uniquement : + // le tilt (mode 13, sous zoom incline) et le motion blur (taps>1) sont + // des increments a venir. `_tex`/`_view`/`_buf` gardent le sprite en vie + // pendant le pass. Miroir de la branche curseur de `compositor_macos`. + struct CursorDraw { + _buf: wgpu::Buffer, + _tex: wgpu::Texture, + _view: wgpu::TextureView, + bind: wgpu::BindGroup, + } + let cursor_draw: Option = (|| { + let track = cursor_ref.as_ref()?; + let plan = plan_cursor( + &g, + &CursorPlanInput { + render_px: [rw, rh], + u_max, + v_max, + cfg, + live: lp, + scene: scene_ref.as_ref(), + track, + t: self + .cursor_time + .borrow() + .unwrap_or(frame / crate::frame_geometry::FPS), + }, + )?; + let CursorPlacement::Upright { center } = plan.placement else { + // ponytail: tilt (mode 13) a porter quand un zoom incline sera teste. + return None; + }; + let sprites = scene_ref + .as_ref() + .map(|s| s.cursor.cursor_sprites.clone()) + .unwrap_or_default(); + let sprite = plan + .cursor_type + .as_deref() + .and_then(|t| sprites.get(t)) + .or_else(|| sprites.get("arrow"))?; + // Charge (ou recupere du cache) le sprite. Emprunt isole AVANT le + // borrow_mut, comme cote macOS (piege du double emprunt 1re frame). + let cached = self.img_cache.borrow().get(sprite.path.as_str()).cloned(); + let (tex, iw, ih) = match cached { + Some(v) => v, + None => match self.load_image_texture(&sprite.path) { + Ok(v) => { + self.img_cache.borrow_mut().insert(sprite.path.clone(), v.clone()); + v + } + Err(e) => { + eprintln!("[curseur] sprite \"{}\" : {e:#}", sprite.path); + return None; + } + }, + }; + // Ratio preserve : le sprite tient dans un carre de `size_px` de cote. + let ar = iw as f32 / ih.max(1) as f32; + let (pw, ph) = if ar >= 1.0 { + (plan.size_px, plan.size_px / ar) + } else { + (plan.size_px * ar, plan.size_px) + }; + let hotspot = [sprite.hotspot_x, sprite.hotspot_y]; + let cb = LayerCB { + dst: cursor_sprite_dst(center, pw / rw, ph / rh, hotspot), + src: [0.0, 0.0, 1.0, 1.0], + mode: 7.0, + color: [1.0, 1.0, 1.0, 1.0], + fx: plan.clip, + ..Default::default() + }; + let view = tex.create_view(&wgpu::TextureViewDescriptor::default()); + // Sprite RGBA au binding 1 (texY) que le mode 7 echantillonne. + let (buf, bind) = self.make_bind(&cb, Some((&view, &view)), &dummy); + Some(CursorDraw { _buf: buf, _tex: tex, _view: view, bind }) + })(); + let mut encoder = self.gpu.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("compose"), }); @@ -764,6 +894,11 @@ impl Compositor { rpass.set_bind_group(0, &a.bind, &[]); rpass.draw(0..4, 0..1); } + // Curseur en dernier : au-dessus de l'ecran et des annotations. + if let Some(c) = &cursor_draw { + rpass.set_bind_group(0, &c.bind, &[]); + rpass.draw(0..4, 0..1); + } } self.gpu.context.submit(std::iter::once(encoder.finish())); Ok(()) diff --git a/crates/compositor/src/vk_shaders/layer.wgsl b/crates/compositor/src/vk_shaders/layer.wgsl index dc5f297f2..1fd0611cd 100644 --- a/crates/compositor/src/vk_shaders/layer.wgsl +++ b/crates/compositor/src/vk_shaders/layer.wgsl @@ -105,6 +105,18 @@ fn fs_main(i: VsOut) -> @location(0) vec4 { let cov = textureSample(texY, samp, i.uv).r; let a = layer.color.a * cov; return vec4(layer.color.rgb * a, a); + } else if layer.mode > 6.5 && layer.mode < 7.5 { + // Mode 7 -- sprite curseur (PNG RGBA, alpha droite) echantillonne sur + // texY (comme le mode 11 y lie son atlas). `fx` = rect de clip "Clip to + // canvas" [x,y,w,h] en sortie 0..1 (= s_dst si actif, sinon un rect + // englobant : sans effet). Sortie en alpha premultiplie. + if i.pout.x < layer.fx.x || i.pout.x > layer.fx.x + layer.fx.z + || i.pout.y < layer.fx.y || i.pout.y > layer.fx.y + layer.fx.w { + return vec4(0.0, 0.0, 0.0, 0.0); + } + let s = textureSample(texY, samp, i.uv); + let ca = s.a * layer.color.a; + return vec4(s.rgb * ca, ca); } else { // Mode 2 ÔÇö ombre port├®e (SDF d'un quad arrondi ├®largi de `fx.x`). let spread = layer.fx.x; diff --git a/crates/compositor/tests/compose_linux.rs b/crates/compositor/tests/compose_linux.rs index b733144bf..823b81b8f 100644 --- a/crates/compositor/tests/compose_linux.rs +++ b/crates/compositor/tests/compose_linux.rs @@ -12,6 +12,7 @@ use std::path::Path; use openscreen_compositor::compositor::Compositor; use openscreen_compositor::config::Cfg; +use openscreen_compositor::cursor::CursorTrack; use openscreen_compositor::d3d::Gpu; use openscreen_compositor::pipeline::Decoder; use openscreen_compositor::scene::Scene; @@ -75,3 +76,68 @@ fn compose_linux_rend_une_frame() { "mean R={mean_r} hors plage plausible (5..250) — frame vide ?" ); } + +/// Curseur : sprite thematise (mode 7) dessine au centre. Sprite VERT (data URI +/// PNG) distinct du fond sombre et de l'ecran, pour l'affirmer sans ambiguite. +#[test] +fn compose_linux_dessine_le_curseur() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("compose_linux curseur: opt-in (OPENSCREEN_LINUX_COMPOSE=1 + fixture). Skip."); + return; + } + + // Sprite vert 16x16 opaque en data URI (decode_data_uri -> crate image). + const SPRITE: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAACXBIWXMAAAABAAAAAQBPJcTWAAAAGElEQVR4nGNk+MdAEmAhTfmohlENQ0kDAGoRATwbkCdPAAAAAElFTkSuQmCC"; + + let gpu = Gpu::create(false).expect("Gpu::create"); + let comp = Compositor::new_sized(&gpu, W, H).expect("Compositor::new_sized"); + let mut dec = Decoder::open(FIXTURE, &gpu).expect("Decoder::open"); + + // Scene : curseur visible (size 3 pour un sprite bien lisible), sprite "arrow". + let scene_json = format!( + r##"{{"clips":[],"layout":{{"preset":"no-webcam","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}},"effects":{{"padding":0,"blur":false,"shadow":0,"roundnessFrac":0.03,"motionBlur":0}},"background":{{"kind":"color","color":"#101015"}},"zoomRegions":[],"annotations":[],"cursor":{{"show":true,"size":3,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default","cursorSprites":{{"arrow":{{"path":"{SPRITE}","hotspotX":0.5,"hotspotY":0.5}}}}}},"cropByClip":[],"output":{{"width":1920,"height":1080,"fps":30}}}}"## + ); + comp.set_scene(Some(Scene::from_json(&scene_json).expect("scene json"))); + + // Piste curseur : un echantillon au centre (0.5, 0.5). `load` lit un fichier. + let track_path = std::env::temp_dir().join("os_cursor_track.json"); + std::fs::write( + &track_path, + r#"{"samples":[{"timeMs":0,"cx":0.5,"cy":0.5,"cursorType":"arrow"}]}"#, + ) + .expect("write track"); + let track = CursorTrack::load(track_path.to_str().unwrap(), 0.0, 2.0).expect("CursorTrack::load"); + comp.set_cursor(track); + comp.set_cursor_time(Some(0.0)); + + let (w, h, rgba) = unsafe { + let sf = dec.seek_to(1.0).expect("Decoder::seek_to"); + let cfg = Cfg::c8(); + comp.compose_frame(sf, sf, 0.0, &cfg).expect("compose_frame"); + comp.readback_direct().expect("readback_direct") + }; + + // Le sprite vert doit apparaitre franchement (G haut, R/B bas). + let green = rgba + .chunks_exact(4) + .filter(|p| p[1] > 180 && p[0] < 120 && p[2] < 120) + .count(); + println!("compose_linux curseur : {w}x{h} pixels verts={green}"); + + let out = std::env::var("OPENSCREEN_VK_OUT").unwrap_or_else(|_| "target".into()); + let _ = std::fs::create_dir_all(&out); + let ppm = format!("{out}/compose_linux_cursor.ppm"); + { + use std::io::Write; + let mut f = std::fs::File::create(&ppm).expect("create ppm"); + write!(f, "P6\n{w} {h}\n255\n").unwrap(); + let mut rgb = vec![0u8; (w * h * 3) as usize]; + for (d, s) in rgb.chunks_exact_mut(3).zip(rgba.chunks_exact(4)) { + d.copy_from_slice(&s[0..3]); + } + f.write_all(&rgb).unwrap(); + } + println!("wrote {ppm}"); + + assert!(green > 50, "sprite curseur vert absent (verts={green}) — mode 7 ?"); +} From a5fd6eb19f2201ac010c8884e63b96b2ddce65b0 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Thu, 30 Jul 2026 19:02:56 +0200 Subject: [PATCH 06/46] =?UTF-8?q?feat(compositor):=20Linux=20video=20expor?= =?UTF-8?q?t=20(WP6)=20=E2=80=94=20software=20encode=20+=20MP4=20mux?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement run_composited_multi for Linux: software VideoEncoder (libopenh264 / libkvazaar, the LGPL-build encoders that need no HW device), the composited RGBA frame read back (readback_direct) and converted to YUV420P via sws_scale, fed through the SHARED walk_composited_timeline into an MP4 muxer (avformat + the sn_fmt_set_pb C shim, as on Windows/macOS). Video-only for now; AAC audio mux is the next increment (audio.rs / AacEncoder are already shared), as is VAAPI/Vulkan hardware encode. Verified: export_linux_mp4 test renders ~1s of the fixture -> a valid 30-frame H264 MP4 (ffprobe: h264 640x360, 30 packets), content correct. --- crates/compositor/src/pipeline_linux.rs | 327 +++++++++++++++++++++-- crates/compositor/tests/compose_linux.rs | 56 +++- 2 files changed, 365 insertions(+), 18 deletions(-) diff --git a/crates/compositor/src/pipeline_linux.rs b/crates/compositor/src/pipeline_linux.rs index 2693f6ff3..b819cb3eb 100644 --- a/crates/compositor/src/pipeline_linux.rs +++ b/crates/compositor/src/pipeline_linux.rs @@ -5,13 +5,18 @@ //! surface publique consommee par le code partage (`Decoder`, `ClipSource`, //! `ExportCodec`, `ExportParams`, `Stats`, `run_composited_multi`). //! -//! **Export.** `run_composited_multi` (encode + mux MP4) est un **stub** ici, -//! comme `pipeline_macos` stube plusieurs de ses propres fonctions d'export : -//! le cablage VAAPI/libopenh264 + muxer (via le shim C pour `AVFormatContext. -//! streams`, opaque en bindgen) arrive avec WP6. La PREVIEW (le `Decoder`) -//! est, elle, complete -- c'est ce dont `live.rs` a besoin. +//! **Export (WP6).** `run_composited_multi` encode + mux un MP4 **vidéo** : +//! encodeur SOFTWARE (`libopenh264` H264 / `libkvazaar` H265 -- les seuls du +//! build LGPL BtbN qui marchent sans device HW, VAAPI/Vulkan-encode = suivi), +//! la frame composée est relue en RGBA (`readback_direct`) puis convertie +//! YUV420P par `sws_scale`. La marche de timeline est PARTAGÉE +//! (`timeline_walk::walk_composited_timeline`) et le muxer passe par le shim C +//! `sn_fmt_set_pb` (comme Windows/macOS). **L'audio AAC n'est pas encore muxé** +//! (increment suivant : `audio.rs` + `AacEncoder` sont déjà partagés). use anyhow::{bail, Result}; +use std::collections::HashMap; +use std::ffi::CString; use std::ptr; use crate::config::Cfg; @@ -20,6 +25,10 @@ use crate::ffi::AVFrame; use crate::linux_decode::SwDecoder; use crate::linux_frames::CpuFrames; +/// `SWS_POINT` (plus proche voisin). Bindgen ne genere pas les `SWS_*` (macros), +/// valeur figee par l'ABI de libswscale -- comme `linux_frames::SWS_POINT`. +const SWS_POINT: i32 = 0x10; + /// Bilan d'un run d'export. Memes champs que `pipeline_macos::Stats`. pub struct Stats { pub frames: u64, @@ -143,18 +152,302 @@ impl Decoder { } } -/// Export multiclip -- **STUB (WP6)**. L'encode (VAAPI/libopenh264) + le muxer -/// MP4 (via shim C pour `AVFormatContext.streams`) arrivent plus tard ; -/// `pipeline_macos` stube de meme son chemin d'export. On echoue lisiblement -/// plutot que d'ecrire un fichier vide. +/// Encodeur video SOFTWARE (`libopenh264` / `libkvazaar`). Pas de zero-copy HW +/// (VAAPI/Vulkan-encode = suivi) : la frame composee est relue RGBA +/// (`readback_direct`) puis convertie YUV420P par `sws_scale`. Surface +/// `open`/`send_composited`/`flush` alignee sur le chemin software de +/// `pipeline_macos::VideoEncoder`. +pub struct VideoEncoder { + ctx: *mut crate::ffi::AVCodecContext, + /// AVFrame YUV420P envoyee a l'encodeur. + sw: *mut AVFrame, + /// RGBA (sortie compositeur) -> YUV420P. Cree paresseusement (dims du readback). + sws: *mut crate::ffi::SwsContext, + w: i32, + h: i32, +} + +// SAFETY : pointeurs FFI sans affinite thread ; caller mono-thread (idem Decoder). +unsafe impl Send for VideoEncoder {} + +impl VideoEncoder { + /// Encodeurs software candidats du build LGPL, par codec. La premiere qui + /// ouvre gagne ; `OPENSCREEN_EXPORT_ENCODER=` force un choix. + fn candidate_names(codec: &ExportCodec) -> &'static [&'static str] { + match codec { + ExportCodec::H264 => &["libopenh264"], + ExportCodec::H265 => &["libkvazaar"], + } + } + + pub fn open(codec: &ExportCodec, w: i32, h: i32, fps: i32, bit_rate: i64) -> Result { + let forced = std::env::var("OPENSCREEN_EXPORT_ENCODER").ok(); + let mut refused: Vec = Vec::new(); + // Liste par defaut, plus l'encodeur force s'il n'y figure pas (ex. h264_vaapi). + let defaults = Self::candidate_names(codec); + let extra: Vec<&str> = forced + .as_deref() + .filter(|f| !defaults.contains(f)) + .into_iter() + .collect(); + for &name in defaults.iter().chain(extra.iter()) { + if forced.as_deref().is_some_and(|f| f != name) { + continue; + } + match unsafe { Self::try_open(name, w, h, fps, bit_rate) } { + Ok(enc) => { + eprintln!("[pipeline] encodeur video : {name} (software YUV420P)"); + return Ok(enc); + } + Err(e) => refused.push(format!("{name}: {e}")), + } + } + match forced { + Some(name) => bail!("OPENSCREEN_EXPORT_ENCODER={name} inutilisable : {}", refused.join(" ; ")), + None => bail!("aucun encodeur video utilisable : {}", refused.join(" ; ")), + } + } + + unsafe fn try_open(name: &str, w: i32, h: i32, fps: i32, bit_rate: i64) -> Result { + use crate::ffi::*; + let cname = CString::new(name)?; + let enc = avcodec_find_encoder_by_name(cname.as_ptr()); + if enc.is_null() { + bail!("absent de ce build ffmpeg"); + } + let mut ctx = avcodec_alloc_context3(enc); + if ctx.is_null() { + bail!("avcodec_alloc_context3"); + } + (*ctx).width = w; + (*ctx).height = h; + (*ctx).pix_fmt = AVPixelFormat::AV_PIX_FMT_YUV420P; + (*ctx).time_base = AVRational { num: 1, den: fps }; + (*ctx).framerate = AVRational { num: fps, den: 1 }; + (*ctx).bit_rate = bit_rate; + // MP4 : header global dans l'extradata (pas par-paquet). + (*ctx).flags |= AV_CODEC_FLAG_GLOBAL_HEADER as i32; + if let Err(e) = averr(avcodec_open2(ctx, enc, ptr::null_mut()), "avcodec_open2(enc)") { + avcodec_free_context(&mut ctx); + return Err(e); + } + match alloc_sw_frame(AVPixelFormat::AV_PIX_FMT_YUV420P, w, h) { + Ok(sw) => Ok(VideoEncoder { ctx, sw, sws: ptr::null_mut(), w, h }), + Err(e) => { + avcodec_free_context(&mut ctx); + Err(e) + } + } + } + + /// Relit la frame composee (RGBA) et l'envoie a l'encodeur en YUV420P. + pub unsafe fn send_composited( + &mut self, + comp: &crate::compositor::Compositor, + pts: i64, + ) -> Result<()> { + use crate::ffi::*; + let (rw, rh, rgba) = comp.readback_direct()?; + let (rw, rh) = (rw as i32, rh as i32); + if self.sws.is_null() { + self.sws = sws_getContext( + rw, + rh, + AVPixelFormat::AV_PIX_FMT_RGBA, + self.w, + self.h, + AVPixelFormat::AV_PIX_FMT_YUV420P, + // POINT : le compositeur est dimensionne a la sortie -> pas de + // mise a l'echelle, donc echantillonnage exact (cf. mac_frames). + SWS_POINT, + ptr::null_mut(), + ptr::null_mut(), + ptr::null(), + ); + if self.sws.is_null() { + bail!("sws_getContext {rw}x{rh} RGBA -> {}x{} YUV420P", self.w, self.h); + } + } + averr(av_frame_make_writable(self.sw), "make_writable")?; + // RGBA est un plan unique : data[0] + stride rw*4, les autres nuls. + let src_data: [*const u8; 4] = [rgba.as_ptr(), ptr::null(), ptr::null(), ptr::null()]; + let src_stride: [i32; 4] = [rw * 4, 0, 0, 0]; + let converted = sws_scale( + self.sws, + src_data.as_ptr(), + src_stride.as_ptr(), + 0, + rh, + (*self.sw).data.as_ptr() as *const *mut u8, + (*self.sw).linesize.as_ptr(), + ); + if converted <= 0 { + bail!("sws_scale RGBA->YUV420P : {converted} lignes"); + } + (*self.sw).pts = pts; + averr(avcodec_send_frame(self.ctx, self.sw), "send_frame") + } + + /// Flush : une frame nulle finalise le bitstream de l'encodeur. + pub unsafe fn flush(&mut self) -> Result<()> { + crate::ffi::averr( + crate::ffi::avcodec_send_frame(self.ctx, ptr::null_mut()), + "send_frame_flush", + ) + } +} + +impl Drop for VideoEncoder { + fn drop(&mut self) { + unsafe { + crate::ffi::avcodec_free_context(&mut self.ctx); + if !self.sw.is_null() { + crate::ffi::av_frame_free(&mut self.sw); + } + if !self.sws.is_null() { + crate::ffi::sws_freeContext(self.sws); + } + } + } +} + +/// Alloue une AVFrame systeme au format demande. Symetrique de +/// `pipeline_macos::alloc_sw_frame`. +unsafe fn alloc_sw_frame(pix_fmt: crate::ffi::AVPixelFormat::Type, w: i32, h: i32) -> Result<*mut AVFrame> { + let mut frame = crate::ffi::av_frame_alloc(); + if frame.is_null() { + bail!("av_frame_alloc (encodeur)"); + } + (*frame).format = pix_fmt as i32; + (*frame).width = w; + (*frame).height = h; + if crate::ffi::av_frame_get_buffer(frame, 32) < 0 { + crate::ffi::av_frame_free(&mut frame); + bail!("av_frame_get_buffer {w}x{h} pix_fmt={pix_fmt}"); + } + Ok(frame) +} + +/// Draine les paquets de l'encodeur vers le muxer. Symetrique de +/// `pipeline_macos::drain_encoder`. +unsafe fn drain_encoder( + ectx: *mut crate::ffi::AVCodecContext, + octx: *mut crate::ffi::AVFormatContext, + ostream: *mut crate::ffi::AVStream, + opkt: *mut crate::ffi::AVPacket, +) -> Result<()> { + use crate::ffi::*; + loop { + let r = avcodec_receive_packet(ectx, opkt); + if r == AVERROR_EOF || r == AVERROR_EAGAIN { + return Ok(()); + } + averr(r, "receive_packet")?; + av_packet_rescale_ts(opkt, (*ectx).time_base, (*ostream).time_base); + averr(av_interleaved_write_frame(octx, opkt), "interleaved_write_frame")?; + av_packet_unref(opkt); + } +} + +/// Export multiclip VIDEO (WP6). Encode software + mux MP4. Audio AAC = suivi +/// (`audio.rs`/`AacEncoder` partages, il ne manque que le branchement du 2e flux +/// + l'assemblage PCM par clip, cf. `pipeline_macos::run_composited_multi`). +/// +/// La marche de timeline est PARTAGEE (`walk_composited_timeline`) : elle compose +/// chaque frame de sortie (vitesse/fenetrage/curseur inclus) puis appelle +/// `on_frame(n)`, ou on relit + encode + draine. pub fn run_composited_multi( - _clips: &[ClipSource], - _out: &str, - _gpu: &Gpu, - _comp: &crate::compositor::Compositor, - _cfg: &Cfg, - _params: &ExportParams, - _progress: &mut dyn FnMut(u64), + clips: &[ClipSource], + out: &str, + gpu: &Gpu, + comp: &crate::compositor::Compositor, + cfg: &Cfg, + params: &ExportParams, + progress: &mut dyn FnMut(u64), ) -> Result { - bail!("run_composited_multi: export natif Linux pas encore implemente (WP6)") + if clips.is_empty() { + bail!("run_composited_multi: aucun clip a exporter"); + } + let (out_w, out_h) = (params.width, params.height); + let out_fps = params.fps.unwrap_or(30) as i32; + // bitrate proportionnel a la surface (reference : 8 Mbps @ 1920x1080). + let bit_rate = ((out_w as i64 * out_h as i64 * 8_000_000) / (1920 * 1080)).max(2_000_000); + let t0 = std::time::Instant::now(); + + let mut enc = VideoEncoder::open(¶ms.codec, out_w as i32, out_h as i32, out_fps, bit_rate)?; + let ectx = enc.ctx; + + let mut screen_decs: HashMap = HashMap::new(); + let mut webcam_decs: HashMap = HashMap::new(); + + // ---- muxer MP4 (flux video seul) ---- + let outc = CString::new(out)?; + let mut octx: *mut crate::ffi::AVFormatContext = ptr::null_mut(); + let mut pb: *mut crate::ffi::AVIOContext = ptr::null_mut(); + let ostream; + let opkt; + unsafe { + crate::ffi::averr( + crate::ffi::avformat_alloc_output_context2(&mut octx, ptr::null(), ptr::null(), outc.as_ptr()), + "alloc_output_context2", + )?; + ostream = crate::ffi::avformat_new_stream(octx, ptr::null()); + if ostream.is_null() { + bail!("avformat_new_stream"); + } + crate::ffi::averr( + crate::ffi::avcodec_parameters_from_context((*ostream).codecpar, ectx), + "params_from_ctx", + )?; + (*ostream).time_base = (*ectx).time_base; + crate::ffi::averr( + crate::ffi::avio_open(&mut pb, outc.as_ptr(), crate::ffi::AVIO_FLAG_WRITE as i32), + "avio_open", + )?; + crate::ffi::sn_fmt_set_pb(octx, pb); + crate::ffi::averr( + crate::ffi::avformat_write_header(octx, ptr::null_mut()), + "write_header", + )?; + opkt = crate::ffi::av_packet_alloc(); + } + + let scene = comp.scene_snapshot(); + let frames = unsafe { + crate::timeline_walk::walk_composited_timeline( + clips, + gpu, + comp, + cfg, + out_fps, + &scene, + &mut screen_decs, + &mut webcam_decs, + &mut |n| { + enc.send_composited(comp, n as i64)?; + drain_encoder(ectx, octx, ostream, opkt)?; + progress(n + 1); + Ok(()) + }, + &mut |_clip, _end, _n, _speed| Ok(()), // audio : increment suivant + )? + }; + + unsafe { + enc.flush()?; + drain_encoder(ectx, octx, ostream, opkt)?; + crate::ffi::averr(crate::ffi::av_write_trailer(octx), "write_trailer")?; + crate::ffi::avio_closep(&mut pb); + crate::ffi::avformat_free_context(octx); + let mut opkt = opkt; + crate::ffi::av_packet_free(&mut opkt); + } + + let wall_s = t0.elapsed().as_secs_f64(); + Ok(Stats { + frames, + wall_s, + fps: if wall_s > 0.0 { frames as f64 / wall_s } else { 0.0 }, + video_duration_s: frames as f64 / out_fps as f64, + }) } diff --git a/crates/compositor/tests/compose_linux.rs b/crates/compositor/tests/compose_linux.rs index 823b81b8f..66d361f4f 100644 --- a/crates/compositor/tests/compose_linux.rs +++ b/crates/compositor/tests/compose_linux.rs @@ -14,7 +14,9 @@ use openscreen_compositor::compositor::Compositor; use openscreen_compositor::config::Cfg; use openscreen_compositor::cursor::CursorTrack; use openscreen_compositor::d3d::Gpu; -use openscreen_compositor::pipeline::Decoder; +use openscreen_compositor::pipeline::{ + run_composited_multi, ClipSource, Decoder, ExportCodec, ExportParams, +}; use openscreen_compositor::scene::Scene; const FIXTURE: &str = "../fixture/screen.mp4"; @@ -141,3 +143,55 @@ fn compose_linux_dessine_le_curseur() { assert!(green > 50, "sprite curseur vert absent (verts={green}) — mode 7 ?"); } + +/// Export (WP6) : ~1s de la fixture -> MP4 H264 software. Verifie que la marche +/// de timeline + l'encodeur + le muxer produisent un fichier non trivial. Le +/// contenu est re-validable par ffprobe (cf. la commande dans le run manuel). +#[test] +fn export_linux_mp4() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("export_linux: opt-in (OPENSCREEN_LINUX_COMPOSE=1 + fixture). Skip."); + return; + } + + let gpu = Gpu::create(false).expect("Gpu::create"); + // Petite sortie : l'export est un smoke test, pas un bench. + let comp = Compositor::new_sized(&gpu, 640, 360).expect("Compositor::new_sized"); + + let out = std::env::var("OPENSCREEN_EXPORT_OUT") + .unwrap_or_else(|_| std::env::temp_dir().join("os_export_linux.mp4").to_string_lossy().into()); + let clips = vec![ClipSource { + screen: FIXTURE.to_string(), + webcam: FIXTURE.to_string(), + source_start_sec: 0.0, + source_end_sec: 1.0, + webcam_offset_sec: 0.0, + has_audio: false, + }]; + let params = ExportParams { + width: 640, + height: 360, + fps: Some(30), + codec: ExportCodec::H264, + }; + + let mut last = 0u64; + let stats = run_composited_multi( + &clips, + &out, + &gpu, + &comp, + &Cfg::c8(), + ¶ms, + &mut |n| last = n, + ) + .expect("run_composited_multi"); + println!( + "export_linux : {} frames, {:.1} fps encode, {:.2}s video, progress={last} -> {out}", + stats.frames, stats.fps, stats.video_duration_s + ); + + assert!(stats.frames > 0, "aucune frame exportee"); + let meta = std::fs::metadata(&out).expect("mp4 metadata"); + assert!(meta.len() > 2000, "mp4 trop petit ({} octets) — muxer ?", meta.len()); +} From aa8d310578ca0a090b2d3228be7913d9c371670a Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Thu, 30 Jul 2026 19:12:26 +0200 Subject: [PATCH 07/46] feat(compositor): mux AAC audio in Linux export (WP6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire the shared audio machinery (audio.rs: AacEncoder, decode_clip_audio, stretch_clip_pcm_by_speed, build_audio_concat_plan, assemble_concatenated_pcm) into run_composited_multi: add the AAC stream before the header, collect one PCM per clip in on_clip_end (speed-stretched to the frames the clip actually produced), then a single AAC encode after the video walk. Silent track when a clip has no audio — parity with Windows/macOS which always mux AAC. Verified: export_linux_mp4 (has_audio=true) yields an MP4 with h264 (30 pkts) + aac (48 pkts), duration 1.0s. --- crates/compositor/src/pipeline_linux.rs | 41 ++++++++++++++++++++++-- crates/compositor/tests/compose_linux.rs | 2 +- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/crates/compositor/src/pipeline_linux.rs b/crates/compositor/src/pipeline_linux.rs index b819cb3eb..02587f8be 100644 --- a/crates/compositor/src/pipeline_linux.rs +++ b/crates/compositor/src/pipeline_linux.rs @@ -19,6 +19,10 @@ use std::collections::HashMap; use std::ffi::CString; use std::ptr; +use crate::audio::{ + assemble_concatenated_pcm, build_audio_concat_plan, decode_clip_audio, + stretch_clip_pcm_by_speed, AacEncoder, PlanarPcm, +}; use crate::config::Cfg; use crate::d3d::Gpu; use crate::ffi::AVFrame; @@ -380,12 +384,13 @@ pub fn run_composited_multi( let mut screen_decs: HashMap = HashMap::new(); let mut webcam_decs: HashMap = HashMap::new(); - // ---- muxer MP4 (flux video seul) ---- + // ---- muxer MP4 (flux video + flux AAC) ---- let outc = CString::new(out)?; let mut octx: *mut crate::ffi::AVFormatContext = ptr::null_mut(); let mut pb: *mut crate::ffi::AVIOContext = ptr::null_mut(); let ostream; let opkt; + let mut audio_encoder; unsafe { crate::ffi::averr( crate::ffi::avformat_alloc_output_context2(&mut octx, ptr::null(), ptr::null(), outc.as_ptr()), @@ -405,6 +410,10 @@ pub fn run_composited_multi( "avio_open", )?; crate::ffi::sn_fmt_set_pb(octx, pb); + // Le flux AAC doit exister AVANT l'en-tete (le muxer y fige sa table de flux). + // Meme si aucun clip n'a d'audio, on ecrit une piste silencieuse -- parite + // avec Windows/macOS, qui muxent toujours l'AAC. + audio_encoder = AacEncoder::open(octx)?; crate::ffi::averr( crate::ffi::avformat_write_header(octx, ptr::null_mut()), "write_header", @@ -412,6 +421,11 @@ pub fn run_composited_multi( opkt = crate::ffi::av_packet_alloc(); } + // Un PCM par clip, assemble apres la marche video (elle seule dit combien de + // frames chaque clip a produit, donc combien d'audio lui revient). + let mut clip_pcm: Vec> = (0..clips.len()).map(|_| None).collect(); + let mut clip_frame_counts: Vec = vec![0; clips.len()]; + let scene = comp.scene_snapshot(); let frames = unsafe { crate::timeline_walk::walk_composited_timeline( @@ -429,13 +443,36 @@ pub fn run_composited_multi( progress(n + 1); Ok(()) }, - &mut |_clip, _end, _n, _speed| Ok(()), // audio : increment suivant + &mut |clip_index, source_end_sec, frames_in_clip, speed_segments| { + clip_frame_counts[clip_index] = frames_in_clip; + let clip = &clips[clip_index]; + if clip.has_audio && frames_in_clip > 0 { + match decode_clip_audio(&clip.screen, clip.source_start_sec, source_end_sec) { + Ok(Some(pcm)) => { + clip_pcm[clip_index] = + Some(stretch_clip_pcm_by_speed(&pcm, speed_segments, out_fps as f64)); + } + Ok(None) => eprintln!( + "[pipeline] warning: clip #{clip_index} declare audio mais sans flux decodable; silence", + ), + Err(error) => eprintln!( + "[pipeline] warning: decodage audio clip #{clip_index} echoue ({error:#}); silence", + ), + } + } + Ok(()) + }, )? }; unsafe { enc.flush()?; drain_encoder(ectx, octx, ostream, opkt)?; + // Audio : le plan part des frames REELLEMENT produites par clip (un clip + // raccourci voit son audio raccourci d'autant), puis un seul encode AAC. + let declared_audio: Vec = clips.iter().map(|c| c.has_audio).collect(); + let plan = build_audio_concat_plan(&clip_frame_counts, &declared_audio, out_fps as f64); + audio_encoder.encode(&assemble_concatenated_pcm(&clip_pcm, &plan), octx)?; crate::ffi::averr(crate::ffi::av_write_trailer(octx), "write_trailer")?; crate::ffi::avio_closep(&mut pb); crate::ffi::avformat_free_context(octx); diff --git a/crates/compositor/tests/compose_linux.rs b/crates/compositor/tests/compose_linux.rs index 66d361f4f..f9f560b82 100644 --- a/crates/compositor/tests/compose_linux.rs +++ b/crates/compositor/tests/compose_linux.rs @@ -166,7 +166,7 @@ fn export_linux_mp4() { source_start_sec: 0.0, source_end_sec: 1.0, webcam_offset_sec: 0.0, - has_audio: false, + has_audio: true, }]; let params = ExportParams { width: 640, From a433d7b1bb4b8b382a29d5a703fa73cd5e192569 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Thu, 30 Jul 2026 19:30:19 +0200 Subject: [PATCH 08/46] feat(compositor): image background (WGSL mode 6 wallpaper) in compositor_linux Handle SceneBackground::Image in compose_frame: load the wallpaper via load_image_texture (reused from the cursor), cover-fit its UV rect to the output aspect, draw it as WGSL mode 6 (opaque RGBA on texY) in the bg-pass. The bg handling is refactored into a BgLayer/BgDraw pair covering gradient (mode 5) and image (mode 6) uniformly. 1:1 port of the macOS draw_image_bg cover math. Verified: compose_linux_fond_image renders an orange data-URI wallpaper filling the background around the inset screen (orange pixels 487 -> 152970 once the scene padding is wired through set_live_params). --- crates/compositor/src/compositor_linux.rs | 76 ++++++++++++++++++--- crates/compositor/src/vk_shaders/layer.wgsl | 5 ++ crates/compositor/tests/compose_linux.rs | 52 ++++++++++++++ 3 files changed, 123 insertions(+), 10 deletions(-) diff --git a/crates/compositor/src/compositor_linux.rs b/crates/compositor/src/compositor_linux.rs index ef7052259..a594266f5 100644 --- a/crates/compositor/src/compositor_linux.rs +++ b/crates/compositor/src/compositor_linux.rs @@ -618,10 +618,14 @@ impl Compositor { }); let _ = (wtw, wth); // webcam PiP : calque a venir. - // Fond : Color -> clear a la couleur ; Gradient -> clear transparent + - // draw mode 5 ; Image -> repli couleur (calque image a venir). Le blur - // (si cfg.bg_blur) floute ensuite ce fond, avant l'ecran. - let (bg_clear, gradient_cb) = match scene_ref.as_ref().map(|s| s.background.clone()) { + // Fond : Color -> clear a la couleur ; Gradient -> mode 5 ; Image -> + // mode 6 wallpaper cover-fit (via load_image_texture). Le blur (si + // cfg.bg_blur) floute ensuite ce fond, avant l'ecran. + enum BgLayer { + Gradient(LayerCB), + Image(String), + } + let (bg_clear, bg_layer) = match scene_ref.as_ref().map(|s| s.background.clone()) { Some(SceneBackground::Color { color }) => { (parse_hex(&color).unwrap_or(lp.bg_color), None) } @@ -638,9 +642,12 @@ impl Compositor { fx: [a.sin(), -a.cos(), 0.0, 0.0], ..Default::default() }; - ([0.0, 0.0, 0.0, 1.0], Some(cb)) + ([0.0, 0.0, 0.0, 1.0], Some(BgLayer::Gradient(cb))) + } + Some(SceneBackground::Image { path }) => { + ([0.0, 0.0, 0.0, 1.0], Some(BgLayer::Image(path))) } - _ => (lp.bg_color, None), + None => (lp.bg_color, None), }; // Calque ecran (mode 0 : NV12 -> RGB), place par plan_frame (cover-fit + @@ -660,8 +667,57 @@ impl Compositor { let (_screen_uniform, screen_bind) = self.make_bind(&screen_layer, Some((&sy, &suv)), &dummy); - // Fond gradient (mode 5), dessine dans la passe de fond si present. - let gradient_bind = gradient_cb.as_ref().map(|cb| self.make_bind(cb, None, &dummy)); + // Fond (gradient mode 5 OU image mode 6), dessine dans la passe de fond. + // `_tex`/`_view` gardent l'image en vie pendant le pass. + struct BgDraw { + _buf: wgpu::Buffer, + _tex: Option, + _view: Option, + bind: wgpu::BindGroup, + } + let bg_draw = bg_layer.and_then(|bl| match bl { + BgLayer::Gradient(cb) => { + let (buf, bind) = self.make_bind(&cb, None, &dummy); + Some(BgDraw { _buf: buf, _tex: None, _view: None, bind }) + } + BgLayer::Image(path) => { + // Charge (ou recupere du cache) le wallpaper. Emprunt isole AVANT + // le borrow_mut (piege du double emprunt 1re frame, cf. macOS). + let cached = self.img_cache.borrow().get(path.as_str()).cloned(); + let (tex, iw, ih) = match cached { + Some(v) => v, + None => match self.load_image_texture(&path) { + Ok(v) => { + self.img_cache.borrow_mut().insert(path.clone(), v.clone()); + v + } + Err(e) => { + eprintln!("[fond image] \"{path}\" : {e:#}"); + return None; + } + }, + }; + // Cover-fit : l'image remplit tout le cadre, on rogne l'axe long. + let ai = iw as f32 / ih.max(1) as f32; + let ao = rw / rh; + let src = if ai > ao { + let vis = ao / ai; + [(1.0 - vis) * 0.5, 0.0, 1.0 - (1.0 - vis) * 0.5, 1.0] + } else { + let vis = ai / ao; + [0.0, (1.0 - vis) * 0.5, 1.0, 1.0 - (1.0 - vis) * 0.5] + }; + let cb = LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + src, + mode: 6.0, + ..Default::default() + }; + let view = tex.create_view(&wgpu::TextureViewDescriptor::default()); + let (buf, bind) = self.make_bind(&cb, Some((&view, &view)), &dummy); + Some(BgDraw { _buf: buf, _tex: Some(tex), _view: Some(view), bind }) + } + }); // Webcam PiP (mode 0) -- placee par plan_frame (`g.w_dst`, coins // `g.w_radius`), gardee par `g.shape_fade > 0` (webcam visible). @@ -856,9 +912,9 @@ impl Compositor { timestamp_writes: None, occlusion_query_set: None, }); - if let Some((_buf, bind)) = &gradient_bind { + if let Some(bg) = &bg_draw { rpass.set_pipeline(&self.pipeline); - rpass.set_bind_group(0, bind, &[]); + rpass.set_bind_group(0, &bg.bind, &[]); rpass.draw(0..4, 0..1); } } diff --git a/crates/compositor/src/vk_shaders/layer.wgsl b/crates/compositor/src/vk_shaders/layer.wgsl index 1fd0611cd..379fe04a4 100644 --- a/crates/compositor/src/vk_shaders/layer.wgsl +++ b/crates/compositor/src/vk_shaders/layer.wgsl @@ -117,6 +117,11 @@ fn fs_main(i: VsOut) -> @location(0) vec4 { let s = textureSample(texY, samp, i.uv); let ca = s.a * layer.color.a; return vec4(s.rgb * ca, ca); + } else if layer.mode > 5.5 && layer.mode < 6.5 { + // Mode 6 -- fond image (wallpaper RGBA) cover-fit, echantillonne sur + // texY. `src` porte le rect UV cover-fit (calcule cote Rust). Opaque : + // le fond couvre tout le cadre. + return vec4(textureSample(texY, samp, i.uv).rgb, 1.0); } else { // Mode 2 ÔÇö ombre port├®e (SDF d'un quad arrondi ├®largi de `fx.x`). let spread = layer.fx.x; diff --git a/crates/compositor/tests/compose_linux.rs b/crates/compositor/tests/compose_linux.rs index f9f560b82..e63f4734a 100644 --- a/crates/compositor/tests/compose_linux.rs +++ b/crates/compositor/tests/compose_linux.rs @@ -144,6 +144,58 @@ fn compose_linux_dessine_le_curseur() { assert!(green > 50, "sprite curseur vert absent (verts={green}) — mode 7 ?"); } +/// Fond image (mode 6 wallpaper) : un PNG orange en data URI remplit le fond +/// (cover-fit) autour de l'ecran inset (padding). Distinct de l'ecran et du +/// gris par defaut, pour l'affirmer sans ambiguite. +#[test] +fn compose_linux_fond_image() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("compose_linux fond image: opt-in. Skip."); + return; + } + const BG: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAACXBIWXMAAAABAAAAAQBPJcTWAAAAKklEQVR4nO3NwQ0AAAQAMRJ721wswa83wDWn47X63QMAAAAAAAAAAIC7FhLfAfuIQEbyAAAAAElFTkSuQmCC"; + + let gpu = Gpu::create(false).expect("Gpu::create"); + let comp = Compositor::new_sized(&gpu, W, H).expect("Compositor::new_sized"); + let mut dec = Decoder::open(FIXTURE, &gpu).expect("Decoder::open"); + + let scene_json = format!( + r##"{{"clips":[],"layout":{{"preset":"no-webcam","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}},"effects":{{"padding":0.4,"blur":false,"shadow":0,"roundnessFrac":0.05,"motionBlur":0}},"background":{{"kind":"image","path":"{BG}"}},"zoomRegions":[],"annotations":[],"cursor":{{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}},"cropByClip":[],"output":{{"width":1920,"height":1080,"fps":30}}}}"## + ); + let scene = Scene::from_json(&scene_json).expect("scene json"); + // Le padding (et les autres effets) transitent par les live_params, pas la + // scene brute -> sans ca l'ecran remplit tout le cadre et masque le fond. + comp.set_live_params(openscreen_compositor::compositor::live_params_from_scene(&scene)); + comp.set_scene(Some(scene)); + + let (w, h, rgba) = unsafe { + let sf = dec.seek_to(1.0).expect("Decoder::seek_to"); + let cfg = Cfg::c8(); + comp.compose_frame(sf, sf, 0.0, &cfg).expect("compose_frame"); + comp.readback_direct().expect("readback_direct") + }; + // Orange (255,128,0) : R haut, G moyen, B bas. + let orange = rgba + .chunks_exact(4) + .filter(|p| p[0] > 200 && p[1] > 90 && p[1] < 170 && p[2] < 70) + .count(); + println!("compose_linux fond image : {w}x{h} pixels orange={orange}"); + + let out = std::env::var("OPENSCREEN_VK_OUT").unwrap_or_else(|_| "target".into()); + let _ = std::fs::create_dir_all(&out); + { + use std::io::Write; + let mut f = std::fs::File::create(format!("{out}/compose_linux_bgimage.ppm")).expect("ppm"); + write!(f, "P6\n{w} {h}\n255\n").unwrap(); + let mut rgb = vec![0u8; (w * h * 3) as usize]; + for (d, s) in rgb.chunks_exact_mut(3).zip(rgba.chunks_exact(4)) { + d.copy_from_slice(&s[0..3]); + } + f.write_all(&rgb).unwrap(); + } + assert!(orange > 2000, "fond image absent (orange={orange}) — mode 6 ?"); +} + /// Export (WP6) : ~1s de la fixture -> MP4 H264 software. Verifie que la marche /// de timeline + l'encodeur + le muxer produisent un fichier non trivial. Le /// contenu est re-validable par ffprobe (cf. la commande dans le run manuel). From 3d5c0380c3eb2e14a9ecded7874ef9d2bf34c3f3 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Thu, 30 Jul 2026 21:33:35 +0200 Subject: [PATCH 09/46] fix(hud): -0 out of Math.round crashed the main process on Wayland MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Math.round` returns negative zero for any delta in [-0.5, 0), which is routine under fractional scaling where screenY deltas are fractional. V8's IsInt32() rejects -0, so gin refuses the conversion and BrowserWindow's native setPosition throws "Error processing argument at index 1, conversion failure from" — fatal, because installMainProcessErrorGuards re-throws. `Number.isFinite(-0)` is true, so the existing finiteness guard could not catch it. `| 0` collapses -0 to 0 and pins the value to int32. Reproduced on Electron 41.2.1: flipping which axis carries the -0 flips the reported argument index, and the empty tail after "from " identifies the bad value as a Number rather than undefined/null. --- electron/windows.ts | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/electron/windows.ts b/electron/windows.ts index 32046fe36..bf219ca9c 100644 --- a/electron/windows.ts +++ b/electron/windows.ts @@ -115,8 +115,14 @@ ipcMain.on("hud-overlay-drag-start", () => { return; } + // Under Wayland this origin is a lie: Electron documents getPosition() as returning + // [0, 0] there, because the protocol prohibits a client from introspecting or + // setting its own global coordinates. The origin+delta scheme below therefore + // resolves against 0 rather than the window's real position, and setPosition() is + // itself a no-op — so dragging cannot work on Wayland by this route at all. The + // finiteness check only keeps a garbage origin from reaching a native setter. const [x, y] = hudOverlayWindow.getPosition(); - hudDragOrigin = { x, y }; + hudDragOrigin = Number.isFinite(x) && Number.isFinite(y) ? { x, y } : null; }); ipcMain.on("hud-overlay-drag-to", (_event, deltaX: number, deltaY: number) => { @@ -130,11 +136,17 @@ ipcMain.on("hud-overlay-drag-to", (_event, deltaX: number, deltaY: number) => { return; } - hudOverlayWindow.setPosition( - Math.round(hudDragOrigin.x + deltaX), - Math.round(hudDragOrigin.y + deltaY), - false, - ); + // `| 0` is load-bearing, not defensive noise. Math.round returns NEGATIVE ZERO for + // any delta in [-0.5, 0) — routine under fractional scaling, where screenY deltas + // are fractional. V8's IsInt32() rejects -0, so gin refuses to convert it and the + // main process dies with "Error processing argument at index 1, conversion failure + // from". `Number.isFinite(-0)` is true, so a finiteness check does NOT catch this; + // `| 0` collapses -0 to 0 and pins the value to int32. (`+ 0` would not: -0 + 0 is + // still -0, and Math.trunc preserves it too.) + const x = Math.round(hudDragOrigin.x + deltaX) | 0; + const y = Math.round(hudDragOrigin.y + deltaY) | 0; + + hudOverlayWindow.setPosition(x, y, false); }); ipcMain.on("hud-overlay-drag-end", () => { From 8c939a9ab782fd35a3b741cee6636c97c68f37d4 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Thu, 30 Jul 2026 21:33:48 +0200 Subject: [PATCH 10/46] fix(hud): the drag region sat on the window root, not the grab handle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The HUD root carried `-webkit-app-region: drag` while the grab handle explicitly opted out with `no-drag` — exactly inverted. One cause, both reported symptoms on Wayland: - pressing empty space beside the bar dragged the HUD, because the root is the whole 820x560 window and a compositor honours a drag region whether or not anything is painted there. It reads as "dragging the window behind". - pressing the grab handle did nothing, because it fell through to the IPC/setPosition path, which Wayland prohibits: getPosition() answers [0,0] and setPosition() only updates Electron's own cache. Windows and macOS never saw this: setIgnoreMouseEvents makes the transparent area input-transparent at the OS level there, so the root region was never pressable. That call is a no-op on Wayland. The handle now takes the native drag region only where the platform cannot position itself; Windows and macOS keep the pointer-handler path unchanged, since a drag region swallows pointer events and the two cannot coexist. --- src/components/launch/HudControls.tsx | 19 ++++++++++++++++++- src/components/launch/LaunchWindow.tsx | 10 +++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/components/launch/HudControls.tsx b/src/components/launch/HudControls.tsx index 788328059..360499c79 100644 --- a/src/components/launch/HudControls.tsx +++ b/src/components/launch/HudControls.tsx @@ -50,11 +50,26 @@ export const HudDivider = memo(function HudDivider({ vertical }: { vertical: boo export const HudDragHandle = memo(function HudDragHandle({ vertical, + nativeDrag, onPointerDown, onPointerMove, onPointerEnd, }: { vertical: boolean; + /** + * Hand the gesture to the compositor instead of the pointer handlers below. + * + * Wayland forbids a client from reading or setting its own global position: + * `getPosition()` answers [0, 0] and `setPosition()` only updates Electron's + * own cache, so the origin+delta scheme the handlers implement cannot move + * the window there. `-webkit-app-region: drag` is the one path that does — + * it routes to `xdg_toplevel.move` and the compositor performs the move. + * + * A drag region swallows pointer events, so the two are mutually exclusive: + * platforms that can position themselves keep the handler path, which gives + * finer control and lets the HUD suppress resizes mid-gesture. + */ + nativeDrag: boolean; onPointerDown: (event: React.PointerEvent) => void; onPointerMove: (event: React.PointerEvent) => void; onPointerEnd: (event: React.PointerEvent) => void; @@ -62,7 +77,9 @@ export const HudDragHandle = memo(function HudDragHandle({ return ( ); diff --git a/src/lib/cursor/cursorCapabilities.ts b/src/lib/cursor/cursorCapabilities.ts new file mode 100644 index 000000000..271372b04 --- /dev/null +++ b/src/lib/cursor/cursorCapabilities.ts @@ -0,0 +1,26 @@ +import { getPlatform } from "@/utils/platformUtils"; + +/** + * Can this platform tell us when a mouse button was pressed? + * + * Not a preference and not a feature flag — a hard limit of the display server. + * On Wayland an unprivileged process cannot observe mouse buttons at all: the + * ScreenCast portal reports pointer POSITION as frame metadata and nothing else, + * there is no portal for input events, and `/dev/input/event*` is `root:input`. + * So the Linux capture helper stamps every sample `interactionType: "move"` + * (see `pipeWireCursorAccumulator.ts`), the compositor's `CursorTrack.clicks` + * vector is always empty, and `CursorTrack::bounce()` returns exactly 1.0 — + * which `frame_geometry.rs` multiplies the cursor size by, leaving it unchanged + * for every possible slider value. + * + * macOS and Windows both read real button state in their native cursor helpers + * (`macNativeCursorRecordingSession.ts`, `windowsNativeRecordingSession.ts`), so + * the effect works there. + * + * Same shape and same intent as `supportsWebcamReactiveZoom` in + * `compositeLayout.ts`: a control that provably cannot change a pixel is + * dropped rather than shown doing nothing. + */ +export function supportsCursorClickEffects(): boolean { + return getPlatform() !== "linux"; +} From 6cfbfaa0a530f76a894f9ae8b5cfdb2f383a52eb Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Fri, 31 Jul 2026 17:47:49 +0200 Subject: [PATCH 33/46] feat(compositor): draw the screen and camera shadows on Linux MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit layer.wgsl has had the mode-2 drop shadow since the port began — a rounded-rect SDF whose alpha falls off across the spread. What was missing was the Rust half: compose_frame never built a mode-2 LayerCB and never issued the two draws, so `cfg.shadow` and `lp.shadow_scale` were read by nobody and the Shadow slider did nothing at all on Linux. Both shadows use the tuning fractions already shared with macOS/Windows (SCREEN_/WEBCAM_SHADOW_SPREAD_FRAC and friends in frame_geometry), so they are the same relative size on all three platforms at any output resolution. Each is drawn immediately before the layer it belongs to: under that layer, over what is behind it. The camera shadow is skipped in the dual-frame and vertical-stack presets — the camera is tiled flush against the screen there, and a shadow between them would draw a seam. Same condition as macOS. Adds a rendered-frame test that exercises this together with the two previous fixes: shadow on, camera present, and a text annotation with a background. It asserts the annotation's plate is actually on screen — counting pink pixels in the region — which fails outright when the plate is not drawn, rather than only being visible to someone opening the PPM. --- crates/compositor/src/compositor_linux.rs | 85 +++++++++++++++++++++++ crates/compositor/tests/compose_linux.rs | 71 +++++++++++++++++++ 2 files changed, 156 insertions(+) diff --git a/crates/compositor/src/compositor_linux.rs b/crates/compositor/src/compositor_linux.rs index 566b39a3e..5b4cd0f34 100644 --- a/crates/compositor/src/compositor_linux.rs +++ b/crates/compositor/src/compositor_linux.rs @@ -563,6 +563,37 @@ impl Compositor { /// (uniform + deux textures + sampler). Cree AVANT la render pass pour que /// les ressources vivent pendant tout le pass. Un buffer PAR draw : /// `write_buffer` entre draws d'une meme pass ne s'entrelace pas. + /// `LayerCB` d'une ombre portee (mode 2), identique a `draw_shadow` cote + /// macOS et au bloc equivalent cote Windows. + /// + /// Le quad est ELARGI de `spread` de chaque cote et decale de `offset_px` ; + /// le shader y trace un SDF de rect arrondi dont l'alpha decroit sur la + /// largeur du spread. C'est pour ca que `fx.x` porte le spread : le + /// fragment en a besoin pour normaliser sa penombre. + fn shadow_cb( + &self, + dst: [f32; 4], + size_px: [f32; 2], + radius: f32, + spread: f32, + offset_px: [f32; 2], + opacity: f32, + ) -> LayerCB { + let (rw, rh) = (self.render_w as f32, self.render_h as f32); + let (sx, sy) = (spread / rw, spread / rh); + let (ox, oy) = (offset_px[0] / rw, offset_px[1] / rh); + LayerCB { + dst: [dst[0] - sx + ox, dst[1] - sy + oy, dst[2] + 2.0 * sx, dst[3] + 2.0 * sy], + quad_px: [size_px[0] + 2.0 * spread, size_px[1] + 2.0 * spread], + radius_px: radius, + mode: 2.0, + color: [0.0, 0.0, 0.0, opacity], + fx: [spread, 0.0, 0.0, 0.0], + mb: [0.0, 1.0, 1.0, 0.0], + ..Default::default() + } + } + fn make_bind( &self, cb: &LayerCB, @@ -738,6 +769,26 @@ impl Compositor { let (_screen_uniform, screen_bind) = self.make_bind(&screen_layer, Some((&sy, &suv)), &dummy); + // OMBRE PORTEE de l'ecran (mode 2), dessinee JUSTE AVANT le calque + // ecran. Le shader la connait depuis le debut ; ce qui manquait etait + // uniquement le draw cote Rust, si bien que le curseur « Ombre » de + // l'UI ne faisait rien sur Linux. + // + // Les fractions de reglage viennent de `frame_geometry`, partagees avec + // macOS/Windows : l'ombre a la meme taille relative sur les trois + // plateformes quelle que soit la resolution de sortie. + let screen_shadow = cfg.shadow.then(|| { + let cb = self.shadow_cb( + g.s_dst, + [g.s_dst[2] * rw, g.s_dst[3] * rh], + g.s_radius, + crate::frame_geometry::SCREEN_SHADOW_SPREAD_FRAC * g.frame_min_px, + [0.0, crate::frame_geometry::SCREEN_SHADOW_OFFSET_FRAC * g.frame_min_px], + 0.45 * lp.shadow_scale, + ); + self.make_bind(&cb, None, &dummy) + }); + // Fond (gradient mode 5 OU image mode 6), dessine dans la passe de fond. // `_tex`/`_view` gardent l'image en vie pendant le pass. struct BgDraw { @@ -832,6 +883,29 @@ impl Compositor { self.make_bind(&cb, Some((wy, wuv)), &dummy) }); + // OMBRE de la camera. Pas dans les presets « bloc » (dual-frame, + // vertical-stack) : la camera y est collee a l'ecran comme une tuile, + // et une ombre entre les deux dessinerait une couture. Meme condition + // que macOS. + let webcam_shadow = (cfg.shadow + && g.shape_fade > 0.0 + && webcam_draw.is_some() + && !matches!( + g.scene_preset.as_deref(), + Some("dual-frame") | Some("vertical-stack") + )) + .then(|| { + let cb = self.shadow_cb( + g.w_dst, + g.w_px, + g.w_radius, + crate::frame_geometry::WEBCAM_SHADOW_SPREAD_FRAC * g.frame_min_px, + [0.0, crate::frame_geometry::WEBCAM_SHADOW_OFFSET_FRAC * g.frame_min_px], + crate::frame_geometry::WEBCAM_SHADOW_OPACITY * g.shape_fade, + ); + self.make_bind(&cb, None, &dummy) + }); + // Annotations TEXTE (mode 11) -- placees relativement au rect ecran // `g.s_dst` (les coords x/y/w/h de l'annotation sont des fractions de ce // rect, cf. `scene.rs`). Mirroir de la branche "text" de @@ -1080,8 +1154,19 @@ impl Compositor { occlusion_query_set: None, }); rpass.set_pipeline(&self.pipeline); + // Chaque ombre est dessinee JUSTE AVANT le calque qu'elle porte : + // elle doit passer sous lui mais au-dessus du fond (et, pour la + // camera, au-dessus de l'ecran). + if let Some((_buf, bind)) = &screen_shadow { + rpass.set_bind_group(0, bind, &[]); + rpass.draw(0..4, 0..1); + } rpass.set_bind_group(0, &screen_bind, &[]); rpass.draw(0..4, 0..1); + if let Some((_buf, bind)) = &webcam_shadow { + rpass.set_bind_group(0, bind, &[]); + rpass.draw(0..4, 0..1); + } if let Some((_buf, bind)) = &webcam_draw { rpass.set_bind_group(0, bind, &[]); rpass.draw(0..4, 0..1); diff --git a/crates/compositor/tests/compose_linux.rs b/crates/compositor/tests/compose_linux.rs index e63f4734a..3a2bb9071 100644 --- a/crates/compositor/tests/compose_linux.rs +++ b/crates/compositor/tests/compose_linux.rs @@ -247,3 +247,74 @@ fn export_linux_mp4() { let meta = std::fs::metadata(&out).expect("mp4 metadata"); assert!(meta.len() > 2000, "mp4 trop petit ({} octets) — muxer ?", meta.len()); } + +/// Rend une frame qui exerce EN MEME TEMPS les trois corrections de cette +/// serie : ombre portee (ecran + camera), cover-crop de la webcam sous un +/// masque CERCLE (le cas ou l'etirement etait le plus violent : la boite est +/// forcee carree, donc une camera 16:9 s'ecrasait de 1,78x), et une annotation +/// texte avec un fond. +/// +/// Opt-in comme les autres tests de ce fichier ; ecrit un PPM a inspecter. +#[test] +fn compose_linux_ombre_webcam_ronde_et_texte() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("compose_linux ombre/webcam/texte: opt-in. Skip."); + return; + } + let webcam_fixture = "../fixture/webcam.mp4"; + if !Path::new(webcam_fixture).is_file() { + eprintln!("compose_linux ombre/webcam/texte: pas de fixture webcam. Skip."); + return; + } + + let gpu = Gpu::create(false).expect("Gpu::create"); + let comp = Compositor::new_sized(&gpu, W, H).expect("Compositor::new_sized"); + let mut screen = Decoder::open(FIXTURE, &gpu).expect("Decoder::open screen"); + let mut cam = Decoder::open(webcam_fixture, &gpu).expect("Decoder::open webcam"); + + // `shadow: 1` + camera en cercle + une annotation texte visible a t=1s. + let scene_json = r##"{"clips":[],"layout":{"preset":"picture-in-picture","webcamSize":1,"webcamShape":"circle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false},"effects":{"padding":0.14,"blur":false,"shadow":1,"roundnessFrac":0.04,"motionBlur":0},"background":{"kind":"gradient","angleDeg":45,"stops":["#1f2933","#3b6bff"]},"zoomRegions":[],"annotations":[{"id":"a1","kind":"text","x":0.08,"y":0.08,"w":0.5,"h":0.14,"startSec":0,"endSec":10,"zIndex":1,"text":{"content":"Ombre + fond","color":"#ffffff","backgroundColor":"#e0245e","fontSizeRel":0.09,"fontFamily":"","fontWeight":"normal","fontStyle":"normal","textDecoration":"none","textAlign":"center"}}],"cursor":{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"},"cropByClip":[],"output":{"width":1920,"height":1080,"fps":30}}"##; + comp.set_scene(Some(Scene::from_json(scene_json).expect("scene json"))); + + let (w, h, rgba) = unsafe { + let sf = screen.seek_to(1.0).expect("seek screen"); + let wf = cam.seek_to(1.0).expect("seek webcam"); + let mut cfg = Cfg::c8(); + cfg.shadow = true; + comp.compose_frame(sf, wf, 1.0, &cfg).expect("compose_frame"); + comp.readback_direct().expect("readback_direct") + }; + + let out = std::env::var("OPENSCREEN_VK_OUT").unwrap_or_else(|_| "target".into()); + let _ = std::fs::create_dir_all(&out); + let ppm = format!("{out}/compose_linux_shadow_webcam_text.ppm"); + { + use std::io::Write; + let mut f = std::fs::File::create(&ppm).expect("create ppm"); + write!(f, "P6\n{w} {h}\n255\n").unwrap(); + let mut rgb = vec![0u8; (w * h * 3) as usize]; + for (d, s) in rgb.chunks_exact_mut(3).zip(rgba.chunks_exact(4)) { + d.copy_from_slice(&s[0..3]); + } + f.write_all(&rgb).unwrap(); + } + println!("wrote {ppm}"); + + // L'annotation a un fond ROSE (#e0245e) : il doit exister des pixels + // nettement rouges-magenta dans le quart haut-gauche, ce qui n'etait pas le + // cas quand la plaque n'etait pas dessinee du tout. + let mut plate_px = 0usize; + for y in 0..(h / 3) { + for x in 0..(w / 2) { + let i = ((y * w + x) * 4) as usize; + let (r, g_, b) = (rgba[i] as i32, rgba[i + 1] as i32, rgba[i + 2] as i32); + if r > 140 && g_ < 90 && b > 40 && b < 140 { + plate_px += 1; + } + } + } + assert!( + plate_px > 200, + "fond d'annotation introuvable ({plate_px} px roses) — la plaque n'est pas dessinee" + ); +} From e6bcd791f49d919dd646140403045236f1ee294c Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Fri, 31 Jul 2026 18:12:36 +0200 Subject: [PATCH 34/46] feat(compositor): tilted 3D zoom on Linux (shader modes 8, 12 and 13) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports the perspective-rotated screen layer, its projected-quad shadow, and the tilted cursor. Verified independently of the agent that wrote it: 99 lib tests and 8 rendered-frame tests green in this checkout, and the renders inspected by eye — the iso preset is an unmistakable trapezoid with the right edge receding, rounded corners following the tilt, and the full source content present (so crop and zoom really are applied to (s,t) rather than sampled raw). None of the maths is new: regions.rs was already cross-platform and its tilt tests already ran on this host. What was missing was GPU code and uniform packing. Two porting traps avoided, both called out in the Metal source: - mode 8 has NO dst_prev clip test, because dst_prev.xy holds plane_px in pixels there rather than a 0..1 rect; - mode 12 reads its spread from mb.y, not fx.x as mode 2 does. The MSL's two runtime-indexed local-array loops are hand-unrolled rather than translated literally: naga 24 emits invalid SPIR-V for that pattern and RADV segfaults inside vkCreateGraphicsPipelines. The HLSL uses a dynamic array index for sd_convex_quad, which is exactly why the Metal was the right source to translate from. The new branches sit BEFORE the mode-2 catch-all `else`, which is unbounded — anything falling through it silently renders as a drop shadow. Mode 13 was optional and got done anyway, because modes 8 and 12 make tilted zooms reachable and the old cursor path made the pointer vanish outright under one. That would have been a regression introduced by this very commit. Assertions are geometric rather than existence checks: the upright reference's top edge is flat to 0px while iso/left/right all swing 27px, and left/right land 1px apart, which is the mirror symmetry the presets should produce. Known gap: `regions.rs:956` calls the left/right presets "pure Y rotations", but they are [-8,-16,-1] and [-8,16,1] — all three axes nonzero. No preset is truly affine, so quad_inverse_bilinear's degenerate `abs(k2) < 1e-5 * abs(k1)` branch is ported faithfully but unexercised by these tests. --- crates/compositor/src/compositor_linux.rs | 234 +++++++++++++--- crates/compositor/src/vk_shaders/layer.wgsl | 205 +++++++++++++- crates/compositor/tests/compose_linux.rs | 284 ++++++++++++++++++++ 3 files changed, 680 insertions(+), 43 deletions(-) diff --git a/crates/compositor/src/compositor_linux.rs b/crates/compositor/src/compositor_linux.rs index 5b4cd0f34..18109f19f 100644 --- a/crates/compositor/src/compositor_linux.rs +++ b/crates/compositor/src/compositor_linux.rs @@ -17,7 +17,7 @@ //! //! **Portee actuelle.** `compose_frame` rend le coeur : fond uni + calque ecran //! cover-fit (coins arrondis). Les calques riches (webcam PiP, curseur, -//! annotations texte mode 11, blur de fond, tilt 3D, motion blur) sont dessines +//! annotations texte mode 11, blur de fond, motion blur) sont dessines //! par les memes primitives (`draw_layer`) et arrivent par iterations, comme le //! port Metal les a ajoutes -- chacun reutilise `layer.wgsl` (modes deja portes) //! ou une passe dediee (`blur.wgsl`). @@ -594,6 +594,98 @@ impl Compositor { } } + /// `LayerCB` de l'ombre d'un ecran INCLINE (mode 12) : la penombre suit le + /// quadrilatere projete, pas son rect englobant. Port de + /// `compositor_macos::draw_quad_shadow`. + fn quad_shadow_cb( + &self, + corners: &[(f32, f32); 4], + center_px: [f32; 2], + radius: f32, + spread: f32, + offset_px: [f32; 2], + opacity: f32, + ) -> LayerCB { + let (rw, rh) = (self.render_w as f32, self.render_h as f32); + let (min_x, max_x) = + corners.iter().fold((f32::MAX, f32::MIN), |(mn, mx), &(x, _)| (mn.min(x), mx.max(x))); + let (min_y, max_y) = + corners.iter().fold((f32::MAX, f32::MIN), |(mn, mx), &(_, y)| (mn.min(y), mx.max(y))); + // La boite doit contenir la penombre entiere, sinon elle se coupe net. + let box_w = (max_x - min_x) + 2.0 * spread; + let box_h = (max_y - min_y) + 2.0 * spread; + let local = |(x, y): (f32, f32)| -> [f32; 2] { [x - min_x + spread, y - min_y + spread] }; + let [tl0, tl1] = local(corners[0]); + let [tr0, tr1] = local(corners[1]); + let [br0, br1] = local(corners[2]); + let [bl0, bl1] = local(corners[3]); + LayerCB { + dst: [ + (center_px[0] + min_x - spread + offset_px[0]) / rw, + (center_px[1] + min_y - spread + offset_px[1]) / rh, + box_w / rw, + box_h / rh, + ], + quad_px: [box_w, box_h], + radius_px: radius, + mode: 12.0, + color: [0.0, 0.0, 0.0, opacity], + fx: [tl0, tl1, tr0, tr1], + src_prev: [br0, br1, bl0, bl1], + // Le spread vit ici et NON dans `fx.x` : `fx` porte deja les coins. + mb: [0.0, spread, 1.0, 0.0], + ..Default::default() + } + } + + /// `LayerCB` de l'ecran incline (mode 8) : le quad projete est dessine dans sa + /// BBOX et le fragment remonte au (s,t) du plan par warp bilineaire inverse. + /// Port de `compositor_macos::draw_tilted_screen`. Pas de motion blur sur ce + /// chemin -- le tilt est bref, la simplification ne se voit pas. + fn tilted_screen_cb( + &self, + quad: &crate::regions::TiltedQuad, + s_px: [f32; 2], + center_px: [f32; 2], + cut: [f32; 4], + radius: f32, + ) -> LayerCB { + let (rw, rh) = (self.render_w as f32, self.render_h as f32); + let corners = quad.corners; + // Taille du plan dans son propre repere, AVANT projection : c'est la que vit + // le rayon, pour qu'il reste constant le long du bord au lieu de s'etirer + // avec la perspective. + let plane_px = [s_px[0] * quad.scale, s_px[1] * quad.scale]; + let (min_x, max_x) = + corners.iter().fold((f32::MAX, f32::MIN), |(mn, mx), &(x, _)| (mn.min(x), mx.max(x))); + let (min_y, max_y) = + corners.iter().fold((f32::MAX, f32::MIN), |(mn, mx), &(_, y)| (mn.min(y), mx.max(y))); + let bbox_w = (max_x - min_x).max(1.0); + let bbox_h = (max_y - min_y).max(1.0); + // Coins en px LOCAUX a la bbox, pour matcher `i.local` du shader. + let local = |(x, y): (f32, f32)| -> [f32; 2] { [x - min_x, y - min_y] }; + let [tl0, tl1] = local(corners[0]); + let [tr0, tr1] = local(corners[1]); + let [br0, br1] = local(corners[2]); + let [bl0, bl1] = local(corners[3]); + LayerCB { + dst: [ + (center_px[0] + min_x) / rw, + (center_px[1] + min_y) / rh, + bbox_w / rw, + bbox_h / rh, + ], + src: cut, + quad_px: [bbox_w, bbox_h], + radius_px: radius * quad.scale, + mode: 8.0, + fx: [tl0, tl1, tr0, tr1], + src_prev: [br0, br1, bl0, bl1], + dst_prev: [plane_px[0], plane_px[1], 0.0, 0.0], + ..Default::default() + } + } + fn make_bind( &self, cb: &LayerCB, @@ -752,16 +844,34 @@ impl Compositor { None => (lp.bg_color, None), }; - // Calque ecran (mode 0 : NV12 -> RGB), place par plan_frame (cover-fit + - // coins arrondis). `src = g.cut` (crop utilisateur + zoom en UV texture). - let screen_layer = LayerCB { - dst: g.s_dst, - src: g.cut, - quad_px: [g.s_dst[2] * rw, g.s_dst[3] * rh], - radius_px: g.s_radius, - mode: 0.0, - color: [1.0, 1.0, 1.0, 1.0], - ..Default::default() + // ROTATION 3D (presets iso/left/right d'une zoom region). La geometrie du + // tilt est calculee UNE fois : l'ombre et l'ecran doivent porter exactement + // le meme quadrilatere, sinon l'ombre se decolle des que l'un des deux + // change. `regions` fait toute la trigo (partagee avec macOS/Windows) ; ici + // on ne fait que l'empaqueter. + let s_px = [g.s_dst[2] * rw, g.s_dst[3] * rh]; + let tilt = (!crate::regions::is_identity_rotation(g.zoom_rotation)) + .then(|| crate::regions::rotated_quad_corners_px(s_px[0], s_px[1], g.zoom_rotation)); + let quad_center_px = [ + (g.s_dst[0] + g.s_dst[2] * 0.5) * rw, + (g.s_dst[1] + g.s_dst[3] * 0.5) * rh, + ]; + + // Calque ecran : mode 0 (rect droit, NV12 -> RGB) quand la rotation est + // neutre, mode 8 (warp bilineaire inverse dans la bbox du quad projete) + // sinon. Place par plan_frame (cover-fit + coins arrondis) ; + // `src = g.cut` (crop utilisateur + zoom en UV texture) dans les deux cas. + let screen_layer = match tilt.as_ref() { + None => LayerCB { + dst: g.s_dst, + src: g.cut, + quad_px: s_px, + radius_px: g.s_radius, + mode: 0.0, + color: [1.0, 1.0, 1.0, 1.0], + ..Default::default() + }, + Some(quad) => self.tilted_screen_cb(quad, s_px, quad_center_px, g.cut, g.s_radius), }; // Bind group construit AVANT le pass (doit vivre pendant tout le pass) ; // `_screen_uniform` garde le buffer uniforme en vie (reference par le bind). @@ -769,23 +879,32 @@ impl Compositor { let (_screen_uniform, screen_bind) = self.make_bind(&screen_layer, Some((&sy, &suv)), &dummy); - // OMBRE PORTEE de l'ecran (mode 2), dessinee JUSTE AVANT le calque - // ecran. Le shader la connait depuis le debut ; ce qui manquait etait - // uniquement le draw cote Rust, si bien que le curseur « Ombre » de - // l'UI ne faisait rien sur Linux. + // OMBRE PORTEE de l'ecran, dessinee JUSTE AVANT le calque ecran. Le shader + // la connait depuis le debut ; ce qui manquait etait uniquement le draw + // cote Rust, si bien que le curseur « Ombre » de l'UI ne faisait rien sur + // Linux. // // Les fractions de reglage viennent de `frame_geometry`, partagees avec // macOS/Windows : l'ombre a la meme taille relative sur les trois // plateformes quelle que soit la resolution de sortie. + // + // L'ombre suit la silhouette REELLEMENT affichee : rect arrondi (mode 2) + // quand l'ecran est droit, quadrilatere projete (mode 12) quand il penche. let screen_shadow = cfg.shadow.then(|| { - let cb = self.shadow_cb( - g.s_dst, - [g.s_dst[2] * rw, g.s_dst[3] * rh], - g.s_radius, - crate::frame_geometry::SCREEN_SHADOW_SPREAD_FRAC * g.frame_min_px, - [0.0, crate::frame_geometry::SCREEN_SHADOW_OFFSET_FRAC * g.frame_min_px], - 0.45 * lp.shadow_scale, - ); + let spread = crate::frame_geometry::SCREEN_SHADOW_SPREAD_FRAC * g.frame_min_px; + let offset = [0.0, crate::frame_geometry::SCREEN_SHADOW_OFFSET_FRAC * g.frame_min_px]; + let opacity = 0.45 * lp.shadow_scale; + let cb = match tilt.as_ref() { + None => self.shadow_cb(g.s_dst, s_px, g.s_radius, spread, offset, opacity), + Some(quad) => self.quad_shadow_cb( + &quad.corners, + quad_center_px, + g.s_radius * quad.scale, + spread, + offset, + opacity, + ), + }; self.make_bind(&cb, None, &dummy) }); @@ -1023,10 +1142,11 @@ impl Compositor { } } - // Curseur thematise (mode 7, sprite RGBA). Placement UPRIGHT uniquement : - // le tilt (mode 13, sous zoom incline) et le motion blur (taps>1) sont - // des increments a venir. `_tex`/`_view`/`_buf` gardent le sprite en vie - // pendant le pass. Miroir de la branche curseur de `compositor_macos`. + // Curseur thematise : sprite RGBA droit (mode 7) ou pose sur le plan + // incline (mode 13) selon ce que `plan_cursor` a resolu. Le motion blur + // (taps>1) reste un increment a venir. `_tex`/`_view`/`_buf` gardent le + // sprite en vie pendant le pass. Miroir de la branche curseur de + // `compositor_macos`. struct CursorDraw { _buf: wgpu::Buffer, _tex: wgpu::Texture, @@ -1051,10 +1171,6 @@ impl Compositor { .unwrap_or(frame / crate::frame_geometry::FPS), }, )?; - let CursorPlacement::Upright { center } = plan.placement else { - // ponytail: tilt (mode 13) a porter quand un zoom incline sera teste. - return None; - }; let sprites = scene_ref .as_ref() .map(|s| s.cursor.cursor_sprites.clone()) @@ -1088,13 +1204,55 @@ impl Compositor { (plan.size_px * ar, plan.size_px) }; let hotspot = [sprite.hotspot_x, sprite.hotspot_y]; - let cb = LayerCB { - dst: cursor_sprite_dst(center, pw / rw, ph / rh, hotspot), - src: [0.0, 0.0, 1.0, 1.0], - mode: 7.0, - color: [1.0, 1.0, 1.0, 1.0], - fx: plan.clip, - ..Default::default() + let cb = match plan.placement { + CursorPlacement::Upright { center } => LayerCB { + dst: cursor_sprite_dst(center, pw / rw, ph / rh, hotspot), + src: [0.0, 0.0, 1.0, 1.0], + mode: 7.0, + color: [1.0, 1.0, 1.0, 1.0], + fx: plan.clip, + ..Default::default() + }, + CursorPlacement::Tilted { plane_pt, quad, center_px, screen_px, .. } => { + // Le sprite est pose DANS le plan : sa taille devient une fraction + // du plan et ses quatre coins traversent la meme projection que la + // video. La reduction due au tilt vient donc de la projection -- + // rien a multiplier a la main. + let (wf, hf) = (pw / screen_px[0], ph / screen_px[1]); + let x0 = plane_pt[0] - hotspot[0] * wf; + let y0 = plane_pt[1] - hotspot[1] * hf; + let corners = [(x0, y0), (x0 + wf, y0), (x0 + wf, y0 + hf), (x0, y0 + hf)] + .map(|(fx, fy)| { + let (px, py) = quad.point_px(fx, fy); + (center_px[0] + px, center_px[1] + py) + }); + let (min_x, max_x) = corners + .iter() + .fold((f32::MAX, f32::MIN), |(mn, mx), &(x, _)| (mn.min(x), mx.max(x))); + let (min_y, max_y) = corners + .iter() + .fold((f32::MAX, f32::MIN), |(mn, mx), &(_, y)| (mn.min(y), mx.max(y))); + // Le quad projete d'un sprite peut etre tres fin de biais : une bbox + // d'un pixel de large ferait diverger le warp inverse, d'ou le + // plancher a 1 px. + let (bw, bh) = ((max_x - min_x).max(1.0), (max_y - min_y).max(1.0)); + let local = |(x, y): (f32, f32)| [x - min_x, y - min_y]; + let [tl0, tl1] = local(corners[0]); + let [tr0, tr1] = local(corners[1]); + let [br0, br1] = local(corners[2]); + let [bl0, bl1] = local(corners[3]); + LayerCB { + dst: [min_x / rw, min_y / rh, bw / rw, bh / rh], + quad_px: [bw, bh], + mode: 13.0, + color: [1.0, 1.0, 1.0, 1.0], + fx: [tl0, tl1, tr0, tr1], + src_prev: [br0, br1, bl0, bl1], + // Le clip vit ici et NON dans `fx` (mode 7) : `fx` porte les coins. + dst_prev: plan.clip, + ..Default::default() + } + } }; let view = tex.create_view(&wgpu::TextureViewDescriptor::default()); // Sprite RGBA au binding 1 (texY) que le mode 7 echantillonne. diff --git a/crates/compositor/src/vk_shaders/layer.wgsl b/crates/compositor/src/vk_shaders/layer.wgsl index 987e31f68..225745aa2 100644 --- a/crates/compositor/src/vk_shaders/layer.wgsl +++ b/crates/compositor/src/vk_shaders/layer.wgsl @@ -22,12 +22,12 @@ struct Layer { src: vec4, // u0,v0,u1,v1 source 0..1 quad_px: vec2, // taille du quad en px de sortie (pour la SDF isotrope) radius_px: f32, - mode: f32, // 0 = vidéo NV12, 1 = couleur pleine, 2 = ombre + mode: f32, // 0 = vidéo NV12, 1 = couleur pleine, 2 = ombre, 8 = écran tilté, 12 = ombre du quad tilté, 13 = curseur tilté color: vec4, - fx: vec4, // mode 2 : spread ombre en px - src_prev: vec4, - dst_prev: vec4, - mb: vec4, + fx: vec4, // mode 2 : spread ombre en px ; modes 8/12/13 : coins TL,TR du quad projeté + src_prev: vec4, // modes 8/12/13 : coins BR,BL du quad projeté + dst_prev: vec4, // mode 8 : taille du plan en px AVANT projection (le rayon y vit) ; mode 13 : rect de clip + mb: vec4, // mode 12 : mb.y = spread de la pénombre en px } @group(0) @binding(0) var layer: Layer; @@ -79,6 +79,124 @@ fn sd_round_rect(p: vec2, halfsz: vec2, r: f32) -> f32 { return length(max(q, vec2(0.0))) + min(max(q.x, q.y), 0.0) - r; } +// ---- Primitives du tilt 3D (modes 8 et 12), portees de `shaders.metal` ---- + +// SDF segment a bouts ronds. +fn sd_segment(p: vec2, a: vec2, b: vec2) -> f32 { + let pa = p - a; + let ba = b - a; + let h = clamp(dot(pa, ba) / max(dot(ba, ba), 1e-6), 0.0, 1.0); + return length(pa - ba * h); +} + +// Intersection de deux droites donnees par (normale, offset) : n.x = d. Cramer. +fn line_cross(n1: vec2, d1: f32, n2: vec2, d2: f32) -> vec2 { + let det = n1.x * n2.y - n1.y * n2.x; + if abs(det) < 1e-6 { + return vec2(0.0, 0.0); + } + return vec2(d1 * n2.y - d2 * n1.y, d2 * n1.x - d1 * n2.x) / det; +} + +// Contribution d'une arete a la SDF du quad : .x = distance signee au demi-plan +// porte par l'arete (>0 dehors), .y = distance au SEGMENT. +fn quad_edge(p: vec2, a: vec2, b: vec2) -> vec2 { + let e = b - a; + // Division par la longueur plutot que `normalize` : une arete degeneree + // donnerait un NaN qui effacerait le calque entier. + let n = vec2(e.y, -e.x) / max(length(e), 1e-6); + return vec2(dot(p - a, n), sd_segment(p, a, b)); +} + +// Distance signee EXACTE a un quadrilatere convexe (<0 dedans). La boucle `k` +// du MSL est deroulee : elle indexait un tableau local avec un indice runtime, +// ce que naga 24 traduit en SPIR-V invalide (cf. `blur.wgsl`). +fn sd_convex_quad(p: vec2, v0: vec2, v1: vec2, v2: vec2, v3: vec2) -> f32 { + let e0 = quad_edge(p, v0, v1); + let e1 = quad_edge(p, v1, v2); + let e2 = quad_edge(p, v2, v3); + let e3 = quad_edge(p, v3, v0); + let inside = max(max(e0.x, e1.x), max(e2.x, e3.x)); + let border = min(min(e0.y, e1.y), min(e2.y, e3.y)); + if inside < 0.0 { + return -border; + } + return border; +} + +// Coin d'un quad rentre de `r` : intersection des deux aretes adjacentes, +// chacune decalee de `r` vers l'interieur. Meme deroulement que ci-dessus. +fn inset_corner(prev: vec2, cur: vec2, next: vec2, r: f32) -> vec2 { + let ep = cur - prev; + let ec = next - cur; + // TL->TR->BR->BL tourne dans le sens horaire en y-bas, donc (e.y, -e.x) sort du quad. + let np = vec2(ep.y, -ep.x) / max(length(ep), 1e-6); + let nc = vec2(ec.y, -ec.x) / max(length(ec), 1e-6); + return line_cross(np, dot(prev, np) - r, nc, dot(cur, nc) - r); +} + +// (s, t, ok) du warp inverse du mode 8 pour une racine `t` donnee. +fn quad_st_for_root(t: f32, e: vec2, f: vec2, g: vec2, h: vec2) -> vec3 { + let denom_x = e.x + g.x * t; + let denom_y = e.y + g.y * t; + var s: f32; + if abs(denom_x) > abs(denom_y) { + s = (h.x - f.x * t) / denom_x; + } else { + s = (h.y - f.y * t) / denom_y; + } + // Tolerance de 2 % reprise telle quelle du MSL : sans elle une rangee de + // pixels du bord tombe hors du quad par arrondi et l'ecran se liseree. + var ok = 0.0; + if s >= -0.02 && s <= 1.02 && t >= -0.02 && t <= 1.02 { + ok = 1.0; + } + return vec3(s, t, ok); +} + +// (s, t, ok) du point `P` dans le quad c00->c10->c11->c01 : le warp bilineaire INVERSE. +fn quad_inverse_bilinear(P: vec2, c00: vec2, c10: vec2, c11: vec2, c01: vec2) -> vec3 { + let e = c10 - c00; + let f = c01 - c00; + let g = c00 - c10 - c01 + c11; + let h = P - c00; + let k2 = g.x * f.y - g.y * f.x; + let k1 = e.x * f.y - e.y * f.x + h.x * g.y - h.y * g.x; + let k0 = h.x * e.y - h.y * e.x; + // Quad quasi affine (rotation Y pure, p.ex.) : le terme quadratique s'evanouit + // et resoudre la quadratique diviserait par ~0. + if abs(k2) < 1e-5 * abs(k1) { + var t = 0.0; + if abs(k1) >= 1e-6 { + t = -k0 / k1; + } + return quad_st_for_root(t, e, f, g, h); + } + let disc = k1 * k1 - 4.0 * k2 * k0; + if disc < 0.0 { + return vec3(0.0, 0.0, 0.0); + } + // Forme stable de la quadratique : additionner deux termes de meme signe evite + // l'annulation catastrophique que `(-k1 +- sqrt(disc)) / (2 k2)` produit quand + // `disc` approche `k1^2`. `sign()` de WGSL rend 0 en 0, la ou le ternaire MSL + // rend +1 : d'ou le signe explicite. + var sgn = 1.0; + if k1 < 0.0 { + sgn = -1.0; + } + let q = -0.5 * (k1 + sgn * sqrt(disc)); + let r0 = quad_st_for_root(q / k2, e, f, g, h); + var t1 = q / k2; + if abs(q) > 0.0 { + t1 = k0 / q; + } + let r1 = quad_st_for_root(t1, e, f, g, h); + if r0.z > 0.5 { + return r0; + } + return r1; +} + @fragment fn fs_main(i: VsOut) -> @location(0) vec4 { var rgb: vec3; @@ -122,6 +240,83 @@ fn fs_main(i: VsOut) -> @location(0) vec4 { // texY. `src` porte le rect UV cover-fit (calcule cote Rust). Opaque : // le fond couvre tout le cadre. return vec4(textureSample(texY, samp, i.uv).rgb, 1.0); + } else if layer.mode > 7.5 && layer.mode < 8.5 { + // Mode 8 -- ecran tilte (rotation 3D des zoom regions). Le quad projete est + // dessine dans sa BBOX (le VS ne sait tracer qu'un rect) et chaque fragment + // remonte au (s,t) du plan par warp bilineaire inverse. + // + // PAS de test de clip sur `dst_prev` : en mode 8 `dst_prev.xy` porte + // `plane_px`, la taille du plan en PIXELS (~1600), la ou `i.pout` vit dans + // [0,1]. Un clip la-dessus serait vrai partout et n'afficherait rien. + let r = quad_inverse_bilinear( + i.local, layer.fx.xy, layer.fx.zw, layer.src_prev.xy, layer.src_prev.zw, + ); + if r.z < 0.5 { + return vec4(0.0, 0.0, 0.0, 0.0); // hors du quad projete + } + // La coupe source s'applique ICI : `r` est une position DANS le plan (0..1), + // pas une coordonnee de texture. Echantillonner `r` directement ignorerait le + // crop utilisateur et le zoom. + let uv = vec2( + mix(layer.src.x, layer.src.z, clamp(r.x, 0.0, 1.0)), + mix(layer.src.y, layer.src.w, clamp(r.y, 0.0, 1.0)), + ); + // Coins arrondis DANS LE REPERE DU PLAN : le rayon reste constant le long du + // bord, la ou un arrondi calcule dans la bbox s'etirerait avec la perspective. + // Inconditionnel, rayon 0 compris -- `sd_round_rect` degenere en SDF de + // rectangle et le feather de 1,5 px subsiste, ce qui fait lire une arete + // inclinee COMME une arete plutot que comme un escalier. + let plane_px = layer.dst_prev.xy; + let p = vec2(r.x, r.y) * plane_px - plane_px * 0.5; + let d = sd_round_rect(p, plane_px * 0.5, max(layer.radius_px, 0.0)); + let tilt_a = 1.0 - smoothstep(0.0, 1.5, d); + // L'alpha est cette couverture, pas `color.a` : les draws du mode 8 laissent + // `color` a zero, donc s'en servir rendrait un plan totalement transparent. + return vec4(sample_yuv(uv) * tilt_a, tilt_a); + } else if layer.mode > 11.5 && layer.mode < 12.5 { + // Mode 12 -- ombre du quad projete. La penombre suit le QUADRILATERE, pas son + // rect englobant : un rect droit derriere un ecran incline se lit comme une + // seconde surface, pas comme son ombre. + // + // `fx`/`src_prev` portent les COINS (en px locaux a la bbox, comme `i.local`), + // et le spread vit dans `mb.y` -- pas dans `fx.x` comme au mode 2. + let tl = layer.fx.xy; + let tr = layer.fx.zw; + let br = layer.src_prev.xy; + let bl = layer.src_prev.zw; + // Coins arrondis du meme rayon que le plan : une ombre a coins vifs derriere un + // ecran arrondi depasse en pointe a chaque coin, d'autant plus que le rayon monte. + let r = max(layer.radius_px, 0.0); + let v0 = inset_corner(bl, tl, tr, r); + let v1 = inset_corner(tl, tr, br, r); + let v2 = inset_corner(tr, br, bl, r); + let v3 = inset_corner(br, bl, tl, r); + let d = sd_convex_quad(i.local, v0, v1, v2, v3) - r; + let spread = max(layer.mb.y, 1e-3); + let a = layer.color.a * (1.0 - smoothstep(0.0, spread, d)); + return vec4(layer.color.rgb * a, a); + } else if layer.mode > 12.5 && layer.mode < 13.5 { + // Mode 13 -- sprite de curseur POSE sur l'ecran incline : ses quatre coins + // ont traverse la meme projection que la video, et le fragment remonte a sa + // position dans le sprite par le meme warp inverse que le mode 8. + // + // Le rect de clip « Clip to canvas » est ici dans `dst_prev` (en sortie + // 0..1, [x,y,w,h]) et NON dans `fx` comme au mode 7 : `fx` porte les coins. + if i.pout.x < layer.dst_prev.x || i.pout.x > layer.dst_prev.x + layer.dst_prev.z + || i.pout.y < layer.dst_prev.y || i.pout.y > layer.dst_prev.y + layer.dst_prev.w { + return vec4(0.0, 0.0, 0.0, 0.0); + } + let r = quad_inverse_bilinear( + i.local, layer.fx.xy, layer.fx.zw, layer.src_prev.xy, layer.src_prev.zw, + ); + if r.z < 0.5 { + return vec4(0.0, 0.0, 0.0, 0.0); + } + // Sprite RGBA a alpha DROITE sur texY (comme le mode 7 y lie le sien) : + // on premultiplie ici. + let s = textureSample(texY, samp, clamp(vec2(r.x, r.y), vec2(0.0), vec2(1.0))); + let ca = s.a * layer.color.a; + return vec4(s.rgb * ca, ca); } else { // Mode 2 — ombre portée (SDF d'un quad arrondi élargi de `fx.x`). let spread = layer.fx.x; diff --git a/crates/compositor/tests/compose_linux.rs b/crates/compositor/tests/compose_linux.rs index 3a2bb9071..492cd084d 100644 --- a/crates/compositor/tests/compose_linux.rs +++ b/crates/compositor/tests/compose_linux.rs @@ -79,6 +79,290 @@ fn compose_linux_rend_une_frame() { ); } +// --------------------------------------------------------------------------- +// Rotation 3D (modes 8 et 12) +// --------------------------------------------------------------------------- + +/// Scene « ecran seul sur fond plat magenta », avec ou sans preset de rotation. +/// Le fond est une couleur SATUREE que l'enregistrement d'ecran de la fixture ne +/// produit nulle part : c'est ce qui permet de separer l'ecran du fond au pixel +/// pres, donc de mesurer la forme reellement dessinee. +fn tilt_scene_json(rotation: &str, shadow: u32, roundness: f32) -> String { + format!( + r##"{{"clips":[],"layout":{{"preset":"no-webcam","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}},"effects":{{"padding":0.2,"blur":false,"shadow":{shadow},"roundnessFrac":{roundness},"motionBlur":0}},"background":{{"kind":"color","color":"#ff00ff"}},"zoomRegions":[{{"clipIndex":0,"startSec":0,"endSec":6,"scale":1.0,"focusX":0.5,"focusY":0.5,"rotation":{rotation}}}],"annotations":[],"cursor":{{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}},"cropByClip":[],"output":{{"width":1920,"height":1080,"fps":30}}}}"## + ) +} + +/// `true` si le pixel n'est PAS le fond magenta. Seuil large : le feather des +/// bords et le degrade du sampler ne doivent pas compter comme du fond. +fn not_bg(px: &[u8]) -> bool { + !(px[0] > 200 && px[1] < 60 && px[2] > 200) +} + +/// Pour chaque colonne, la premiere ligne non-fond. `None` = colonne entierement +/// de fond. C'est la trace du BORD HAUT de ce qui est dessine : horizontale pour +/// un ecran droit, oblique pour un ecran incline. +fn top_edge(rgba: &[u8], w: u32, h: u32) -> Vec> { + (0..w) + .map(|x| { + (0..h).find(|&y| { + let i = ((y * w + x) * 4) as usize; + not_bg(&rgba[i..i + 4]) + }) + }) + .collect() +} + +/// Ecart max du bord haut, mesure sur les colonnes centrales uniquement : aux +/// deux extremites le bord haut d'un quad incline bascule sur le bord LATERAL, +/// ce qui ajouterait une variation qui n'est pas celle qu'on veut mesurer. +fn top_edge_swing(edge: &[Option]) -> u32 { + let n = edge.len(); + let seen: Vec = edge[n / 4..3 * n / 4].iter().flatten().copied().collect(); + match (seen.iter().min(), seen.iter().max()) { + (Some(&lo), Some(&hi)) => hi - lo, + _ => 0, + } +} + +fn write_ppm(name: &str, w: u32, h: u32, rgba: &[u8]) { + use std::io::Write; + let out = std::env::var("OPENSCREEN_VK_OUT").unwrap_or_else(|_| "target".into()); + let _ = std::fs::create_dir_all(&out); + let path = format!("{out}/{name}.ppm"); + let mut f = std::fs::File::create(&path).expect("create ppm"); + write!(f, "P6\n{w} {h}\n255\n").unwrap(); + let mut rgb = vec![0u8; (w * h * 3) as usize]; + for (d, s) in rgb.chunks_exact_mut(3).zip(rgba.chunks_exact(4)) { + d.copy_from_slice(&s[0..3]); + } + f.write_all(&rgb).unwrap(); + println!("wrote {path}"); +} + +/// Ecran incline (mode 8). Rend DEUX fois la meme scene, seule la rotation +/// change, et compare la silhouette obtenue. +/// +/// L'assertion porte sur la GEOMETRIE, pas sur la presence d'un fichier : le +/// bord haut de l'ecran droit est horizontal a moins de 2 px pres, celui de +/// l'ecran incline balaie des dizaines de lignes. Un mode 8 non branche cote +/// Rust, un warp inverse faux, ou un `quad_st_for_root` qui rejetterait tout +/// casse l'une des trois bornes. +#[test] +fn compose_linux_ecran_tilte() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("compose_linux tilt: opt-in (OPENSCREEN_LINUX_COMPOSE=1 + fixture). Skip."); + return; + } + + let gpu = Gpu::create(false).expect("Gpu::create"); + let comp = Compositor::new_sized(&gpu, W, H).expect("Compositor::new_sized"); + let mut dec = Decoder::open(FIXTURE, &gpu).expect("Decoder::open"); + + let mut cfg = Cfg::c8(); + cfg.shadow = false; + let (w, h, upright, tilted) = unsafe { + let sf = dec.seek_to(1.0).expect("Decoder::seek_to"); + // `frame` = 90 -> source_t = 3 s, au coeur de la region [0, 6] : la rampe + // d'entree est finie, la rotation est a pleine force. + let render = |json: String| { + let scene = Scene::from_json(&json).expect("scene json"); + // Le padding transite par les live_params, pas la scene brute. + comp.set_live_params(openscreen_compositor::compositor::live_params_from_scene(&scene)); + comp.set_scene(Some(scene)); + comp.compose_frame(sf, sf, 90.0, &cfg).expect("compose_frame"); + comp.readback_direct().expect("readback_direct") + }; + let (w, h, upright) = render(tilt_scene_json("null", 0, 0.0)); + let tilted: Vec<(&str, Vec)> = ["iso", "left", "right"] + .iter() + .map(|p| (*p, render(tilt_scene_json(&format!("\"{p}\""), 0, 0.0)).2)) + .collect(); + (w, h, upright, tilted) + }; + + write_ppm("compose_linux_tilt_upright", w, h, &upright); + + let up_swing = top_edge_swing(&top_edge(&upright, w, h)); + let up_area = upright.chunks_exact(4).filter(|p| not_bg(p)).count(); + println!("compose_linux tilt : {w}x{h} droit bord_haut={up_swing}px aire={up_area}"); + + // Garde-fou du detecteur lui-meme : si le fond magenta ne separait pas + // proprement l'ecran, le bord de la reference droite ne serait pas plat et + // toute la mesure serait du bruit. + assert!( + up_swing <= 2, + "reference droite : bord haut non horizontal ({up_swing} px) — le detecteur de fond derape" + ); + + // Les TROIS presets. Ils ne donnent pas le meme quadrilatere : iso penche le + // plus, left/right sont dominés par leur rotateY, donc leur quad approche le + // cas quasi affine que `quad_inverse_bilinear` traite par une branche a part. + for (preset, tilted) in &tilted { + write_ppm(&format!("compose_linux_tilt_{preset}"), w, h, tilted); + let swing = top_edge_swing(&top_edge(tilted, w, h)); + let area = tilted.chunks_exact(4).filter(|p| not_bg(p)).count(); + println!("compose_linux tilt {preset} : bord_haut={swing}px aire={area}"); + + // Chaque preset combine un rotateX et un rotateZ non nuls : sur une largeur + // d'ecran de ~600 px le bord haut ne peut pas rester horizontal. + assert!( + swing >= 15, + "{preset} : bord haut plat a {swing} px — mode 8 pas dessine (rect droit ?)" + ); + // Le containment reduit le plan pour qu'il tienne dans le rect d'origine : + // l'aire couverte baisse. La borne basse attrape le cas « mode 8 ne rend + // rien » (quad_inverse_bilinear qui rejette tout, alpha a zero...). + assert!( + area > up_area * 4 / 10 && area < up_area * 95 / 100, + "{preset} : aire {area} hors de (0.40, 0.95) x {up_area} — mode 8 vide ou inopérant" + ); + } +} + +/// Ombre du quad projete (mode 12). L'ombre doit suivre le QUADRILATERE : si +/// elle retombait sur le mode 2 (rect arrondi axis-aligned), sa bordure exterieure +/// serait horizontale en haut. On isole l'ombre en soustrayant le meme rendu sans +/// ombre, puis on mesure la pente de la bordure de la zone assombrie. +#[test] +fn compose_linux_ombre_du_quad_tilte() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("compose_linux ombre tiltee: opt-in. Skip."); + return; + } + + let gpu = Gpu::create(false).expect("Gpu::create"); + let comp = Compositor::new_sized(&gpu, W, H).expect("Compositor::new_sized"); + let mut dec = Decoder::open(FIXTURE, &gpu).expect("Decoder::open"); + + let (w, h, sans, avec) = unsafe { + let sf = dec.seek_to(1.0).expect("Decoder::seek_to"); + let render = |json: String, shadow: bool| { + let scene = Scene::from_json(&json).expect("scene json"); + comp.set_live_params(openscreen_compositor::compositor::live_params_from_scene(&scene)); + comp.set_scene(Some(scene)); + let mut cfg = Cfg::c8(); + cfg.shadow = shadow; + comp.compose_frame(sf, sf, 90.0, &cfg).expect("compose_frame"); + comp.readback_direct().expect("readback_direct") + }; + // Rayon non nul : c'est la seule facon d'exercer `inset_corner`/`line_cross` + // (le rentrant des coins de l'ombre) et l'arrondi en repere PLAN du mode 8. + let (w, h, sans) = render(tilt_scene_json("\"iso\"", 0, 0.04), false); + let (_, _, avec) = render(tilt_scene_json("\"iso\"", 1, 0.04), true); + (w, h, sans, avec) + }; + + write_ppm("compose_linux_tilt_shadow", w, h, &avec); + + // Masque de l'ombre : pixels du FOND assombris par le calque 12. On ignore + // l'ecran lui-meme (l'ombre passe dessous, il n'y change rien). + let mut mask = vec![false; (w * h) as usize]; + let mut count = 0usize; + for p in 0..(w * h) as usize { + let i = p * 4; + let dark = sans[i] as i32 - avec[i] as i32 > 12 && !not_bg(&sans[i..i + 4]); + mask[p] = dark; + count += dark as usize; + } + // Bordure HAUTE de la penombre, colonne par colonne. + let edge: Vec> = (0..w) + .map(|x| (0..h).find(|&y| mask[(y * w + x) as usize])) + .collect(); + let swing = top_edge_swing(&edge); + println!("compose_linux ombre tiltee : {count} px assombris, bordure haute swing={swing}px"); + + assert!(count > 3000, "ombre absente ({count} px assombris) — mode 12 pas dessine ?"); + // Un repli sur le mode 2 donnerait une bordure haute rigoureusement plate. + assert!( + swing >= 15, + "bordure haute de l'ombre plate a {swing} px — l'ombre est un rect droit, pas le quad projete" + ); +} + +/// Curseur pose sur l'ecran incline (mode 13). Le curseur est place HORS du +/// centre : c'est la que le plan incline le deplace vraiment. Au centre, la +/// position tiltee et la position droite coincident et le test ne prouverait rien. +/// +/// L'assertion est que le sprite BOUGE quand on incline. Un repli sur le +/// placement droit (mode 7) laisserait les deux barycentres au meme endroit ; un +/// mode 13 absent ferait disparaitre le curseur (compte a zero). +#[test] +fn compose_linux_curseur_sur_ecran_tilte() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("compose_linux curseur tilte: opt-in. Skip."); + return; + } + // Sprite vert 16x16 opaque (le meme que le test du mode 7). + const SPRITE: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAACXBIWXMAAAABAAAAAQBPJcTWAAAAGElEQVR4nGNk+MdAEmAhTfmohlENQ0kDAGoRATwbkCdPAAAAAElFTkSuQmCC"; + + let gpu = Gpu::create(false).expect("Gpu::create"); + let comp = Compositor::new_sized(&gpu, W, H).expect("Compositor::new_sized"); + let mut dec = Decoder::open(FIXTURE, &gpu).expect("Decoder::open"); + + let track_path = std::env::temp_dir().join("os_cursor_track_tilt.json"); + std::fs::write( + &track_path, + r#"{"samples":[{"timeMs":3000,"cx":0.22,"cy":0.24,"cursorType":"arrow"}]}"#, + ) + .expect("write track"); + let track = CursorTrack::load(track_path.to_str().unwrap(), 0.0, 6.0).expect("CursorTrack::load"); + comp.set_cursor(track); + comp.set_cursor_time(Some(3.0)); + + let scene_json = |rotation: &str| { + format!( + r##"{{"clips":[],"layout":{{"preset":"no-webcam","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}},"effects":{{"padding":0.2,"blur":false,"shadow":0,"roundnessFrac":0,"motionBlur":0}},"background":{{"kind":"color","color":"#ff00ff"}},"zoomRegions":[{{"clipIndex":0,"startSec":0,"endSec":6,"scale":1.0,"focusX":0.5,"focusY":0.5,"rotation":{rotation}}}],"annotations":[],"cursor":{{"show":true,"size":4,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default","cursorSprites":{{"arrow":{{"path":"{SPRITE}","hotspotX":0.5,"hotspotY":0.5}}}}}},"cropByClip":[],"output":{{"width":1920,"height":1080,"fps":30}}}}"## + ) + }; + + let (w, h, upright, tilted) = unsafe { + let sf = dec.seek_to(1.0).expect("Decoder::seek_to"); + let render = |json: String| { + let scene = Scene::from_json(&json).expect("scene json"); + comp.set_live_params(openscreen_compositor::compositor::live_params_from_scene(&scene)); + comp.set_scene(Some(scene)); + comp.compose_frame(sf, sf, 90.0, &Cfg::c8()).expect("compose_frame"); + comp.readback_direct().expect("readback_direct") + }; + let (w, h, upright) = render(scene_json("null")); + let (_, _, tilted) = render(scene_json("\"iso\"")); + (w, h, upright, tilted) + }; + + write_ppm("compose_linux_tilt_cursor", w, h, &tilted); + + // Barycentre des pixels verts du sprite. + let centroid = |rgba: &[u8]| -> (f32, f32, usize) { + let (mut sx, mut sy, mut n) = (0.0f32, 0.0f32, 0usize); + for y in 0..h { + for x in 0..w { + let i = ((y * w + x) * 4) as usize; + if rgba[i + 1] > 180 && rgba[i] < 120 && rgba[i + 2] < 120 { + sx += x as f32; + sy += y as f32; + n += 1; + } + } + } + (sx / n.max(1) as f32, sy / n.max(1) as f32, n) + }; + let (ux, uy, un) = centroid(&upright); + let (tx, ty, tn) = centroid(&tilted); + let shift = ((tx - ux).powi(2) + (ty - uy).powi(2)).sqrt(); + println!( + "compose_linux curseur tilte : droit=({ux:.1},{uy:.1}) n={un} \ + incline=({tx:.1},{ty:.1}) n={tn} deplacement={shift:.1}px" + ); + + assert!(un > 50, "curseur droit absent (n={un}) — la scene de reference est cassee"); + assert!(tn > 50, "curseur absent sous rotation (n={tn}) — mode 13 pas dessine"); + assert!( + shift >= 12.0, + "curseur deplace de {shift:.1}px seulement — il est reste sur le rect droit (mode 7 ?)" + ); +} + /// Curseur : sprite thematise (mode 7) dessine au centre. Sprite VERT (data URI /// PNG) distinct du fond sombre et de l'ecran, pour l'affirmer sans ambiguite. #[test] From ac79c9082508afe54f6526282b222195fa49e110 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Fri, 31 Jul 2026 18:27:52 +0200 Subject: [PATCH 35/46] wip: motion blur (agent output, pre-merge) --- crates/compositor/src/compositor_linux.rs | 372 ++++++++++++++++---- crates/compositor/src/vk_shaders/blur.wgsl | 35 ++ crates/compositor/src/vk_shaders/layer.wgsl | 36 +- crates/compositor/tests/compose_linux.rs | 311 +++++++++++++++- 4 files changed, 664 insertions(+), 90 deletions(-) diff --git a/crates/compositor/src/compositor_linux.rs b/crates/compositor/src/compositor_linux.rs index 18109f19f..a2aff4474 100644 --- a/crates/compositor/src/compositor_linux.rs +++ b/crates/compositor/src/compositor_linux.rs @@ -109,6 +109,13 @@ pub struct Compositor { // Pipeline de calque (VS + FS `layer.wgsl`), sampler lineaire, bind group // layout (uniform + 2 textures + sampler). Immuables apres `new_sized`. pipeline: wgpu::RenderPipeline, + /// Meme shader et meme layout que `pipeline`, blend ADDITIF pondere par la + /// constante de blend. Sert a sommer les copies de la trainee du curseur + /// dans `accum` ; cf. `blend_add` cote Windows. + pipeline_add: wgpu::RenderPipeline, + /// Copie plein ecran d'`accum` vers le RT en « over » premultiplie + /// (`blur.wgsl` : `vs_fullscreen` + `fs_copy`). Utilise le layout du blur. + pipeline_copy: wgpu::RenderPipeline, bind_group_layout: wgpu::BindGroupLayout, sampler: wgpu::Sampler, @@ -125,6 +132,10 @@ pub struct Compositor { // Render target offscreen + ring de staging de la relecture (recrees au resize). rt: wgpu::Texture, rt_view: wgpu::TextureView, + /// Cible ISOLEE d'accumulation, meme taille et meme format que le RT. + /// `_accum` garde la texture en vie ; seule la vue est utilisee. + _accum: wgpu::Texture, + accum_view: wgpu::TextureView, /// `bytes_per_row` padde a 256 (contrainte wgpu de copy_texture_to_buffer). readback_bpr: u32, /// Ring de buffers de staging (cf. `ReadbackRing`). `RefCell` : les methodes @@ -215,34 +226,53 @@ impl Compositor { bind_group_layouts: &[&bind_group_layout], push_constant_ranges: &[], }); - let pipeline = gpu.device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { - label: Some("layer"), - layout: Some(&pipeline_layout), - vertex: wgpu::VertexState { - module: &module, - entry_point: Some("vs_main"), - compilation_options: wgpu::PipelineCompilationOptions::default(), - buffers: &[], - }, - fragment: Some(wgpu::FragmentState { - module: &module, - entry_point: Some("fs_main"), - compilation_options: wgpu::PipelineCompilationOptions::default(), - targets: &[Some(wgpu::ColorTargetState { - format: wgpu::TextureFormat::Rgba8Unorm, - blend: Some(wgpu::BlendState::PREMULTIPLIED_ALPHA_BLENDING), - write_mask: wgpu::ColorWrites::ALL, - })], - }), - primitive: wgpu::PrimitiveState { - topology: wgpu::PrimitiveTopology::TriangleStrip, - ..Default::default() - }, - depth_stencil: None, - multisample: wgpu::MultisampleState::default(), - multiview: None, - cache: None, - }); + // Deux pipelines pour le MEME shader de calque : seul le blend change. + let mk_layer = |label: &str, blend: wgpu::BlendState| { + gpu.device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some(label), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &module, + entry_point: Some("vs_main"), + compilation_options: wgpu::PipelineCompilationOptions::default(), + buffers: &[], + }, + fragment: Some(wgpu::FragmentState { + module: &module, + entry_point: Some("fs_main"), + compilation_options: wgpu::PipelineCompilationOptions::default(), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Rgba8Unorm, + blend: Some(blend), + write_mask: wgpu::ColorWrites::ALL, + })], + }), + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleStrip, + ..Default::default() + }, + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview: None, + cache: None, + }) + }; + let pipeline = mk_layer("layer", wgpu::BlendState::PREMULTIPLIED_ALPHA_BLENDING); + // SOMME pondere : `src * constante + dst`. La constante (posee par pass + // via `set_blend_constant`) vaut 1/taps, donc N copies d'un curseur + // parfaitement immobile redonnent exactement ce curseur. Transcription + // du `blend_add` D3D11 (BLEND_FACTOR / ONE / OP_ADD sur couleur ET + // alpha) ; l'alpha doit suivre la couleur, sinon la somme n'est plus + // premultipliee et la composition finale delave la trainee. + let add = wgpu::BlendComponent { + src_factor: wgpu::BlendFactor::Constant, + dst_factor: wgpu::BlendFactor::One, + operation: wgpu::BlendOperation::Add, + }; + let pipeline_add = mk_layer( + "layer-add", + wgpu::BlendState { color: add, alpha: add }, + ); // --- Chaine de blur Kawase du fond (`blur.wgsl`) --- let blur_module = gpu.device.create_shader_module(wgpu::ShaderModuleDescriptor { @@ -318,6 +348,38 @@ impl Compositor { }; let blur_down = mk_blur("fs_kawase_down"); let blur_up = mk_blur("fs_kawase_up"); + // Composition d'`accum` sur le RT : meme layout que le blur (uniform + + // 1 texture + sampler) et blend « over » premultiplie. Son VS est + // `vs_fullscreen` et non le `vs_main` du Kawase -- une passe UNIQUE ne + // pardonne pas une erreur d'orientation, cf. le commentaire la-bas. + let pipeline_copy = gpu.device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("accum-copy"), + layout: Some(&blur_pl), + vertex: wgpu::VertexState { + module: &blur_module, + entry_point: Some("vs_fullscreen"), + compilation_options: wgpu::PipelineCompilationOptions::default(), + buffers: &[], + }, + fragment: Some(wgpu::FragmentState { + module: &blur_module, + entry_point: Some("fs_copy"), + compilation_options: wgpu::PipelineCompilationOptions::default(), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Rgba8Unorm, + blend: Some(wgpu::BlendState::PREMULTIPLIED_ALPHA_BLENDING), + write_mask: wgpu::ColorWrites::ALL, + })], + }), + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleList, + ..Default::default() + }, + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview: None, + cache: None, + }); let mk_pyr = |dw: u32, dh: u32, label: &str| { gpu.device .create_texture(&wgpu::TextureDescriptor { @@ -341,7 +403,7 @@ impl Compositor { let blur_qtr = mk_pyr(w / 4, h / 4, "blur-qtr"); let blur_oct = mk_pyr(w / 8, h / 8, "blur-oct"); - let (rt, rt_view, readback_bpr) = Self::make_targets(&gpu, w, h); + let (rt, rt_view, accum, accum_view, readback_bpr) = Self::make_targets(&gpu, w, h); // Profondeur 1 par defaut = chemin synchrone historique, a l'octet et a // la latence pres. C'est l'export qui demande explicitement 2 (cf. // `set_readback_depth`) ; tout autre appelant garde l'ancien contrat. @@ -356,6 +418,8 @@ impl Compositor { render_w: w, render_h: h, pipeline, + pipeline_add, + pipeline_copy, bind_group_layout, sampler, blur_bgl, @@ -366,6 +430,8 @@ impl Compositor { blur_oct, rt, rt_view, + _accum: accum, + accum_view, readback_bpr, readback, live_params: RefCell::new(LiveParams::default()), @@ -378,27 +444,42 @@ impl Compositor { }) } - /// RT RGBA8 + `bytes_per_row` de la relecture (padde a 256). - fn make_targets(gpu: &Gpu, w: u32, h: u32) -> (wgpu::Texture, wgpu::TextureView, u32) { - let rt = gpu.device.create_texture(&wgpu::TextureDescriptor { - label: Some("rt"), - size: wgpu::Extent3d { - width: w, - height: h, - depth_or_array_layers: 1, - }, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: wgpu::TextureFormat::Rgba8Unorm, - usage: wgpu::TextureUsages::RENDER_ATTACHMENT - | wgpu::TextureUsages::COPY_SRC - | wgpu::TextureUsages::TEXTURE_BINDING, - view_formats: &[], - }); + /// RT RGBA8, cible d'accumulation de meme geometrie, et `bytes_per_row` de + /// la relecture (padde a 256). + /// + /// `accum` est alloue ICI et pas ailleurs pour qu'il soit impossible de le + /// laisser a l'ancienne taille apres un changement de resolution : c'est le + /// meme appel qui produit les deux, et un accum plus petit que le RT ferait + /// une passe de composition tronquee. + fn make_targets( + gpu: &Gpu, + w: u32, + h: u32, + ) -> (wgpu::Texture, wgpu::TextureView, wgpu::Texture, wgpu::TextureView, u32) { + let mk = |label: &str, extra: wgpu::TextureUsages| { + gpu.device.create_texture(&wgpu::TextureDescriptor { + label: Some(label), + size: wgpu::Extent3d { + width: w, + height: h, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT + | wgpu::TextureUsages::TEXTURE_BINDING + | extra, + view_formats: &[], + }) + }; + let rt = mk("rt", wgpu::TextureUsages::COPY_SRC); + let accum = mk("accum", wgpu::TextureUsages::empty()); let rt_view = rt.create_view(&wgpu::TextureViewDescriptor::default()); + let accum_view = accum.create_view(&wgpu::TextureViewDescriptor::default()); let bpr = (w * 4).div_ceil(256) * 256; - (rt, rt_view, bpr) + (rt, rt_view, accum, accum_view, bpr) } /// Un buffer de staging de la ring. La taille depend de `bpr` (donc de la @@ -861,6 +942,21 @@ impl Compositor { // neutre, mode 8 (warp bilineaire inverse dans la bbox du quad projete) // sinon. Place par plan_frame (cover-fit + coins arrondis) ; // `src = g.cut` (crop utilisateur + zoom en UV texture) dans les deux cas. + // + // FLOU DE VELOCITE, ET POURQUOI SEULEMENT SUR LE MODE 0. `src_prev`/ + // `dst_prev` decrivent le MEME calque a la frame precedente ; le shader + // remappe chaque pixel de sortie par ce couple pour retrouver l'UV qu'il + // occupait alors, et floute le long du segment. `src_prev = g.cut` et non + // un `cut` d'avant : la coupe est identique aux deux frames (`plan_frame` + // ne fait varier que le rect de DESTINATION entre `s_dst` et + // `s_dst_prev`), ce que Windows documente aussi. Le mouvement vient donc + // entierement de `dst_prev`. + // + // Le mode 8 n'en recoit PAS, et ce n'est pas un oubli : ces deux champs y + // portent deja les coins projetes du quad (BR/BL dans `src_prev`, + // `plane_px` dans `dst_prev`). Les deux sens ne peuvent pas cohabiter dans + // un meme draw. macOS et Windows sautent egalement le flou sur le chemin + // incline, pour la meme raison. let screen_layer = match tilt.as_ref() { None => LayerCB { dst: g.s_dst, @@ -869,6 +965,9 @@ impl Compositor { radius_px: g.s_radius, mode: 0.0, color: [1.0, 1.0, 1.0, 1.0], + src_prev: g.cut, + dst_prev: g.s_dst_prev, + mb: [g.mb_taps, 1.0, 1.0, 0.0], ..Default::default() }, Some(quad) => self.tilted_screen_cb(quad, s_px, quad_center_px, g.cut, g.s_radius), @@ -990,6 +1089,11 @@ impl Compositor { // cover-crop les deux bornes sont strictement a l'interieur de la // texture, donc le sampler ClampToEdge ne bave pas sur les bords. let (u0, u1) = if lp.webcam_mirror { (cu1, cu0) } else { (cu0, cu1) }; + // `src_prev` doit valoir EXACTEMENT le `src` de ce draw, miroir + // compris : le shader s'en sert pour reconstruire l'UV de la frame + // precedente, et un rect source qui ne correspond pas au calque + // dessine ferait diverger la trainee vers une zone de la texture qui + // n'a jamais ete affichee. Seul `dst_prev` porte le mouvement. let cb = LayerCB { dst: g.w_dst, src: [u0, cv0, u1, cv1], @@ -997,6 +1101,9 @@ impl Compositor { radius_px: g.w_radius, mode: 0.0, color: [0.0, 0.0, 0.0, 1.0], + src_prev: [u0, cv0, u1, cv1], + dst_prev: g.w_dst_prev, + mb: [g.mb_taps, 1.0, 1.0, 0.0], ..Default::default() }; self.make_bind(&cb, Some((wy, wuv)), &dummy) @@ -1143,15 +1250,19 @@ impl Compositor { } // Curseur thematise : sprite RGBA droit (mode 7) ou pose sur le plan - // incline (mode 13) selon ce que `plan_cursor` a resolu. Le motion blur - // (taps>1) reste un increment a venir. `_tex`/`_view`/`_buf` gardent le - // sprite en vie pendant le pass. Miroir de la branche curseur de - // `compositor_macos`. + // incline (mode 13) selon ce que `plan_cursor` a resolu. + // `_tex`/`_view`/`_bufs` gardent le sprite et les uniformes en vie + // pendant le pass. Miroir de la branche curseur de `compositor_macos`. + // + // TRAINEE (`plan.taps > 1`) : `binds` porte une copie par echantillon, + // interpolee entre `prev_placement` et le placement courant. Elles ne + // sont PAS dessinees sur le RT mais dans `accum`, puis compositees en une + // fois -- cf. le commentaire au point de dessin. struct CursorDraw { - _buf: wgpu::Buffer, + _bufs: Vec, _tex: wgpu::Texture, _view: wgpu::TextureView, - bind: wgpu::BindGroup, + binds: Vec, } let cursor_draw: Option = (|| { let track = cursor_ref.as_ref()?; @@ -1171,6 +1282,7 @@ impl Compositor { .unwrap_or(frame / crate::frame_geometry::FPS), }, )?; + let sprites = scene_ref .as_ref() .map(|s| s.cursor.cursor_sprites.clone()) @@ -1204,15 +1316,39 @@ impl Compositor { (plan.size_px * ar, plan.size_px) }; let hotspot = [sprite.hotspot_x, sprite.hotspot_y]; - let cb = match plan.placement { - CursorPlacement::Upright { center } => LayerCB { - dst: cursor_sprite_dst(center, pw / rw, ph / rh, hotspot), - src: [0.0, 0.0, 1.0, 1.0], - mode: 7.0, - color: [1.0, 1.0, 1.0, 1.0], - fx: plan.clip, - ..Default::default() - }, + // `taps == 1` : un seul placement, celui de l'instant rendu -- le + // chemin net d'avant, inchange. Au-dela, on echelonne les copies + // regulierement de `prev_placement` (inclus) au placement courant + // (inclus) : c'est ce que font les deux autres backends, et inclure + // les deux bornes est ce qui fait que la trainee touche a la fois + // l'endroit d'ou le curseur vient et celui ou il est. + // + // On interpole des PLACEMENTS et non des centres : `lerp` sait + // traiter le cas incline, si bien qu'une trainee sous zoom incline + // reste dans le plan au lieu de repasser par un centre 2D qui + // l'aplatirait. + let placements: Vec = if plan.taps <= 1 { + vec![plan.placement] + } else { + (0..plan.taps) + .map(|k| { + let f = k as f32 / (plan.taps - 1) as f32; + plan.prev_placement.lerp(plan.placement, f) + }) + .collect() + }; + let view = tex.create_view(&wgpu::TextureViewDescriptor::default()); + let (mut bufs, mut binds) = (Vec::new(), Vec::new()); + for placement in placements { + let cb = match placement { + CursorPlacement::Upright { center } => LayerCB { + dst: cursor_sprite_dst(center, pw / rw, ph / rh, hotspot), + src: [0.0, 0.0, 1.0, 1.0], + mode: 7.0, + color: [1.0, 1.0, 1.0, 1.0], + fx: plan.clip, + ..Default::default() + }, CursorPlacement::Tilted { plane_pt, quad, center_px, screen_px, .. } => { // Le sprite est pose DANS le plan : sa taille devient une fraction // du plan et ses quatre coins traversent la meme projection que la @@ -1253,12 +1389,48 @@ impl Compositor { ..Default::default() } } - }; - let view = tex.create_view(&wgpu::TextureViewDescriptor::default()); - // Sprite RGBA au binding 1 (texY) que le mode 7 echantillonne. - let (buf, bind) = self.make_bind(&cb, Some((&view, &view)), &dummy); - Some(CursorDraw { _buf: buf, _tex: tex, _view: view, bind }) + }; + // Sprite RGBA au binding 1 (texY) que le mode 7 echantillonne. + let (buf, bind) = self.make_bind(&cb, Some((&view, &view)), &dummy); + bufs.push(buf); + binds.push(bind); + } + Some(CursorDraw { _bufs: bufs, _tex: tex, _view: view, binds }) })(); + // Bind group de la passe de composition d'`accum` (layout du blur : + // uniform + texture + sampler). Construit hors de la pass, comme les + // autres. L'uniforme n'est pas lu par `fs_copy` mais le layout l'exige. + let accum_bind = cursor_draw + .as_ref() + .filter(|c| c.binds.len() > 1) + .map(|_| { + let cb = LayerCB::default(); + let uniform = + self.gpu.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("accum-copy-uniform"), + contents: layer_bytes(&cb), + usage: wgpu::BufferUsages::UNIFORM, + }); + let bind = self.gpu.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("accum-copy"), + layout: &self.blur_bgl, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: uniform.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::TextureView(&self.accum_view), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: wgpu::BindingResource::Sampler(&self.sampler), + }, + ], + }); + (uniform, bind) + }); let mut encoder = self.gpu.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("compose"), @@ -1334,11 +1506,71 @@ impl Compositor { rpass.draw(0..4, 0..1); } // Curseur en dernier : au-dessus de l'ecran et des annotations. - if let Some(c) = &cursor_draw { - rpass.set_bind_group(0, &c.bind, &[]); + // Une seule copie = curseur net, il tient dans cette pass. La + // trainee, elle, a besoin de sa propre cible (voir plus bas). + if let Some(c) = cursor_draw.as_ref().filter(|c| c.binds.len() == 1) { + rpass.set_bind_group(0, &c.binds[0], &[]); rpass.draw(0..4, 0..1); } } + // TRAINEE DU CURSEUR : flou REEL, pas des copies discretes. + // + // Les N echantillons s'accumulent dans une cible ISOLEE partie de zero, + // puis sont compositees « over » sur la scene. Les additionner + // directement sur le RT reviendrait a AJOUTER la couleur du curseur + // (souvent du blanc) a ce qui est deja dessous : sur un fond clair, deja + // proche du blanc, ajouter du blanc*(1/taps) ne change presque rien -- + // curseur quasi invisible. Dans une cible a part la somme reste + // correctement normalisee (alpha ~1 la ou les copies se recouvrent), et + // la composition finale est un « over » ordinaire, correct quel que soit + // le fond. Meme raisonnement, mot pour mot, cote macOS et Windows. + if let (Some(c), Some((_ubuf, abind))) = ( + cursor_draw.as_ref().filter(|c| c.binds.len() > 1), + accum_bind.as_ref(), + ) { + { + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("cursor-accum-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.accum_view, + resolve_target: None, + ops: wgpu::Operations { + // Le clear EST la raison d'etre de cette cible : elle + // doit partir vide a chaque frame, pas cumuler. + load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + rpass.set_pipeline(&self.pipeline_add); + let w = 1.0 / c.binds.len() as f64; + rpass.set_blend_constant(wgpu::Color { r: w, g: w, b: w, a: w }); + for bind in &c.binds { + rpass.set_bind_group(0, bind, &[]); + rpass.draw(0..4, 0..1); + } + } + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("cursor-accum-composite"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.rt_view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Load, + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + rpass.set_pipeline(&self.pipeline_copy); + rpass.set_bind_group(0, abind, &[]); + rpass.draw(0..3, 0..1); + } self.gpu.context.submit(std::iter::once(encoder.finish())); Ok(()) } diff --git a/crates/compositor/src/vk_shaders/blur.wgsl b/crates/compositor/src/vk_shaders/blur.wgsl index 9b2b756f8..c5f8f88e6 100644 --- a/crates/compositor/src/vk_shaders/blur.wgsl +++ b/crates/compositor/src/vk_shaders/blur.wgsl @@ -64,6 +64,41 @@ fn vs_main(@builtin(vertex_index) vid: u32) -> VsOut { return o; } +// Triangle plein écran pour une COPIE 1:1, séparé de `vs_main` à dessein. +// +// Les deux mappings sont aujourd'hui identiques, et c'est précisément le piège : +// `vs_main` appartient à la chaîne Kawase, qui enchaîne SIX passes. Une +// inversion en Y y serait invisible (six inversions se compensent), donc rien +// dans cette chaîne ne défend l'orientation. Une copie unique, elle, la porte +// entière. Dupliquer les quatre lignes coûte moins qu'une traînée de curseur +// retournée le jour où quelqu'un ajuste la convention du Kawase. +// +// Orientation : en NDC wgpu (calqué sur D3D/Metal) y=+1 est le HAUT de la cible +// et v=0 la PREMIÈRE ligne de la texture, donc v doit croître quand y décroît. +// Vérifié par `compose_linux_trainee_de_curseur`, qui échoue en trouvant la +// traînée dans la bande miroir si on écrit `0.5 + p.y * 0.5`. +@vertex +fn vs_fullscreen(@builtin(vertex_index) vid: u32) -> VsOut { + let pos = array, 3>( + vec2(-1.0, -1.0), + vec2( 3.0, -1.0), + vec2(-1.0, 3.0), + ); + // Un seul accès indexé, cf. la note naga 24 / RADV dans `vs_main`. + let p = pos[vid]; + var o: VsOut; + o.pos = vec4(p, 0.0, 1.0); + o.uv = vec2(p.x * 0.5 + 0.5, 0.5 - p.y * 0.5); + return o; +} + +// Copie telle quelle. La source est en alpha PRÉMULTIPLIÉ (tout le compositeur +// l'est), donc le blend « over » de la pipeline la composite sans reconversion. +@fragment +fn fs_copy(i: VsOut) -> @location(0) vec4 { + return textureSample(tex, samp, i.uv); +} + // Kawase down : 5-tap linéaire à offset `texel_offset` en coords source. // `texel_offset` est 2.2 typiquement (le spread mesuré du filtre). @fragment diff --git a/crates/compositor/src/vk_shaders/layer.wgsl b/crates/compositor/src/vk_shaders/layer.wgsl index 225745aa2..c60275356 100644 --- a/crates/compositor/src/vk_shaders/layer.wgsl +++ b/crates/compositor/src/vk_shaders/layer.wgsl @@ -203,11 +203,37 @@ fn fs_main(i: VsOut) -> @location(0) vec4 { var alpha: f32; if layer.mode < 0.5 { - // Mode 0 — vidéo NV12. Le motion blur (taps) et le flou de mouvement par - // vélocité sont délibérément omis de la tranche verticale (ils vivent en - // WP4, après la validation du chemin iso). Couleur échantillonnée une fois - // à l'UV interpolée. - rgb = sample_yuv(i.uv); + // Mode 0 — vidéo NV12 + flou de mouvement par vélocité (§8), port 1:1 du + // HLSL/MSL. Pour CE pixel de sortie, l'UV qu'il occupait à la frame + // précédente se retrouve en le remappant par (dst_prev, src_prev) : on + // floute le long de ce segment, ce qui capture la translation ET le zoom + // du calque sans avoir à transporter un champ de vitesse. + let taps = i32(layer.mb.x); + // `taps` d'abord : un draw qui a oublié `dst_prev` le laisse à zéro, et + // la division par `dst_prev.zw` produirait des UV infinis. Dégrader vers + // le chemin net est le seul échec acceptable pour un effet cosmétique. + if taps <= 1 || layer.dst_prev.z <= 0.0 || layer.dst_prev.w <= 0.0 { + rgb = sample_yuv(i.uv); + } else { + let localp = (i.pout - layer.dst_prev.xy) / layer.dst_prev.zw; + let uv_prev = layer.src_prev.xy + localp * (layer.src_prev.zw - layer.src_prev.xy); + let duv = i.uv - uv_prev; + if dot(duv, duv) < 1e-9 { + rgb = sample_yuv(i.uv); + } else { + // Borne 16 en dur, identique au HLSL et au MSL : `taps` vient d'un + // uniform et une boucle sans borne statique ne se déroule pas. + // L'échelle de l'inspector s'arrête pile à 16 (1 + 15·blur), donc + // c'est `taps` qui coupe, jamais la borne. + var acc = vec3(0.0); + let step = 1.0 / f32(taps - 1); + for (var k: i32 = 0; k < 16; k = k + 1) { + if k >= taps { break; } + acc = acc + sample_yuv(uv_prev + duv * (f32(k) * step)); + } + rgb = acc / f32(taps); + } + } } else if layer.mode < 1.5 { // Mode 1 — couleur pleine. rgb = layer.color.rgb; diff --git a/crates/compositor/tests/compose_linux.rs b/crates/compositor/tests/compose_linux.rs index 492cd084d..9e86aa1d5 100644 --- a/crates/compositor/tests/compose_linux.rs +++ b/crates/compositor/tests/compose_linux.rs @@ -23,6 +23,22 @@ const FIXTURE: &str = "../fixture/screen.mp4"; const W: u32 = 960; const H: u32 = 540; +/// Ecrit un PPM P6 dans `OPENSCREEN_VK_OUT` (defaut `target`) pour inspection. +fn write_ppm(name: &str, w: u32, h: u32, rgba: &[u8]) { + use std::io::Write; + let out = std::env::var("OPENSCREEN_VK_OUT").unwrap_or_else(|_| "target".into()); + let _ = std::fs::create_dir_all(&out); + let path = format!("{out}/{name}.ppm"); + let mut f = std::fs::File::create(&path).expect("create ppm"); + write!(f, "P6\n{w} {h}\n255\n").unwrap(); + let mut rgb = vec![0u8; (w * h * 3) as usize]; + for (d, s) in rgb.chunks_exact_mut(3).zip(rgba.chunks_exact(4)) { + d.copy_from_slice(&s[0..3]); + } + f.write_all(&rgb).unwrap(); + println!("wrote {path}"); +} + #[test] fn compose_linux_rend_une_frame() { if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { @@ -125,21 +141,6 @@ fn top_edge_swing(edge: &[Option]) -> u32 { } } -fn write_ppm(name: &str, w: u32, h: u32, rgba: &[u8]) { - use std::io::Write; - let out = std::env::var("OPENSCREEN_VK_OUT").unwrap_or_else(|_| "target".into()); - let _ = std::fs::create_dir_all(&out); - let path = format!("{out}/{name}.ppm"); - let mut f = std::fs::File::create(&path).expect("create ppm"); - write!(f, "P6\n{w} {h}\n255\n").unwrap(); - let mut rgb = vec![0u8; (w * h * 3) as usize]; - for (d, s) in rgb.chunks_exact_mut(3).zip(rgba.chunks_exact(4)) { - d.copy_from_slice(&s[0..3]); - } - f.write_all(&rgb).unwrap(); - println!("wrote {path}"); -} - /// Ecran incline (mode 8). Rend DEUX fois la meme scene, seule la rotation /// change, et compare la silhouette obtenue. /// @@ -480,6 +481,286 @@ fn compose_linux_fond_image() { assert!(orange > 2000, "fond image absent (orange={orange}) — mode 6 ?"); } +/// Flou de mouvement par VELOCITE du calque ecran (mode 0 du shader). +/// +/// La velocite vient d'un zoom en pleine rampe : `plan_frame` calcule alors un +/// `s_dst_prev` different de `s_dst`, et le shader floute chaque pixel le long +/// du segment qui relie son UV d'avant a son UV d'aujourd'hui. +/// +/// La MEME frame decodee est composee deux fois, seul `effects.motionBlur` +/// change — toute difference mesuree ne peut donc venir que de l'effet. Deux +/// assertions, parce que « les deux images different » ne dirait pas dans quel +/// SENS : on verifie aussi que la version floutee a moins de detail haute +/// frequence. Un cablage errone de `src_prev`/`dst_prev` ferait bien differer +/// les images, mais pas forcement dans ce sens-la. +#[test] +fn compose_linux_flou_de_velocite_ecran() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("compose_linux flou de velocite: opt-in. Skip."); + return; + } + + let gpu = Gpu::create(false).expect("Gpu::create"); + let comp = Compositor::new_sized(&gpu, W, H).expect("Compositor::new_sized"); + let mut dec = Decoder::open(FIXTURE, &gpu).expect("Decoder::open"); + + // La region de zoom demarre a 2 s ; sa rampe d'entree commence ~1 s plus tot + // (`ZOOM_IN_TRANSITION_WINDOW_S`). A t = 66/60 = 1,1 s on est donc en pleine + // montee : `plan_frame` y donne s_dst 1,629 contre s_dst_prev 1,559, soit + // 4,5 % d'echelle en une frame — largement de quoi etaler le calque. + let scene_of = |mblur: f32| { + format!( + r##"{{"clips":[],"layout":{{"preset":"no-webcam","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}},"effects":{{"padding":0,"blur":false,"shadow":0,"roundnessFrac":0,"motionBlur":{mblur}}},"background":{{"kind":"color","color":"#101015"}},"zoomRegions":[{{"id":"z1","startSec":2,"endSec":4,"scale":2.5,"focusX":0.5,"focusY":0.5,"focusMode":"manual","rotation":null}}],"annotations":[],"cursor":{{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}},"cropByClip":[],"output":{{"width":1920,"height":1080,"fps":30}}}}"## + ) + }; + + // UN SEUL `seek_to` : les deux rendus partagent la meme AVFrame, donc le + // decodeur ne peut pas introduire de difference qu'on prendrait pour l'effet. + let (sharp, blurred) = unsafe { + let sf = dec.seek_to(1.1).expect("Decoder::seek_to"); + let cfg = Cfg::c8(); + let render = |mblur: f32| { + comp.set_scene(Some(Scene::from_json(&scene_of(mblur)).expect("scene json"))); + comp.compose_frame(sf, sf, 66.0, &cfg).expect("compose_frame"); + comp.readback_direct().expect("readback_direct").2 + }; + (render(0.0), render(1.0)) + }; + + write_ppm("compose_linux_mb_screen_off", W, H, &sharp); + write_ppm("compose_linux_mb_screen_on", W, H, &blurred); + + let mut diff_sum = 0u64; + for (a, b) in sharp.chunks_exact(4).zip(blurred.chunks_exact(4)) { + for c in 0..3 { + diff_sum += (a[c] as i32 - b[c] as i32).unsigned_abs() as u64; + } + } + let mean_diff = diff_sum as f32 / (W * H * 3) as f32; + + // Detail haute frequence : somme des gradients voisins sur le canal vert. + let sharpness = |img: &[u8]| -> f32 { + let g = |x: u32, y: u32| img[((y * W + x) * 4 + 1) as usize] as i32; + let mut acc = 0u64; + for y in 0..H - 1 { + for x in 0..W - 1 { + acc += (g(x + 1, y) - g(x, y)).unsigned_abs() as u64; + acc += (g(x, y + 1) - g(x, y)).unsigned_abs() as u64; + } + } + acc as f32 / ((W - 1) * (H - 1) * 2) as f32 + }; + let (s_sharp, s_blur) = (sharpness(&sharp), sharpness(&blurred)); + println!( + "compose_linux flou de velocite : mean_diff={mean_diff:.2} gradient net={s_sharp:.2} floute={s_blur:.2}" + ); + + // Mesure observee : mean_diff 12,5 et gradient 7,9 -> 3,0. Les seuils gardent + // de la marge tout en restant loin du « ca a bouge d'un poil ». + assert!( + mean_diff > 4.0, + "motionBlur 0 vs 1 rend (quasi) la MEME image (mean_diff={mean_diff:.3}) — mb/src_prev/dst_prev non cables ?" + ); + assert!( + s_blur < s_sharp * 0.7, + "le rendu floute n'est pas plus doux (gradient {s_blur:.2} vs {s_sharp:.2}) — le flou ne suit pas la velocite" + ); +} + +/// Meme flou de velocite, mais sur le calque CAMERA — un draw distinct, avec son +/// propre `src_prev` (qui doit suivre le cover-crop et le miroir) et son propre +/// `dst_prev`. +/// +/// La velocite vient d'une region « Full Camera » en pleine ouverture : la boite +/// camera passe de 0,526 a 0,579 de large en une frame pendant que `s_dst` ne +/// bouge PAS d'un pouce. C'est ce qui rend le test concluant — une difference +/// mesuree dans la boite camera ne peut pas venir du calque ecran, et +/// l'assertion sur le coin haut-gauche (hors boite) le verifie explicitement. +#[test] +fn compose_linux_flou_de_velocite_camera() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("compose_linux flou de velocite camera: opt-in. Skip."); + return; + } + let webcam_fixture = "../fixture/webcam.mp4"; + if !Path::new(webcam_fixture).is_file() { + eprintln!("compose_linux flou de velocite camera: pas de fixture webcam. Skip."); + return; + } + + let gpu = Gpu::create(false).expect("Gpu::create"); + let comp = Compositor::new_sized(&gpu, W, H).expect("Compositor::new_sized"); + let mut screen = Decoder::open(FIXTURE, &gpu).expect("Decoder::open screen"); + let mut cam = Decoder::open(webcam_fixture, &gpu).expect("Decoder::open webcam"); + + // `webcamMirror: true` : le miroir inverse les bornes u de `src`, et + // `src_prev` doit inverser les MEMES. S'il gardait l'ancien [0,0,1,1] la + // reprojection viserait une zone de texture jamais affichee. + let scene_of = |mblur: f32| { + format!( + r##"{{"clips":[],"layout":{{"preset":"picture-in-picture","webcamSize":1,"webcamShape":"rectangle","webcamMirror":true,"webcamPosition":null,"webcamReactiveZoom":false}},"effects":{{"padding":0,"blur":false,"shadow":0,"roundnessFrac":0,"motionBlur":{mblur}}},"background":{{"kind":"color","color":"#101015"}},"zoomRegions":[],"cameraFullscreenRegions":[{{"startSec":2,"endSec":4}}],"annotations":[],"cursor":{{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}},"cropByClip":[],"output":{{"width":1920,"height":1080,"fps":30}}}}"## + ) + }; + + let (sharp, blurred) = unsafe { + let sf = screen.seek_to(2.1).expect("seek screen"); + let wf = cam.seek_to(2.1).expect("seek webcam"); + let cfg = Cfg::c8(); + let render = |mblur: f32| { + let scene = Scene::from_json(&scene_of(mblur)).expect("scene json"); + comp.set_live_params(openscreen_compositor::compositor::live_params_from_scene(&scene)); + comp.set_scene(Some(scene)); + // frame 126 = t 2,1 s : la camera est a mi-ouverture. + comp.compose_frame(sf, wf, 126.0, &cfg).expect("compose_frame"); + comp.readback_direct().expect("readback_direct").2 + }; + (render(0.0), render(1.0)) + }; + + write_ppm("compose_linux_mb_camera_off", W, H, &sharp); + write_ppm("compose_linux_mb_camera_on", W, H, &blurred); + + let mean_diff = |x0: u32, x1: u32, y0: u32, y1: u32| -> f32 { + let mut sum = 0u64; + for y in y0..y1 { + for x in x0..x1 { + let i = ((y * W + x) * 4) as usize; + for c in 0..3 { + sum += (sharp[i + c] as i32 - blurred[i + c] as i32).unsigned_abs() as u64; + } + } + } + sum as f32 / ((x1 - x0) * (y1 - y0) * 3) as f32 + }; + // Boite camera a t = 2,1 s : [0,411 ; 0,403 ; 0,579 ; 0,579] de la sortie, + // soit x 394..950 et y 217..530 en pixels — retrecie ici pour rester loin des + // bords adoucis. Le coin haut-gauche, lui, ne montre que l'ecran. + let inside = mean_diff(420, 920, 245, 505); + let outside = mean_diff(0, 300, 0, 150); + println!("compose_linux flou de velocite camera : dans la boite={inside:.2} hors boite={outside:.4}"); + + assert!( + inside > 4.0, + "la camera n'est pas floutee (diff={inside:.3}) — src_prev/dst_prev du calque webcam non cables ?" + ); + assert!( + outside < 0.01, + "l'ecran a bouge aussi (diff={outside:.3}) — le test ne prouve alors rien sur la camera" + ); +} + +/// Trainee fantome du curseur (accumulation temporelle, pas un mode de shader). +/// +/// Le curseur traverse le cadre ; a `cursor.motionBlur = 1` `plan_cursor` rend +/// 11 taps entre sa position d'il y a 8 frames (8/60 s) et sa position courante. +/// On compare au meme rendu sans trainee : la seule difference possible etant le +/// curseur, un exces de vert la ou le curseur N'EST PAS (mais est PASSE) est la +/// signature de la trainee. +/// +/// « Exces de vert » = G - max(R,B), pas G brut : une copie a 1/taps d'opacite +/// sur un fond CLAIR fait a peine monter le vert (le fond y est deja) mais fait +/// nettement chuter le rouge et le bleu. Mesurer G seul rendrait le test +/// dependant de ce qui passe sous le curseur dans la video. +/// +/// Le trajet est volontairement hors de l'axe median (cy = 0,28) : une passe de +/// composition qui retournerait `accum` verticalement enverrait la trainee dans +/// la bande miroir, ce que la seconde assertion interdit. A cy = 0,5 le defaut +/// serait invisible. +#[test] +fn compose_linux_trainee_de_curseur() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("compose_linux trainee curseur: opt-in. Skip."); + return; + } + const SPRITE: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAACXBIWXMAAAABAAAAAQBPJcTWAAAAGElEQVR4nGNk+MdAEmAhTfmohlENQ0kDAGoRATwbkCdPAAAAAElFTkSuQmCC"; + + let gpu = Gpu::create(false).expect("Gpu::create"); + let comp = Compositor::new_sized(&gpu, W, H).expect("Compositor::new_sized"); + let mut dec = Decoder::open(FIXTURE, &gpu).expect("Decoder::open"); + + // Piste : deplacement horizontal regulier cx 0,1 -> 0,9 en 0,4 s, a cy fixe. + // Assez rapide pour que les 8/60 s de recul de la trainee separent nettement + // les deux extremites (~256 px a 960 de large) : sans quoi la trainee se + // superpose au curseur lui-meme et on ne pourrait plus les distinguer. + let mut samples = String::new(); + for k in 0..=8 { + let (ms, cx) = (k * 50, 0.1 + 0.1 * k as f32); + if k > 0 { + samples.push(','); + } + samples.push_str(&format!( + r#"{{"timeMs":{ms},"cx":{cx},"cy":0.28,"cursorType":"arrow"}}"# + )); + } + let track_path = std::env::temp_dir().join("os_cursor_trail_track.json"); + std::fs::write(&track_path, format!(r#"{{"samples":[{samples}]}}"#)).expect("write track"); + let track = CursorTrack::load(track_path.to_str().unwrap(), 0.0, 2.0).expect("CursorTrack::load"); + comp.set_cursor(track); + comp.set_cursor_time(Some(0.35)); + + let scene_of = |mblur: f32| { + format!( + r##"{{"clips":[],"layout":{{"preset":"no-webcam","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}},"effects":{{"padding":0,"blur":false,"shadow":0,"roundnessFrac":0,"motionBlur":0}},"background":{{"kind":"color","color":"#101015"}},"zoomRegions":[],"annotations":[],"cursor":{{"show":true,"size":3,"smoothing":0,"motionBlur":{mblur},"clickBounce":0,"clipToBounds":false,"theme":"default","cursorSprites":{{"arrow":{{"path":"{SPRITE}","hotspotX":0.5,"hotspotY":0.5}}}}}},"cropByClip":[],"output":{{"width":1920,"height":1080,"fps":30}}}}"## + ) + }; + + let (sharp, trail) = unsafe { + let sf = dec.seek_to(1.0).expect("Decoder::seek_to"); + let cfg = Cfg::c8(); + let render = |mblur: f32| { + let scene = Scene::from_json(&scene_of(mblur)).expect("scene json"); + // `cursor.motionBlur` ET `cursor.size` transitent par les LiveParams, + // pas par la scene brute : sans ca `plan_cursor` verrait toujours 0. + comp.set_live_params(openscreen_compositor::compositor::live_params_from_scene(&scene)); + comp.set_scene(Some(scene)); + comp.compose_frame(sf, sf, 15.0, &cfg).expect("compose_frame"); + comp.readback_direct().expect("readback_direct").2 + }; + (render(0.0), render(1.0)) + }; + + write_ppm("compose_linux_cursor_trail_off", W, H, &sharp); + write_ppm("compose_linux_cursor_trail_on", W, H, &trail); + + // A t = 0,35 s le curseur est en cx 0,8 (x ~ 768 px) et 8/60 s plus tot en + // cx ~ 0,533 (x ~ 512 px) ; le sprite fait 51 px de cote a size 3 (34/1080 + // de frame_min_px, x3), donc le curseur COURANT occupe x = 742..794. La + // fenetre ci-dessous couvre le milieu du trajet, franchement a sa gauche : + // sans trainee il n'y a rien du tout. La bande miroir est son reflet par + // rapport a l'axe horizontal de l'image (cy = 0,28 est hors de cet axe + // exprès), donc un `accum` composite a l'envers y atterrirait. + let greener = |x0: u32, x1: u32, y0: u32, y1: u32| -> usize { + let excess = |img: &[u8], i: usize| { + img[i + 1] as i32 - (img[i] as i32).max(img[i + 2] as i32) + }; + let mut n = 0; + for y in y0..y1 { + for x in x0..x1 { + let i = ((y * W + x) * 4) as usize; + if excess(&trail, i) - excess(&sharp, i) > 10 { + n += 1; + } + } + } + n + }; + let on_path = greener(530, 700, 130, 172); + let mirrored = greener(530, 700, 368, 410); + println!("compose_linux trainee curseur : sur le trajet={on_path} bande miroir={mirrored}"); + + // La fenetre fait 170x42 = 7140 px et la trainee la remplit entierement. + // Le seuil a 4000 laisse de la marge tout en refusant une trainee qui ne + // couvrirait qu'un bout du trajet. + assert!( + on_path > 4000, + "pas de trainee au milieu du trajet ({on_path} px plus verts) — le curseur n'est dessine qu'a sa position courante" + ); + assert!( + mirrored < 50, + "trainee dans la bande MIROIR ({mirrored} px) — la passe de composition d'accum retourne l'image en Y" + ); +} + /// Export (WP6) : ~1s de la fixture -> MP4 H264 software. Verifie que la marche /// de timeline + l'encodeur + le muxer produisent un fichier non trivial. Le /// contenu est re-validable par ffprobe (cf. la commande dans le run manuel). From d2764394080f0a214d77049190076a0a419b5442 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Fri, 31 Jul 2026 19:05:13 +0200 Subject: [PATCH 36/46] test(compositor): pin the Linux webcam circle mask, and fix two tests that ignored their own scene MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I told the user the circle mask was missing on Linux. It is not — I was wrong, and the way I was wrong is worth recording. Two rendered-frame tests I wrote called set_scene() without set_live_params(). compose_frame reads LiveParams, not the raw scene, so webcamShape/padding/effects never arrived: the scene parsed fine and was then ignored, and the renders showed the DEFAULT "rounded" shape. I read those rounded corners in a downscaled frame as "rectangular" and reported a missing feature. This gotcha is documented — it is the first item under "Test gotchas" for this crate — and I walked into it anyway. The new test measures instead of eyeballing. It renders the same scene three times (no camera / rectangle / circle) and diffs each against the no-camera pass, so the diff IS the camera's footprint including its mask, with no need to guess where plan_frame put it or to tell camera pixels from background. Then it checks the bounding box's fill ratio, which separates the two shapes unambiguously: rectangle 256x192 AR 1.333 fill 0.993 circle 256x256 AR 1.000 fill 0.798 (pi/4 = 0.785) So the circle both forces a square box and crops to a real disc. Had this test existed with set_live_params missing, it would have reported identical numbers for both shapes — which is exactly what it did before I fixed it, and is what made the omission obvious. --- crates/compositor/tests/compose_linux.rs | 107 ++++++++++++++++++++++- 1 file changed, 106 insertions(+), 1 deletion(-) diff --git a/crates/compositor/tests/compose_linux.rs b/crates/compositor/tests/compose_linux.rs index 9e86aa1d5..097f787a6 100644 --- a/crates/compositor/tests/compose_linux.rs +++ b/crates/compositor/tests/compose_linux.rs @@ -839,7 +839,11 @@ fn compose_linux_ombre_webcam_ronde_et_texte() { // `shadow: 1` + camera en cercle + une annotation texte visible a t=1s. let scene_json = r##"{"clips":[],"layout":{"preset":"picture-in-picture","webcamSize":1,"webcamShape":"circle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false},"effects":{"padding":0.14,"blur":false,"shadow":1,"roundnessFrac":0.04,"motionBlur":0},"background":{"kind":"gradient","angleDeg":45,"stops":["#1f2933","#3b6bff"]},"zoomRegions":[],"annotations":[{"id":"a1","kind":"text","x":0.08,"y":0.08,"w":0.5,"h":0.14,"startSec":0,"endSec":10,"zIndex":1,"text":{"content":"Ombre + fond","color":"#ffffff","backgroundColor":"#e0245e","fontSizeRel":0.09,"fontFamily":"","fontWeight":"normal","fontStyle":"normal","textDecoration":"none","textAlign":"center"}}],"cursor":{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"},"cropByClip":[],"output":{"width":1920,"height":1080,"fps":30}}"##; - comp.set_scene(Some(Scene::from_json(scene_json).expect("scene json"))); + let parsed = Scene::from_json(scene_json).expect("scene json"); + // Cf. le commentaire dans compose_linux_forme_webcam_cercle : sans les + // LiveParams, la scene est parsee et ignoree. + comp.set_live_params(openscreen_compositor::compositor::live_params_from_scene(&parsed)); + comp.set_scene(Some(parsed)); let (w, h, rgba) = unsafe { let sf = screen.seek_to(1.0).expect("seek screen"); @@ -883,3 +887,104 @@ fn compose_linux_ombre_webcam_ronde_et_texte() { "fond d'annotation introuvable ({plate_px} px roses) — la plaque n'est pas dessinee" ); } + +/// Forme de la webcam : `rectangle` contre `circle`. +/// +/// Le masque n'est pas un mode de shader dedie — il sort de `radius_px`, que +/// `plan_frame` met a la moitie du cote pour `circle`. Ce test le MESURE au +/// lieu de le supposer. +/// +/// La methode : rendre trois fois la meme scene — sans camera, camera +/// rectangle, camera cercle — et diffe chacune des deux dernieres contre la +/// premiere. Le diff EST l'empreinte de la camera, masque compris, sans avoir +/// a deviner ou `plan_frame` l'a posee ni a distinguer la camera du fond. Un +/// disque remplit pi/4 ~= 0,785 de sa boite englobante, un rectangle la +/// remplit entierement : le taux de remplissage separe les deux sans ambiguite. +#[test] +fn compose_linux_forme_webcam_cercle() { + if std::env::var("OPENSCREEN_LINUX_COMPOSE").is_err() || !Path::new(FIXTURE).is_file() { + eprintln!("compose_linux forme webcam: opt-in. Skip."); + return; + } + let webcam_fixture = "../fixture/webcam.mp4"; + if !Path::new(webcam_fixture).is_file() { + eprintln!("compose_linux forme webcam: pas de fixture webcam. Skip."); + return; + } + + let gpu = Gpu::create(false).expect("Gpu::create"); + let comp = Compositor::new_sized(&gpu, W, H).expect("Compositor::new_sized"); + let mut screen = Decoder::open(FIXTURE, &gpu).expect("Decoder::open screen"); + let mut cam = Decoder::open(webcam_fixture, &gpu).expect("Decoder::open webcam"); + + let scene = |preset: &str, shape: &str| { + format!( + r##"{{"clips":[],"layout":{{"preset":"{preset}","webcamSize":1.6,"webcamShape":"{shape}","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}},"effects":{{"padding":0.1,"blur":false,"shadow":0,"roundnessFrac":0.0,"motionBlur":0}},"background":{{"kind":"color","color":"#00ff00"}},"zoomRegions":[],"annotations":[],"cursor":{{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}},"cropByClip":[],"output":{{"width":1920,"height":1080,"fps":30}}}}"## + ) + }; + + let mut render = |preset: &str, shape: &str| -> Vec { + let parsed = Scene::from_json(&scene(preset, shape)).expect("scene json"); + // OBLIGATOIRE. `compose_frame` lit les LiveParams, PAS la scene brute : + // la forme webcam, le padding et les effets y transitent. Sans cette + // ligne la scene est parsee mais ignoree, et le test mesure la forme par + // defaut ("rounded") en croyant mesurer celle qu'il a demandee. + comp.set_live_params(openscreen_compositor::compositor::live_params_from_scene(&parsed)); + comp.set_scene(Some(parsed)); + unsafe { + let sf = screen.seek_to(1.0).expect("seek screen"); + let wf = cam.seek_to(1.0).expect("seek webcam"); + let mut cfg = Cfg::c8(); + cfg.shadow = false; + comp.compose_frame(sf, wf, 1.0, &cfg).expect("compose_frame"); + comp.readback_direct().expect("readback").2 + } + }; + + let none = render("no-webcam", "rectangle"); + let rect = render("picture-in-picture", "rectangle"); + let circle = render("picture-in-picture", "circle"); + write_ppm("compose_linux_webcam_rect", W, H, &rect); + write_ppm("compose_linux_webcam_circle", W, H, &circle); + + // Empreinte = pixels qui changent quand la camera apparait. + let footprint = |with: &[u8]| -> Vec { + with.chunks_exact(4) + .zip(none.chunks_exact(4)) + .map(|(a, b)| { + (a[0] as i32 - b[0] as i32).abs() + + (a[1] as i32 - b[1] as i32).abs() + + (a[2] as i32 - b[2] as i32).abs() + > 24 + }) + .collect() + }; + // Taux de remplissage de la boite englobante de l'empreinte. + let fill = |mask: &[bool], label: &str| -> f32 { + let (mut x0, mut y0, mut x1, mut y1) = (W, H, 0u32, 0u32); + let mut n = 0u32; + for y in 0..H { + for x in 0..W { + if mask[(y * W + x) as usize] { + x0 = x0.min(x); y0 = y0.min(y); x1 = x1.max(x); y1 = y1.max(y); n += 1; + } + } + } + assert!(x1 > x0 && y1 > y0, "{label} : aucune empreinte de camera"); + let (bw, bh) = (x1 - x0 + 1, y1 - y0 + 1); + let ar = bw as f32 / bh as f32; + let f = n as f32 / (bw * bh) as f32; + println!("{label} : boite {bw}x{bh} (AR {ar:.3}), {n} px, remplissage {f:.3}"); + f + }; + + let rect_fill = fill(&footprint(&rect), "rectangle"); + let circle_fill = fill(&footprint(&circle), "cercle"); + + assert!(rect_fill > 0.95, "le rectangle devrait remplir sa boite ({rect_fill:.3})"); + // pi/4 = 0,785 ; on tolere l'antialiasing du SDF sur le pourtour. + assert!( + (circle_fill - 0.785).abs() < 0.06, + "le masque cercle ne rogne pas comme un disque (remplissage {circle_fill:.3}, attendu ~0.785)" + ); +} From 846ff230f408c228d8096de65a0d80b70c995bd6 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Fri, 31 Jul 2026 19:41:28 +0200 Subject: [PATCH 37/46] fix(annotations): centre Linux text vertically in its box MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two native references genuinely disagree here: Windows centres with DWRITE_PARAGRAPH_ALIGNMENT_CENTER (text_windows.rs:175-176) to match the web overlay's `alignItems: center`, macOS does not, and the Linux port had copied macOS — so text sat at the top of its plate. The web is the product intent, which the user confirmed. The block height is measured from the last laid-out run rather than counting runs times line_height: cosmic-text splits one logical line into several runs when it wraps, so counting runs would overestimate the height of any text that overflows its width and push it up out of centre. Text taller than its box stays anchored at the top instead of overflowing upward, where it would be cropped away entirely. --- crates/compositor/src/text_linux.rs | 42 ++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/crates/compositor/src/text_linux.rs b/crates/compositor/src/text_linux.rs index 014948fdd..c2602d4b1 100644 --- a/crates/compositor/src/text_linux.rs +++ b/crates/compositor/src/text_linux.rs @@ -152,6 +152,25 @@ impl TextRasterizer { } buffer.shape_until_scroll(&mut font_system, false); + // CENTRAGE VERTICAL. L'overlay web pose `alignItems: center` sur le + // conteneur de l'annotation, et Windows reproduit ca avec + // `DWRITE_PARAGRAPH_ALIGNMENT_CENTER` (text_windows.rs:175-176). macOS + // ne le fait pas, et le portage Linux avait copie macOS : le texte + // collait en haut de sa boite. Les deux references natives divergent + // reellement ici ; c'est le web qui porte l'intention produit. + // + // La hauteur du bloc est prise sur la DERNIERE ligne posee plutot que + // sur un compte de lignes x line_height : cosmic-text peut replier une + // ligne logique en plusieurs runs, donc compter les runs surestimerait + // des que le texte deborde en largeur. + let text_h = buffer + .layout_runs() + .map(|run| run.line_top + run.line_height) + .fold(0.0f32, f32::max); + // `max(0)` : un texte plus haut que sa boite reste ancre en haut plutot + // que de sortir par le dessus, ou il serait entierement rogne. + let y_offset = (((h as f32) - text_h) * 0.5).max(0.0).round() as i32; + // Atlas R8 : on n'ecrit que le canal alpha (couverture). Le tint par // `spec.color` se fait cote shader (mode 11). let mut atlas: Vec = vec![0u8; (w * h) as usize]; @@ -192,7 +211,7 @@ impl TextRasterizer { // `placement.top` est la hauteur de l'encre AU-DESSUS de la // ligne de base, donc la premiere rangee du bitmap est a // `baseline - top`. Le signe etait inverse. - let ink_top = glyph_y - placement.top; + let ink_top = glyph_y - placement.top + y_offset; let ink_left = glyph_x + placement.left; for row in 0..img_h as i32 { let dest_y = ink_top + row; @@ -376,6 +395,27 @@ mod tests { ); } + #[test] + fn one_short_line_is_centred_vertically_in_its_box() { + // L'overlay web pose `alignItems: center` et Windows fait pareil + // (DWRITE_PARAGRAPH_ALIGNMENT_CENTER) ; macOS non, et le portage avait + // copie macOS. Une ligne de 40px dans une boite de 200px doit laisser a + // peu pres autant de vide au-dessus qu'en dessous. + let raster = TextRasterizer::new().expect("rasterizer"); + let atlas = raster.build_atlas(&spec("Hx", "center")).expect("atlas"); + let (w, h) = (400usize, 200usize); + let rows = ink_rows(&atlas, w, 0, w); + assert!(!rows.is_empty(), "aucune encre"); + + let (top, bottom) = (rows[0], *rows.last().unwrap()); + let above = top as i32; + let below = (h - 1 - bottom) as i32; + assert!( + (above - below).abs() <= 12, + "texte non centre verticalement : {above}px au-dessus, {below}px en dessous" + ); + } + #[test] fn centering_moves_the_ink_off_the_left_edge() { // `spec.align` n'etait jamais applique : tout sortait ferre a gauche From 86579ef47b0f73bf99d77700a08a8e7ff0def053 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Fri, 31 Jul 2026 20:04:02 +0200 Subject: [PATCH 38/46] fix(stt): stop compiling the shipped whisper helper for the build machine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GGML_NATIVE was ON under a comment claiming it produced a "+avx2 +fma +f16c" baseline. It does not: ggml turns GGML_NATIVE=ON into `-march=native`, so the CPU backend targets whatever CPU ran the build. build-whisper-stt.yml builds the shipped binaries on GitHub-hosted runners, whose x64 images are Xeons with AVX-512 — so a published artifact could carry AVX-512 and die with SIGILL on every AMD before Zen 4 and every Intel consumer part since Alder Lake, and the baseline shifted silently with whatever hardware picked up the job. Same class of bug as the OpenSSL note: an artifact whose contents depend on the machine that built it. Turning GGML_NATIVE off is necessary but not sufficient. ggml derives its instruction-set options from INS_ENB via `option()`, which only seeds a value the first time a build tree is configured — a cache written while NATIVE was ON keeps GGML_AVX2=OFF, and the result is a plain x86-64 binary with no AVX2 at all. Verified: that path produced zero %ymm instructions. So pin the baseline explicitly instead of inheriting it. Ships SSE4.2 + AVX + AVX2 + BMI2 + FMA + F16C, x86-64 hosts only, AVX-512 deliberately excluded. That is the Haswell/Zen 1 floor the installer already requires. OSC_NATIVE_CPU=ON restores -march=native for local benchmarking. Verified on linux-x64: `-msse4.2 -mf16c -mfma -mbmi2 -mavx -mavx2`, 12744 %ymm instructions, 0 %zmm, and the helper still transcribes on Vulkan. --- electron/native/whisper-stt/CMakeLists.txt | 44 ++++++++++++++++++---- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/electron/native/whisper-stt/CMakeLists.txt b/electron/native/whisper-stt/CMakeLists.txt index 5b413924f..e5358527e 100644 --- a/electron/native/whisper-stt/CMakeLists.txt +++ b/electron/native/whisper-stt/CMakeLists.txt @@ -46,13 +46,43 @@ set(FETCHCONTENT_QUIET OFF) # at the cost of a larger single binary. Mirrors the working POC build. set(GGML_BACKEND_DL OFF CACHE BOOL "" FORCE) set(GGML_CPU_ALL_VARIANTS OFF CACHE BOOL "" FORCE) -# GGML_NATIVE compiles for this dev machine's CPU arch (Zen 2 on AMD Ryzen -# 5 7520U). This is what llama.cpp ships by default and it produces one -# `ggml-cpu` library statically linked into `ggml.dll`. The `+avx2 +fma -# +f16c` baseline is broad enough to cover every x86 CPU OpenScreen -# supports (the OpenScreen installer currently does not run on -# pre-Haswell/AMD-Bulldozer hardware). -set(GGML_NATIVE ON CACHE BOOL "" FORCE) +# ponytail: this was ON, under a comment claiming it produced a "+avx2 +fma +# +f16c" baseline. It does not. GGML_NATIVE=ON makes ggml compile the CPU +# backend with `-march=native` (ggml/src/ggml-cpu/CMakeLists.txt), i.e. for +# whichever CPU happened to run the build. `.github/workflows/build-whisper-stt.yml` +# builds the shipped binaries on GitHub-hosted runners, and the x64 images are +# Xeons that carry AVX-512, so a published artifact could contain AVX-512 and +# die with SIGILL on every AMD before Zen 4 and every Intel consumer part since +# Alder Lake — while the baseline silently shifted with whatever hardware +# picked up the job. Same class of bug as the OpenSSL note below: an artifact +# whose contents are a function of the build machine. +# +# OFF is what actually yields the intended baseline. ggml then sets INS_ENB=ON, +# which defaults GGML_SSE42/AVX/AVX2/BMI2/FMA/F16C to ON and leaves every +# GGML_AVX512* OFF — Haswell/Zen 1 and newer, which is what the installer +# already requires. This backend is only the fallback for hosts with no usable +# GPU backend, so the cost is marginal and the artifact becomes reproducible. +# +# OSC_NATIVE_CPU is the escape hatch for local benchmarking. It must never be +# set for anything that gets shipped. +option(OSC_NATIVE_CPU "Tune the CPU backend for this machine (-march=native); local builds only" OFF) +if(OSC_NATIVE_CPU) + set(GGML_NATIVE ON CACHE BOOL "" FORCE) +else() + set(GGML_NATIVE OFF CACHE BOOL "" FORCE) + # Pin the baseline rather than inheriting ggml's INS_ENB defaults. `option()` + # only seeds a value the first time a build tree is configured, so a cache + # that was written while GGML_NATIVE was ON keeps GGML_AVX2=OFF forever and + # quietly downgrades the result to plain x86-64 — no AVX2 at all, which is + # slower than what this project shipped before. Being explicit also states + # the contract in one place instead of leaving it to an upstream default. + # AVX-512 stays off: that is the part that does not run everywhere. + if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|amd64)$") + foreach(osc_isa SSE42 AVX AVX2 BMI2 FMA F16C) + set(GGML_${osc_isa} ON CACHE BOOL "" FORCE) + endforeach() + endif() +endif() set(WHISPER_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) set(WHISPER_BUILD_TESTS OFF CACHE BOOL "" FORCE) set(WHISPER_BUILD_SERVER OFF CACHE BOOL "" FORCE) From fd45999bc9faf8b2ef55983d662977e68405e7f4 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Fri, 31 Jul 2026 20:27:15 +0200 Subject: [PATCH 39/46] ci(stt): stop tripling the whisper libraries in the uploaded artifact MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit build-whisper-stt.sh stages a symlink farm on Linux and macOS (libggml-vulkan.so -> .so.0 -> .so.0.15.1), and plain `cp` follows every link named on the command line, so each one became a full copy of its payload. libggml-vulkan.so.0.15.1 is 60 MB on its own: measured, the bag was 188 MB instead of 63 MB and the uploaded tarball 59 MB instead of 20 MB. gzip cannot recover any of it — its 32 KiB window is far too small to dedupe copies that sit 60 MB apart. `cp -a` preserves the links; tar keeps them from there (no -h) and untars them intact on the consuming side. Verified end to end on linux-x64 with the real libraries: cp -a -> tar czf -> tar xzf leaves 12 symlinks and the extracted helper still answers GET / and transcribes. `dir/.` rather than `dir/*` so the copy does not depend on the shell's glob skipping dotfiles. -v is kept: the step's log output was the point of it. This shrinks the artifact and the CI download only. The installers are unchanged, because scripts/stage-whisper-stt.sh:65 is a second plain `cp` that re-expands the farm before electron-builder runs. Making that one -a as well would cut ~131 MB from every package — electron-builder recreates symlinks (builder-util/out/fs.js:263) and squashfs stores them natively (verified: a 61 MB farm becomes a 19 MB image), but deb and pacman go through fpm, whose behaviour here I could not verify, so that step wants a real package inspection rather than an assumption. --- .github/workflows/build-whisper-stt.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-whisper-stt.yml b/.github/workflows/build-whisper-stt.yml index abc07fb5b..0d50a4006 100644 --- a/.github/workflows/build-whisper-stt.yml +++ b/.github/workflows/build-whisper-stt.yml @@ -172,7 +172,25 @@ jobs: mkdir -p "$BAG" # Copy the whole per-platform directory: the helper executable plus # every ggml backend sidecar/library it needs at runtime. - cp -v "electron/native/bin/${{ matrix.tag }}"/* "$BAG/" + # + # -a, because build-whisper-stt.sh stages a symlink farm on Linux and + # macOS (libggml-vulkan.so -> .so.0 -> .so.0.15.1) and plain cp follows + # every link named on the command line, expanding each into a full copy + # of the payload. libggml-vulkan.so.0.15.1 alone is 60 MB, so the bag + # carried it three times: measured 188 MB instead of 63 MB, and the + # uploaded tarball 59 MB instead of 20 MB (gzip's 32 KiB window cannot + # dedupe copies that far apart). `dir/.` rather than `dir/*` so the copy + # does not depend on the shell's glob skipping dotfiles. + # + # tar preserves the links from here on (no -h below), so this shrinks + # the artifact and the CI download. It does NOT change the installers: + # scripts/stage-whisper-stt.sh:65 is a second plain `cp` that re-expands + # the farm before electron-builder ever sees it. Making that one -a too + # would cut ~131 MB from every package (electron-builder recreates + # symlinks, and squashfs/AppImage stores them natively — both verified), + # but deb/pacman go through fpm, whose behaviour here is unverified, so + # that step needs a real package inspection rather than a guess. + cp -av "electron/native/bin/${{ matrix.tag }}/." "$BAG/" tar -czf "${BAG}.tar.gz" "$BAG" echo "Staged ${BAG}.tar.gz" From 8079d320c69dab412ff8a2e10d0edffd6dda42b7 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Fri, 31 Jul 2026 20:31:06 +0200 Subject: [PATCH 40/46] fix(editor): push every native param on load, not just the open panel's MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The user reported the custom cursor only appearing once they opened the cursor settings panel. The cause is broader than the cursor and worth stating plainly: NOTHING pushed the complete param set. Each group was pushed only by the mount effect of the panel that owns it — VideoEffectsPane, LayoutPane, CursorPane — and the inspector renders exactly one panel at a time, defaulting to "effects". So on load the layout and cursor panels had never mounted and their params sat at the addon's compiled-in defaults: cursor size 1.0 for a project storing 3.0, smoothing 0 for 0.67, motion blur 0 for 0.35, webcam shape and mirror likewise. Opening the panel pushed them and it looked like the panel had switched something on. The store's lastParams replay could not save this: it replays keys that were pushed at least once, and these were never pushed at all. pushAllNativeParams() now sends all fourteen keys the addon understands (live.rs set_param_bool/num/str), called from NativeCompositorOverlay — the one component that exists regardless of which panel is showing, and that already owns the view and the scene push. No viewId guard: setNativeParam memoises and setCurrentNativeViewId replays, so mount order does not matter. It cannot fight a slider drag either, because setLive goes through setDocument, so the overlay's document selector already carries the new value on the same tick. The three mount-sync effects are removed rather than left as belt and braces — duplicated pushes of the same value are how the two paths would silently drift apart. UNITS. The helper sends RAW values, matching the old mount effects and NOT the sliders: SliderCell displays cursor.size * 10 and divides by 10 in its handler, with * 100 for smoothing and motion blur. Sending slider-space values here would scale the preview by 10 or 100 on load. The two base-unit constants moved to src/native/paramUnits.ts so the helper and the panels cannot disagree about them. Also fixes the Camera Shape row overflowing its panel. `repeat(4, 1fr)` is `minmax(auto, 1fr)`, so each track's minimum resolves to its content minimum, and "Rounded" is an unbreakable word: four buttons at 64.83px plus three 8px gaps need 283.3px where the panel has 234, so the tracks refused to shrink and the last button was clipped. `minmax(0, 1fr)` plus minWidth:0 on the buttons lets them shrink. The new test asserts NO addon key is missing, transcribing the list from live.rs — a test that checked only the keys we happen to push would have passed against the bug. --- .../ai-edition/NativeCompositorOverlay.tsx | 28 +++++ src/components/ai-edition/RightPanes.tsx | 75 +++-------- src/native/nativeCompositorStore.test.ts | 118 ++++++++++++++++++ src/native/nativeCompositorStore.ts | 49 ++++++++ src/native/paramUnits.ts | 26 ++++ 5 files changed, 241 insertions(+), 55 deletions(-) create mode 100644 src/native/nativeCompositorStore.test.ts create mode 100644 src/native/paramUnits.ts diff --git a/src/components/ai-edition/NativeCompositorOverlay.tsx b/src/components/ai-edition/NativeCompositorOverlay.tsx index 4ad76f062..198cf89b5 100644 --- a/src/components/ai-edition/NativeCompositorOverlay.tsx +++ b/src/components/ai-edition/NativeCompositorOverlay.tsx @@ -1,13 +1,16 @@ import { useEffect, useMemo, useRef, useSyncExternalStore } from "react"; import { useScopedT } from "@/contexts/I18nContext"; import { noteUiProbeClipSwitch } from "@/lib/ai-edition/perf/uiFrameProbe"; +import { getEditorSettings } from "@/lib/ai-edition/store/editorSettings"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; import { resolveNativePosition } from "@/lib/ai-edition/timeline/timelineMap"; import { + pushAllNativeParams, setActiveClip, setCurrentNativeViewId, setNativePlaying, setNativeScene, + subscribeNativeCompositor, useIsCpuCompositor, useNativeCompositorView, } from "@/native"; @@ -142,6 +145,31 @@ export function NativeCompositorOverlay() { // arriving after mount never gets pushed to native). }, [viewId, document, sources, _webcamSizeRevision]); + // SYNCHRO COMPLETE DES PARAMS, en un seul endroit. + // + // Avant, chaque groupe de params n'etait pousse que par l'effet de montage du + // panneau qui le possede (VideoEffectsPane, LayoutPane, CursorPane). Or + // l'inspecteur n'affiche qu'UN panneau a la fois, donc au chargement les + // params curseur et layout n'avaient jamais ete pousses et l'addon restait + // sur ses defauts compiles : un projet en taille de curseur 3.0 s'affichait a + // 1.0 jusqu'a ce qu'on ouvre le panneau curseur, ou tout se remettait en + // place -- ce qui donnait l'illusion que le panneau « activait » le curseur. + // + // Ici, l'overlay possede deja la vue et pousse deja la scene ; c'est le seul + // point qui existe quel que soit le panneau affiche. + // + // Pas de garde sur `viewId` : `setNativeParam` memoise dans `lastParams` et + // `setCurrentNativeViewId` le rejoue a l'activation, donc l'ordre de montage + // n'a pas d'importance. Et ca ne peut pas lutter contre un drag de slider : + // `setLive` passe par `setDocument`, donc `document` a deja la NOUVELLE + // valeur a chaque tick -- la meme que celle que le handler vient de pousser. + const settings = useMemo(() => getEditorSettings(document), [document]); + useEffect(() => { + const push = () => pushAllNativeParams(settings); + push(); + return subscribeNativeCompositor(push); + }, [settings]); + const activeClipId = activeClip?.id ?? null; const activeClipIndex = activePosition?.clipIndex ?? null; const activeSourceTimeSec = activePosition?.sourceTimeSec ?? null; diff --git a/src/components/ai-edition/RightPanes.tsx b/src/components/ai-edition/RightPanes.tsx index 8a5ccae27..8470b723b 100644 --- a/src/components/ai-edition/RightPanes.tsx +++ b/src/components/ai-edition/RightPanes.tsx @@ -59,7 +59,7 @@ import { supportsCursorClickEffects } from "@/lib/cursor/cursorCapabilities"; import { CURSOR_THEMES, DEFAULT_CURSOR_THEME_ID } from "@/lib/cursor/cursorThemes"; import { buildGradientFromEditor } from "@/lib/gradientBuilder"; import { resolveImageWallpaperUrl, WALLPAPER_PATHS, WALLPAPER_THUMB_PATHS } from "@/lib/wallpaper"; -import { isNativeCompositorActive, setNativeParam, subscribeNativeCompositor } from "@/native"; +import { isNativeCompositorActive, setNativeParam } from "@/native"; import styles from "./NewEditorShell.module.css"; interface PaneProps { @@ -1269,31 +1269,10 @@ export function VideoEffectsPane() { // valeur px de l'UI par ce même rayon de base fait que le coin natif ≈ les px affichés // (au lieu de plafonner à ~24px comme avec /64). const NATIVE_SCREEN_BASE_RADIUS_PX = 24; - useEffect(() => { - const syncToNative = () => { - // pas de garde `isNativeCompositorActive` : setNativeParam mémorise les valeurs - // même sans vue active, et le store les rejoue quand une vue s'active (fix du - // démarrage sur les défauts, indépendant de l'ordre de montage). - setNativeParam("backgroundBlur", settings.showBlur); - setNativeParam("motionBlur", settings.motionBlurAmount); - setNativeParam("shadow", settings.shadowIntensity); - setNativeParam("roundness", settings.borderRadius / NATIVE_SCREEN_BASE_RADIUS_PX); - setNativeParam("padding", settings.padding / 100); - const bg = settings.wallpaper; - if (bg.startsWith("#")) { - setNativeParam("backgroundColor", bg); - } - }; - syncToNative(); - return subscribeNativeCompositor(syncToNative); - }, [ - settings.showBlur, - settings.motionBlurAmount, - settings.shadowIntensity, - settings.borderRadius, - settings.padding, - settings.wallpaper, - ]); + // La synchro initiale de ces params vit desormais dans NativeCompositorOverlay + // (`pushAllNativeParams`) : l'inspecteur n'affiche qu'un panneau a la fois, donc + // un effet de montage ici ne poussait rien tant que ce panneau precis n'avait pas + // ete ouvert. Les handlers par controle ci-dessous poussent toujours leurs diffs. return ( } helpText={ts("effects.help")}> @@ -1419,17 +1398,7 @@ export function LayoutPane() { const { settings, set, setLive, commit, hasDocument } = useEditorSettings(); const document = useProjectStore((s) => s.document); - // Push webcam-layout settings into the native compositor (initial values + on view - // activation); the per-control handlers below also push their diffs live. - useEffect(() => { - const syncToNative = () => { - setNativeParam("webcamSize", settings.webcamSizePreset / NATIVE_WEBCAM_BASE_PCT); - setNativeParam("webcamMirror", settings.webcamMirrored); - setNativeParam("webcamShape", settings.webcamMaskShape); - }; - syncToNative(); - return subscribeNativeCompositor(syncToNative); - }, [settings.webcamSizePreset, settings.webcamMirrored, settings.webcamMaskShape]); + // Synchro initiale : cf. NativeCompositorOverlay (`pushAllNativeParams`). // the mask shape picker only makes sense for Picture-in-Picture. // Dual-frame (side-by-side) and vertical-stack (top/bottom) weld the camera // to the screen as one block — the mask is rectangular and sized off the @@ -1496,7 +1465,15 @@ export function LayoutPane() {
{ @@ -1588,23 +1569,7 @@ export function CursorPane() { // Push cursor settings into the native compositor (initial + on view activation); the // handlers below push diffs live. Sizes are sent as direct scales (1 = fixture default). - useEffect(() => { - const syncToNative = () => { - setNativeParam("cursorShow", settings.cursorShow); - setNativeParam("cursorSize", settings.cursor.size); - setNativeParam("cursorClickBounce", settings.cursor.clickBounce); - setNativeParam("cursorSmoothing", settings.cursor.smoothing); - setNativeParam("cursorMotionBlur", settings.cursor.motionBlur); - }; - syncToNative(); - return subscribeNativeCompositor(syncToNative); - }, [ - settings.cursorShow, - settings.cursor.size, - settings.cursor.clickBounce, - settings.cursor.smoothing, - settings.cursor.motionBlur, - ]); + // Synchro initiale : cf. NativeCompositorOverlay (`pushAllNativeParams`). // Built-in "Default" plus each bundled theme. Thumbnails use the theme's // arrow asset; the persisted value is the theme id. Same shape as the diff --git a/src/native/nativeCompositorStore.test.ts b/src/native/nativeCompositorStore.test.ts new file mode 100644 index 000000000..ae19569cd --- /dev/null +++ b/src/native/nativeCompositorStore.test.ts @@ -0,0 +1,118 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { DEFAULT_EDITOR_SETTINGS } from "@/lib/ai-edition/store/editorSettings"; +import type { CompositorParamValue } from "./contracts"; + +const setCompositorParam = vi.fn(() => Promise.resolve()); + +vi.mock("./compositorViewClient", () => ({ + setCompositorParam: (id: number, key: string, value: CompositorParamValue) => + setCompositorParam(id, key, value), + setCompositorPlaying: vi.fn(() => Promise.resolve()), + setCompositorScene: vi.fn(() => Promise.resolve()), + setCompositorTime: vi.fn(() => Promise.resolve()), +})); + +/** + * The full key set the addon understands, transcribed from + * `crates/compositor/src/live.rs` — `set_param_bool` (3), `set_param_num` (9) + * and `set_param_str` (2). Anything the addon accepts and the app never pushes + * silently keeps its compiled-in default, which is exactly the bug this file + * guards: `cursorSize` was only ever pushed by the cursor panel's mount effect, + * and the inspector shows one panel at a time, so a project's cursor settings + * did not reach the preview until the user opened that panel. + */ +const ADDON_KEYS = [ + "backgroundBlur", + "webcamMirror", + "cursorShow", + "shadow", + "roundness", + "motionBlur", + "padding", + "webcamSize", + "cursorSize", + "cursorClickBounce", + "cursorSmoothing", + "cursorMotionBlur", + "backgroundColor", + "webcamShape", +] as const; + +describe("pushAllNativeParams", () => { + beforeEach(() => { + vi.resetModules(); + setCompositorParam.mockClear(); + }); + + async function pushWith(overrides: Record = {}) { + const store = await import("./nativeCompositorStore"); + store.setCurrentNativeViewId(1); + setCompositorParam.mockClear(); + store.pushAllNativeParams({ + ...DEFAULT_EDITOR_SETTINGS, + ...overrides, + } as Parameters[0]); + return new Map( + setCompositorParam.mock.calls.map((c) => [c[1] as string, c[2] as CompositorParamValue]), + ); + } + + it("pushes every key the addon understands", async () => { + // A hex wallpaper so backgroundColor is included; a gradient or image + // wallpaper legitimately travels in the scene instead. + const pushed = await pushWith({ wallpaper: "#101820" }); + const missing = ADDON_KEYS.filter((k) => !pushed.has(k)); + expect(missing, `params the addon accepts but nothing pushes: ${missing.join(", ")}`).toEqual( + [], + ); + }); + + it("sends cursor values RAW, not in slider space", async () => { + // SliderCell displays cursor.size * 10 and divides by 10 on the way out; + // smoothing and motionBlur use * 100. Pushing a slider-space value from + // here would scale the preview by 10 or 100 on load — a regression that + // would look like "the cursor is enormous when I open a project". + const pushed = await pushWith({ + cursor: { size: 3, clickBounce: 2.5, smoothing: 0.67, motionBlur: 0.35 }, + }); + expect(pushed.get("cursorSize")).toBe(3); + expect(pushed.get("cursorClickBounce")).toBe(2.5); + expect(pushed.get("cursorSmoothing")).toBeCloseTo(0.67, 5); + expect(pushed.get("cursorMotionBlur")).toBeCloseTo(0.35, 5); + }); + + it("converts padding and the two base-unit scales", async () => { + const pushed = await pushWith({ + padding: 50, + borderRadius: 48, + webcamSizePreset: 33.4, + }); + expect(pushed.get("padding")).toBeCloseTo(0.5, 5); + // NATIVE_SCREEN_BASE_RADIUS_PX = 24, NATIVE_WEBCAM_BASE_PCT = 16.7. + expect(pushed.get("roundness")).toBeCloseTo(2, 5); + expect(pushed.get("webcamSize")).toBeCloseTo(2, 5); + }); + + it("omits backgroundColor when the wallpaper is not a literal colour", async () => { + const pushed = await pushWith({ wallpaper: "wallpapers/ridges.jpg" }); + expect(pushed.has("backgroundColor")).toBe(false); + }); + + it("memoises so a view activating later still gets everything", async () => { + // The ordering guarantee the fix leans on: params pushed before a view + // exists are replayed on activation, so the overlay needs no viewId guard. + const store = await import("./nativeCompositorStore"); + store.setCurrentNativeViewId(null); + setCompositorParam.mockClear(); + store.pushAllNativeParams({ + ...DEFAULT_EDITOR_SETTINGS, + wallpaper: "#101820", + } as Parameters[0]); + expect(setCompositorParam).not.toHaveBeenCalled(); + + store.setCurrentNativeViewId(7); + const replayed = new Set(setCompositorParam.mock.calls.map((c) => c[1] as string)); + const missing = ADDON_KEYS.filter((k) => !replayed.has(k)); + expect(missing, `not replayed on activation: ${missing.join(", ")}`).toEqual([]); + }); +}); diff --git a/src/native/nativeCompositorStore.ts b/src/native/nativeCompositorStore.ts index 87954939b..65a1e2cc3 100644 --- a/src/native/nativeCompositorStore.ts +++ b/src/native/nativeCompositorStore.ts @@ -4,6 +4,8 @@ * pousser un paramètre via `setNativeParam` sans connaître l'overlay. No-op tant qu'aucune * vue n'est active (pas encore de document/asset chargé, ou addon absent). */ + +import type { EditorSettingsSnapshot } from "@/lib/ai-edition/store/editorSettings"; import { setCompositorParam, setCompositorPlaying, @@ -11,6 +13,7 @@ import { setCompositorTime, } from "./compositorViewClient"; import type { CompositorParamValue } from "./contracts"; +import { NATIVE_SCREEN_BASE_RADIUS_PX, NATIVE_WEBCAM_BASE_PCT } from "./paramUnits"; let currentViewId: number | null = null; const listeners = new Set<() => void>(); @@ -99,3 +102,49 @@ export function subscribeNativeCompositor(listener: () => void): () => void { listeners.delete(listener); }; } + +/** + * Pushes EVERY param the addon understands, from a settings snapshot. + * + * WHY THIS EXISTS. Until this function, each group of params was pushed only by + * the mount effect of the panel that owns it — VideoEffectsPane, LayoutPane, + * CursorPane. The inspector renders exactly ONE panel at a time + * (FloatingInspector's if/else on `facet`, defaulting to "effects"), so on load + * the cursor and layout panels have never mounted and their params sit at the + * addon's compiled-in defaults. That is the whole bug: a project with cursor + * size 3.0 previewed at 1.0 until the user opened the cursor panel, at which + * point it snapped to the right value and looked like the panel had "enabled" + * something. + * + * The `lastParams` replay above is not a substitute: it can only replay keys + * that were pushed at least once, and a key owned by a panel that never mounted + * was never pushed at all. + * + * UNITS. The values here are RAW, matching what the old mount effects sent and + * NOT what the sliders send. `SliderCell` displays `cursor.size * 10` and its + * handler divides by 10 again on the way out; the same asymmetry applies to + * clickBounce (x10) and to smoothing/motionBlur (x100). Sending a slider-space + * value from here would scale the whole preview by 10 or 100 on load. + */ +export function pushAllNativeParams(settings: EditorSettingsSnapshot): void { + setNativeParam("backgroundBlur", settings.showBlur); + setNativeParam("motionBlur", settings.motionBlurAmount); + setNativeParam("shadow", settings.shadowIntensity); + setNativeParam("roundness", settings.borderRadius / NATIVE_SCREEN_BASE_RADIUS_PX); + setNativeParam("padding", settings.padding / 100); + // Only a literal colour is a param; a gradient or an image wallpaper travels + // in the scene instead, and pushing its name here would be meaningless. + if (settings.wallpaper.startsWith("#")) { + setNativeParam("backgroundColor", settings.wallpaper); + } + + setNativeParam("webcamSize", settings.webcamSizePreset / NATIVE_WEBCAM_BASE_PCT); + setNativeParam("webcamMirror", settings.webcamMirrored); + setNativeParam("webcamShape", settings.webcamMaskShape); + + setNativeParam("cursorShow", settings.cursorShow); + setNativeParam("cursorSize", settings.cursor.size); + setNativeParam("cursorClickBounce", settings.cursor.clickBounce); + setNativeParam("cursorSmoothing", settings.cursor.smoothing); + setNativeParam("cursorMotionBlur", settings.cursor.motionBlur); +} diff --git a/src/native/paramUnits.ts b/src/native/paramUnits.ts new file mode 100644 index 000000000..204802cb9 --- /dev/null +++ b/src/native/paramUnits.ts @@ -0,0 +1,26 @@ +/** + * Unit conversions between the editor's settings and the native compositor's + * params. + * + * These two constants used to live inside the panels that pushed them — one + * declared inside `VideoEffectsPane`'s body, the other at RightPanes module + * scope. They are here because `pushAllNativeParams` needs the same numbers, + * and a second copy of a conversion factor is a silent divergence waiting to + * happen: the preview would drift from the panel by exactly the ratio between + * the two values, which is the kind of bug nobody attributes to a constant. + */ + +/** + * The fixture's screen corner radius at 1920 wide. The native `roundness` param + * is a SCALE on it, so dividing the UI's pixel value by the same base makes the + * rendered corner match the pixels shown (dividing by 64, as an earlier version + * did, capped it at ~24px). + */ +export const NATIVE_SCREEN_BASE_RADIUS_PX = 24; + +/** + * The fixture's webcam width as a percentage of the frame (a_side = 320px @1920 + * ≈ 16.7%). `webcamSizePreset / this` is the native size scale, 1 meaning the + * shipped default, so the slider reads as a direct multiplier. + */ +export const NATIVE_WEBCAM_BASE_PCT = 16.7; From 641bd7dd0a6dc4b1eab9b8f645033fd427166cee Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Fri, 31 Jul 2026 21:16:01 +0200 Subject: [PATCH 41/46] fix(capture): record the microphone the user picked, in one mixed track MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two independent bugs made a Linux recording sound empty. Both are fixed here because either alone still leaves silence. WRONG MICROPHONE. The renderer sent no device at all, so the helper passed no PW_KEY_TARGET_OBJECT and PipeWire linked the stream to the session DEFAULT source. On the machine this was found on that is the headphone jack with nothing plugged into it; the user's built-in microphone was a different node. Measured: the recorded track sat at -61 dB mean while the chosen microphone gave -20 dB. The HUD's level meter moved the whole time because that is getUserMedia, which does honour the picker. This was a documented gap, not an oversight — nativeLinuxRecording.ts said the browser deviceId and a PipeWire node.name are unrelated namespaces and left the field empty. Documenting a hole is not filling it, and I shipped it saying audio worked. There is no id to look up, so the helper now enumerates the graph itself (pw_registry, new in pw_audio.c) and matches the label Chromium reports against node.description — which on a PipeWire system is the same string. Exact node name, then exact description, then containment both ways with the longest description winning so the result cannot depend on registry announcement order. No match means no target, i.e. the old default-source behaviour, plus a warning saying so. Verified live: label "Family 17h/19h HD Audio Controller Digital Microphone" resolves to alsa_input.pci-0000_03_00.6.HiFi__hw_acp6x__source. ONE TRACK, NOT TWO. The MP4 carried system audio and the microphone as separate AAC tracks, following macOS on the grounds that the export mixes every track it finds (audio.rs). The preview does not: it plays an HTML5