my own fork

Store `Sample` directly instead of copying out of it.

Cargo.toml

@@ -8,7 +8,7 @@ keywords = ["gui", "iced", "video"]
 categories = ["gui", "multimedia"]
 version = "0.6.0"
 authors = ["jazzfool"]
-edition = "2021"
+edition = "2024"
 resolver = "2"
 license = "MIT OR Apache-2.0"
 exclude = [
@@ -16,8 +16,8 @@ exclude = [
 ]
 
 [dependencies]
-iced = { version = "0.13", features = ["image", "advanced", "wgpu"] }
-iced_wgpu = "0.13"
+# iced = { git = "https://github.com/iced-rs/iced", branch = "master", features = ["image", "advanced", "wgpu"] }
+iced_wgpu = { git = "https://github.com/iced-rs/iced", branch = "master"}
 gstreamer = "0.23"
 gstreamer-app = "0.23" # appsink
 gstreamer-base = "0.23" # basesrc
@@ -25,7 +25,11 @@ glib = "0.20" # gobject traits and error type
 log = "0.4"
 thiserror = "1"
 url = "2" # media uri
-html-escape = "0.2.13"
+
+[dependencies.iced]
+git = "https://github.com/iced-rs/iced"
+branch = "master"
+features = ["wgpu", "image", "advanced", "svg", "canvas", "hot", "debug", "lazy", "tokio"]
 
 [package.metadata.nix]
 systems = ["x86_64-linux"]

flake.lock

@@ -1,69 +1,126 @@
 {
   "nodes": {
-    "devshell": {
-      "locked": {
-        "lastModified": 1629275356,
-        "narHash": "sha256-R17M69EKXP6q8/mNHaK53ECwjFo1pdF+XaJC9Qq8zjg=",
-        "owner": "numtide",
-        "repo": "devshell",
-        "rev": "26f25a12265f030917358a9632cd600b51af1d97",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "devshell",
-        "type": "github"
-      }
-    },
-    "flakeCompat": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1627913399,
-        "narHash": "sha256-hY8g6H2KFL8ownSiFeMOjwPC8P0ueXpCVEbxgda3pko=",
-        "owner": "edolstra",
-        "repo": "flake-compat",
-        "rev": "12c64ca55c1014cdc1b16ed5a804aa8576601ff2",
-        "type": "github"
-      },
-      "original": {
-        "owner": "edolstra",
-        "repo": "flake-compat",
-        "type": "github"
-      }
-    },
-    "nixCargoIntegration": {
+    "fenix": {
+      "inputs": {
+        "nixpkgs": "nixpkgs",
+        "rust-analyzer-src": "rust-analyzer-src"
+      },
+      "locked": {
+        "lastModified": 1755585599,
+        "narHash": "sha256-tl/0cnsqB/Yt7DbaGMel2RLa7QG5elA8lkaOXli6VdY=",
+        "owner": "nix-community",
+        "repo": "fenix",
+        "rev": "6ed03ef4c8ec36d193c18e06b9ecddde78fb7e42",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "fenix",
+        "type": "github"
+      }
+    },
+    "fenix_2": {
       "inputs": {
-        "devshell": "devshell",
         "nixpkgs": [
+          "naersk",
           "nixpkgs"
         ],
-        "rustOverlay": "rustOverlay"
+        "rust-analyzer-src": "rust-analyzer-src_2"
       },
       "locked": {
-        "lastModified": 1629871751,
-        "narHash": "sha256-QjnDg34ApcnjmXlNLnbHswT9OroCPY7Wip6r9Zkgkfo=",
-        "owner": "yusdacra",
-        "repo": "nix-cargo-integration",
-        "rev": "4f164ecad242537d5893426eef02c47c9e5ced59",
+        "lastModified": 1752475459,
+        "narHash": "sha256-z6QEu4ZFuHiqdOPbYss4/Q8B0BFhacR8ts6jO/F/aOU=",
+        "owner": "nix-community",
+        "repo": "fenix",
+        "rev": "bf0d6f70f4c9a9cf8845f992105652173f4b617f",
         "type": "github"
       },
       "original": {
-        "owner": "yusdacra",
-        "repo": "nix-cargo-integration",
+        "owner": "nix-community",
+        "repo": "fenix",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "naersk": {
+      "inputs": {
+        "fenix": "fenix_2",
+        "nixpkgs": "nixpkgs_2"
+      },
+      "locked": {
+        "lastModified": 1752689277,
+        "narHash": "sha256-uldUBFkZe/E7qbvxa3mH1ItrWZyT6w1dBKJQF/3ZSsc=",
+        "owner": "nix-community",
+        "repo": "naersk",
+        "rev": "0e72363d0938b0208d6c646d10649164c43f4d64",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "naersk",
         "type": "github"
       }
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1629618782,
-        "narHash": "sha256-2K8SSXu3alo/URI3MClGdDSns6Gb4ZaW4LET53UWyKk=",
+        "lastModified": 1755186698,
+        "narHash": "sha256-wNO3+Ks2jZJ4nTHMuks+cxAiVBGNuEBXsT29Bz6HASo=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "fbcf476f790d8a217c3eab4e12033dc4a0f6d23c",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1752077645,
+        "narHash": "sha256-HM791ZQtXV93xtCY+ZxG1REzhQenSQO020cu6rHtAPk=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "870959c7fb3a42af1863bed9e1756086a74eb649",
+        "rev": "be9e214982e20b8310878ac2baa063a961c1bdf6",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_3": {
+      "locked": {
+        "lastModified": 1756125398,
+        "narHash": "sha256-XexyKZpf46cMiO5Vbj+dWSAXOnr285GHsMch8FBoHbc=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "3b9f00d7a7bf68acd4c4abb9d43695afb04e03a5",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
         "ref": "nixos-unstable",
         "repo": "nixpkgs",
         "type": "github"
@@ -71,24 +128,58 @@
     },
     "root": {
       "inputs": {
-        "flakeCompat": "flakeCompat",
-        "nixCargoIntegration": "nixCargoIntegration",
-        "nixpkgs": "nixpkgs"
+        "fenix": "fenix",
+        "flake-utils": "flake-utils",
+        "naersk": "naersk",
+        "nixpkgs": "nixpkgs_3"
       }
     },
-    "rustOverlay": {
+    "rust-analyzer-src": {
       "flake": false,
       "locked": {
-        "lastModified": 1629857564,
-        "narHash": "sha256-dClWiHkbaCDaIl520Miri66UOA8OecWbaVTWJBajHyM=",
-        "owner": "oxalica",
-        "repo": "rust-overlay",
-        "rev": "88848c36934318e16c86097f65dbf97a57968d81",
+        "lastModified": 1755504847,
+        "narHash": "sha256-VX0B9hwhJypCGqncVVLC+SmeMVd/GAYbJZ0MiiUn2Pk=",
+        "owner": "rust-lang",
+        "repo": "rust-analyzer",
+        "rev": "a905e3b21b144d77e1b304e49f3264f6f8d4db75",
         "type": "github"
       },
       "original": {
-        "owner": "oxalica",
-        "repo": "rust-overlay",
+        "owner": "rust-lang",
+        "ref": "nightly",
+        "repo": "rust-analyzer",
+        "type": "github"
+      }
+    },
+    "rust-analyzer-src_2": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1752428706,
+        "narHash": "sha256-EJcdxw3aXfP8Ex1Nm3s0awyH9egQvB2Gu+QEnJn2Sfg=",
+        "owner": "rust-lang",
+        "repo": "rust-analyzer",
+        "rev": "591e3b7624be97e4443ea7b5542c191311aa141d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "rust-lang",
+        "ref": "nightly",
+        "repo": "rust-analyzer",
+        "type": "github"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
         "type": "github"
       }
     }

flake.nix

@@ -1,28 +1,100 @@
 {
+  description = "A video widget";
+
   inputs = {
-    flakeCompat = {
-      url = "github:edolstra/flake-compat";
-      flake = false;
-    };
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
-    nixCargoIntegration = {
-      url = "github:yusdacra/nix-cargo-integration";
-      inputs.nixpkgs.follows = "nixpkgs";
-    };
+    nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
+    naersk.url = "github:nix-community/naersk";
+    flake-utils.url = "github:numtide/flake-utils";
+    fenix.url = "github:nix-community/fenix";
   };
 
-  outputs = inputs:
-    inputs.nixCargoIntegration.lib.makeOutputs {
-      root = ./.;
-      overrides = {
-        shell = common: prev: {
-          env = prev.env ++ [
-            {
-              name = "GST_PLUGIN_PATH";
-              value = "${common.pkgs.gst_all_1.gstreamer}:${common.pkgs.gst_all_1.gst-plugins-bad}:${common.pkgs.gst_all_1.gst-plugins-ugly}:${common.pkgs.gst_all_1.gst-plugins-good}:${common.pkgs.gst_all_1.gst-plugins-base}";
-            }
+  outputs = inputs: with inputs;
+    flake-utils.lib.eachDefaultSystem
+      (system:
+        let
+          pkgs = import nixpkgs {
+            inherit system;
+            overlays = [fenix.overlays.default];
+            # overlays = [cargo2nix.overlays.default];
+          };
+          naersk' = pkgs.callPackage naersk {};
+          nbi = with pkgs; [
+            # Rust tools
+            alejandra
+            (pkgs.fenix.stable.withComponents [
+              "cargo"
+              "clippy"
+              "rust-src"
+              "rustc"
+              "rustfmt"
+            ])
+            rust-analyzer
+            vulkan-loader
+            wayland
+            wayland-protocols
+            libxkbcommon
+            pkg-config
+            sccache
           ];
-        };
-      };
-    };
+
+          bi = with pkgs; [
+            gcc
+            stdenv
+            gnumake
+            gdb
+            lldb
+            cmake
+            makeWrapper
+            vulkan-headers
+            vulkan-loader
+            vulkan-tools
+            libGL
+            cargo-flamegraph
+
+            fontconfig
+            glib
+            alsa-lib
+            gst_all_1.gst-libav
+            gst_all_1.gst-plugins-bad
+            gst_all_1.gst-plugins-good
+            gst_all_1.gst-plugins-ugly
+            gst_all_1.gst-plugins-base
+            gst_all_1.gst-plugins-rs
+            gst_all_1.gst-vaapi
+            gst_all_1.gstreamer
+            # podofo
+            # mpv
+            ffmpeg-full
+            # yt-dlp
+
+            just
+            cargo-watch
+          ];
+        in rec
+        {
+          devShell = pkgs.mkShell.override {
+            # stdenv = pkgs.stdenvAdapters.useMoldLinker pkgs.clangStdenv;
+          } {
+            nativeBuildInputs = nbi;
+            buildInputs = bi;
+            LD_LIBRARY_PATH = "$LD_LIBRARY_PATH:${
+              with pkgs;
+              pkgs.lib.makeLibraryPath [
+                pkgs.vulkan-loader
+                pkgs.wayland
+                pkgs.wayland-protocols
+                pkgs.libxkbcommon
+              ]
+            }";
+          };
+          defaultPackage = naersk'.buildPackage {
+            src = ./.;
+          };
+          packages = {
+            default = naersk'.buildPackage {
+              src = ./.;
+            };
+          };
+        }
+      );
 }

src/pipeline.rs

@@ -1,3 +1,4 @@
+use crate::video::Frame;
 use iced_wgpu::primitive::Primitive;
 use iced_wgpu::wgpu;
 use std::{
@@ -383,7 +384,7 @@ impl VideoPipeline {
 pub(crate) struct VideoPrimitive {
     video_id: u64,
     alive: Arc<AtomicBool>,
-    frame: Arc<Mutex<Vec<u8>>>,
+    frame: Arc<Mutex<Frame>>,
     size: (u32, u32),
     upload_frame: bool,
 }
@@ -392,7 +393,7 @@ impl VideoPrimitive {
     pub fn new(
         video_id: u64,
         alive: Arc<AtomicBool>,
-        frame: Arc<Mutex<Vec<u8>>>,
+        frame: Arc<Mutex<Frame>>,
         size: (u32, u32),
         upload_frame: bool,
     ) -> Self {
@@ -423,14 +424,16 @@ impl Primitive for VideoPrimitive {
         let pipeline = storage.get_mut::<VideoPipeline>().unwrap();
 
         if self.upload_frame {
-            pipeline.upload(
-                device,
-                queue,
-                self.video_id,
-                &self.alive,
-                self.size,
-                self.frame.lock().expect("lock frame mutex").as_slice(),
-            );
+            if let Some(readable) = self.frame.lock().expect("lock frame mutex").readable() {
+                pipeline.upload(
+                    device,
+                    queue,
+                    self.video_id,
+                    &self.alive,
+                    self.size,
+                    readable.as_slice(),
+                );
+            }
         }
 
         pipeline.prepare(

src/video.rs

@@ -41,6 +41,19 @@ impl From<u64> for Position {
     }
 }
 
+#[derive(Debug)]
+pub(crate) struct Frame(gst::Sample);
+
+impl Frame {
+    pub fn empty() -> Self {
+        Self(gst::Sample::builder().build())
+    }
+
+    pub fn readable(&self) -> Option<gst::BufferMap<gst::buffer::Readable>> {
+        self.0.buffer().and_then(|x| x.map_readable().ok())
+    }
+}
+
 #[derive(Debug)]
 pub(crate) struct Internal {
     pub(crate) id: u64,
@@ -57,7 +70,7 @@ pub(crate) struct Internal {
     pub(crate) speed: f64,
     pub(crate) sync_av: bool,
 
-    pub(crate) frame: Arc<Mutex<Vec<u8>>>,
+    pub(crate) frame: Arc<Mutex<Frame>>,
     pub(crate) upload_frame: Arc<AtomicBool>,
     pub(crate) last_frame_time: Arc<Mutex<Instant>>,
     pub(crate) looping: bool,
@@ -104,6 +117,9 @@ impl Internal {
             )?,
         };
 
+        *self.subtitle_text.lock().expect("lock subtitle_text") = None;
+        self.upload_text.store(true, Ordering::SeqCst);
+
         Ok(())
     }
 
@@ -189,7 +205,12 @@ impl Drop for Video {
 
         inner.alive.store(false, Ordering::SeqCst);
         if let Some(worker) = inner.worker.take() {
-            worker.join().expect("failed to stop video thread");
+            if let Err(err) = worker.join() {
+                match err.downcast_ref::<String>() {
+                    Some(e) => log::error!("Video thread panicked: {e}"),
+                    None => log::error!("Video thread panicked with unknown reason"),
+                }
+            }
         }
     }
 }
@@ -200,7 +221,7 @@ impl Video {
     pub fn new(uri: &url::Url) -> Result<Self, Error> {
         gst::init()?;
 
-        let pipeline = format!("playbin uri=\"{}\" text-sink=\"appsink name=iced_text sync=true caps=text/x-raw\" video-sink=\"videoscale ! videoconvert ! appsink name=iced_video drop=true caps=video/x-raw,format=NV12,pixel-aspect-ratio=1/1\"", uri.as_str());
+        let pipeline = format!("playbin uri=\"{}\" text-sink=\"appsink name=iced_text sync=true drop=true\" video-sink=\"videoscale ! videoconvert ! appsink name=iced_video drop=true caps=video/x-raw,format=NV12,pixel-aspect-ratio=1/1\"", uri.as_str());
         let pipeline = gst::parse::launch(pipeline.as_ref())?
             .downcast::<gst::Pipeline>()
             .map_err(|_| Error::Cast)?;
@@ -217,7 +238,6 @@ impl Video {
         let video_sink = video_sink.downcast::<gst_app::AppSink>().unwrap();
 
         let text_sink: gst::Element = pipeline.property("text-sink");
-        //let pad = text_sink.pads().get(0).cloned().unwrap();
         let text_sink = text_sink.downcast::<gst_app::AppSink>().unwrap();
 
         Self::from_gst_pipeline(pipeline, video_sink, Some(text_sink))
@@ -288,11 +308,7 @@ impl Video {
         let sync_av = pipeline.has_property("av-offset", None);
 
         // NV12 = 12bpp
-        let frame = Arc::new(Mutex::new(vec![
-            0u8;
-            (width as usize * height as usize * 3)
-                .div_ceil(2)
-        ]));
+        let frame = Arc::new(Mutex::new(Frame::empty()));
         let upload_frame = Arc::new(AtomicBool::new(false));
         let alive = Arc::new(AtomicBool::new(true));
         let last_frame_time = Arc::new(Mutex::new(Instant::now()));
@@ -329,18 +345,20 @@ impl Video {
                         .lock()
                         .map_err(|_| gst::FlowError::Error)? = Instant::now();
 
+                    let frame_segment = sample.segment().cloned().ok_or(gst::FlowError::Error)?;
                     let buffer = sample.buffer().ok_or(gst::FlowError::Error)?;
-                    let pts = buffer.pts().unwrap_or_default();
-                    let map = buffer.map_readable().map_err(|_| gst::FlowError::Error)?;
-
-                    let mut frame = frame_ref.lock().map_err(|_| gst::FlowError::Error)?;
-                    let frame_len = frame.len();
-                    frame.copy_from_slice(&map.as_slice()[..frame_len]);
+                    let frame_pts = buffer.pts().ok_or(gst::FlowError::Error)?;
+                    let frame_duration = buffer.duration().ok_or(gst::FlowError::Error)?;
+                    {
+                        let mut frame_guard =
+                            frame_ref.lock().map_err(|_| gst::FlowError::Error)?;
+                        *frame_guard = Frame(sample);
+                    }
 
                     upload_frame_ref.swap(true, Ordering::SeqCst);
 
                     if let Some(at) = clear_subtitles_at {
-                        if pts >= at {
+                        if frame_pts >= at {
                             *subtitle_text_ref
                                 .lock()
                                 .map_err(|_| gst::FlowError::Error)? = None;
@@ -353,22 +371,39 @@ impl Video {
                         .as_ref()
                         .and_then(|sink| sink.try_pull_sample(gst::ClockTime::from_seconds(0)));
                     if let Some(text) = text {
+                        let text_segment = text.segment().ok_or(gst::FlowError::Error)?;
                         let text = text.buffer().ok_or(gst::FlowError::Error)?;
-                        let pts = text.pts().unwrap_or_default();
-                        let duration = text.duration().unwrap_or(gst::ClockTime::ZERO);
-                        let map = text.map_readable().map_err(|_| gst::FlowError::Error)?;
+                        let text_pts = text.pts().ok_or(gst::FlowError::Error)?;
+                        let text_duration = text.duration().ok_or(gst::FlowError::Error)?;
 
-                        let text = html_escape::decode_html_entities(
-                            std::str::from_utf8(map.as_slice())
-                                .map_err(|_| gst::FlowError::Error)?,
-                        )
-                        .to_string();
-                        *subtitle_text_ref
-                            .lock()
-                            .map_err(|_| gst::FlowError::Error)? = Some(text);
-                        upload_text_ref.store(true, Ordering::SeqCst);
+                        let frame_running_time = frame_segment.to_running_time(frame_pts).value();
+                        let frame_running_time_end = frame_segment
+                            .to_running_time(frame_pts + frame_duration)
+                            .value();
 
-                        clear_subtitles_at = Some(pts + duration);
+                        let text_running_time = text_segment.to_running_time(text_pts).value();
+                        let text_running_time_end = text_segment
+                            .to_running_time(text_pts + text_duration)
+                            .value();
+
+                        // see gst-plugins-base/ext/pango/gstbasetextoverlay.c (gst_base_text_overlay_video_chain)
+                        // as an example of how to correctly synchronize the text+video segments
+                        if text_running_time_end > frame_running_time
+                            && frame_running_time_end > text_running_time
+                        {
+                            let duration = text.duration().unwrap_or(gst::ClockTime::ZERO);
+                            let map = text.map_readable().map_err(|_| gst::FlowError::Error)?;
+
+                            let text = std::str::from_utf8(map.as_slice())
+                                .map_err(|_| gst::FlowError::Error)?
+                                .to_string();
+                            *subtitle_text_ref
+                                .lock()
+                                .map_err(|_| gst::FlowError::Error)? = Some(text);
+                            upload_text_ref.store(true, Ordering::SeqCst);
+
+                            clear_subtitles_at = Some(text_pts + duration);
+                        }
                     }
 
                     Ok(())
@@ -569,15 +604,13 @@ impl Video {
                     while !inner.upload_frame.load(Ordering::SeqCst) {
                         std::hint::spin_loop();
                     }
+                    let frame_guard = inner.frame.lock().map_err(|_| Error::Lock)?;
+                    let frame = frame_guard.readable().ok_or(Error::Lock)?;
+
                     Ok(img::Handle::from_rgba(
                         inner.width as u32 / downscale,
                         inner.height as u32 / downscale,
-                        yuv_to_rgba(
-                            &inner.frame.lock().map_err(|_| Error::Lock)?,
-                            width as _,
-                            height as _,
-                            downscale,
-                        ),
+                        yuv_to_rgba(frame.as_slice(), width as _, height as _, downscale),
                     ))
                 })
                 .collect()

src/video_player.rs

@@ -107,8 +107,8 @@ where
     }
 }
 
-impl<'a, Message, Theme, Renderer> Widget<Message, Theme, Renderer>
-    for VideoPlayer<'a, Message, Theme, Renderer>
+impl<Message, Theme, Renderer> Widget<Message, Theme, Renderer>
+    for VideoPlayer<'_, Message, Theme, Renderer>
 where
     Message: Clone,
     Renderer: PrimitiveRenderer,