From 1c21e28a0309abf480683af8cd42c01e0da4ea8e Mon Sep 17 00:00:00 2001 From: jazzfool Date: Wed, 30 Oct 2024 20:16:41 +1100 Subject: [PATCH] add subtitle message --- Cargo.lock | 16 +++++++++ Cargo.toml | 1 + src/video.rs | 79 ++++++++++++++++++++++----------------------- src/video_player.rs | 25 ++++++++++++++ 4 files changed, 81 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3c77553..fd610fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1597,6 +1597,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" +[[package]] +name = "html-escape" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d1ad449764d627e22bfd7cd5e8868264fc9236e07c752972b4080cd351cb476" +dependencies = [ + "utf8-width", +] + [[package]] name = "iced" version = "0.13.1" @@ -1731,6 +1740,7 @@ dependencies = [ "gstreamer", "gstreamer-app", "gstreamer-base", + "html-escape", "iced", "iced_wgpu", "log", @@ -3594,6 +3604,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utf8-width" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3" + [[package]] name = "version-compare" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 6d26ed2..55a4955 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ glib = "0.20" # gobject traits and error type log = "0.4" thiserror = "1" url = "2" # media uri +html-escape = "0.2.13" [package.metadata.nix] systems = ["x86_64-linux"] diff --git a/src/video.rs b/src/video.rs index 9464539..b77b1a4 100644 --- a/src/video.rs +++ b/src/video.rs @@ -65,6 +65,8 @@ pub(crate) struct Internal { pub(crate) restart_stream: bool, pub(crate) sync_av_avg: u64, pub(crate) sync_av_counter: u64, + + pub(crate) subtitle_text: Arc>>, } impl Internal { @@ -182,7 +184,7 @@ impl Video { pub fn new(uri: &url::Url) -> Result { gst::init()?; - let pipeline = format!("playbin uri=\"{}\" video-sink=\"videoscale ! videoconvert ! appsink name=iced_video drop=true caps=video/x-raw,format=NV12,pixel-aspect-ratio=1/1\"", uri.as_str()); + let pipeline = format!("playbin uri=\"{}\" text-sink=\"appsink name=iced_text caps=text/x-raw\" video-sink=\"videoscale ! videoconvert ! appsink name=iced_video drop=true caps=video/x-raw,format=NV12,pixel-aspect-ratio=1/1\"", uri.as_str()); let pipeline = gst::parse::launch(pipeline.as_ref())? .downcast::() .map_err(|_| Error::Cast)?; @@ -195,26 +197,34 @@ impl Video { .unwrap() .downcast::() .unwrap(); - let app_sink = bin.by_name("iced_video").unwrap(); - let app_sink = app_sink.downcast::().unwrap(); + let video_sink = bin.by_name("iced_video").unwrap(); + let video_sink = video_sink.downcast::().unwrap(); - Self::from_gst_pipeline(pipeline, app_sink) + let text_sink: gst::Element = pipeline.property("text-sink"); + //let pad = text_sink.pads().get(0).cloned().unwrap(); + let text_sink = text_sink.downcast::().unwrap(); + + Self::from_gst_pipeline(pipeline, video_sink, Some(text_sink)) } /// Creates a new video based on an existing GStreamer pipeline and appsink. /// Expects an `appsink` plugin with `caps=video/x-raw,format=NV12`. /// + /// An optional `text_sink` can be provided, which enables subtitle messages + /// to be emitted. + /// /// **Note:** Many functions of [`Video`] assume a `playbin` pipeline. /// Non-`playbin` pipelines given here may not have full functionality. pub fn from_gst_pipeline( pipeline: gst::Pipeline, - app_sink: gst_app::AppSink, + video_sink: gst_app::AppSink, + text_sink: Option, ) -> Result { gst::init()?; static NEXT_ID: AtomicU64 = AtomicU64::new(0); let id = NEXT_ID.fetch_add(1, Ordering::SeqCst); - let pad = app_sink.pads().first().cloned().unwrap(); + let pad = video_sink.pads().first().cloned().unwrap(); pipeline.set_state(gst::State::Playing)?; @@ -268,15 +278,18 @@ impl Video { let last_frame_time_ref = Arc::clone(&last_frame_time); let paused_ref = Arc::clone(&paused); + let subtitle_text = Arc::new(Mutex::new(None)); + let subtitle_text_ref = Arc::clone(&subtitle_text); + let worker = std::thread::spawn(move || { while alive_ref.load(Ordering::Acquire) { if let Err(gst::FlowError::Error) = (|| -> Result<(), gst::FlowError> { let sample = if paused_ref.load(Ordering::SeqCst) { - app_sink + video_sink .try_pull_preroll(gst::ClockTime::from_mseconds(16)) .ok_or(gst::FlowError::Eos)? } else { - app_sink + video_sink .try_pull_sample(gst::ClockTime::from_mseconds(16)) .ok_or(gst::FlowError::Eos)? }; @@ -294,6 +307,22 @@ impl Video { upload_frame_ref.swap(true, Ordering::SeqCst); + let text = text_sink + .as_ref() + .and_then(|sink| sink.try_pull_sample(gst::ClockTime::from_seconds(0))); + if let Some(text) = text { + let text = text.buffer().ok_or(gst::FlowError::Error)?; + let map = text.map_readable().map_err(|_| gst::FlowError::Error)?; + let text = html_escape::decode_html_entities( + std::str::from_utf8(map.as_slice()) + .map_err(|_| gst::FlowError::Error)?, + ) + .to_string(); + *subtitle_text_ref + .lock() + .map_err(|_| gst::FlowError::Error)? = Some(text); + } + Ok(()) })() { log::error!("error pulling frame"); @@ -325,6 +354,8 @@ impl Video { restart_stream: false, sync_av_avg: 0, sync_av_counter: 0, + + subtitle_text, }))) } @@ -442,38 +473,6 @@ impl Video { url::Url::parse(&self.0.borrow().source.property::("suburi")).ok() } - /// Set the font used to display subtitles. - pub fn set_subtitle_font(&mut self, family: &str, size_pt: u8) { - self.0 - .get_mut() - .source - .set_property("subtitle-font-desc", format!("{}, {}", family, size_pt)); - } - - /// Set whether the subtitle stream is enabled. - pub fn set_subtitles_enabled(&mut self, enabled: bool) { - let source = &self.0.get_mut().source; - let flags = source.property_value("flags"); - let flags_class = glib::FlagsClass::with_type(flags.type_()).unwrap(); - let flags = flags_class.builder_with_value(flags).unwrap(); - let flags = if enabled { - flags.set_by_nick("text") - } else { - flags.unset_by_nick("text") - } - .build() - .unwrap(); - source.set_property_from_value("flags", &flags); - } - - /// Get whether the subtitle stream is enabled. - pub fn subtitles_enabled(&self) -> bool { - let source = &self.0.borrow().source; - let flags = source.property_value("flags"); - let flags_class = glib::FlagsClass::with_type(flags.type_()).unwrap(); - flags_class.is_set_by_nick(&flags, "text") - } - /// Get the underlying GStreamer pipeline. pub fn pipeline(&self) -> gst::Pipeline { self.0.borrow().source.clone() diff --git a/src/video_player.rs b/src/video_player.rs index 60e1fee..8229317 100644 --- a/src/video_player.rs +++ b/src/video_player.rs @@ -20,6 +20,7 @@ where height: iced::Length, on_end_of_stream: Option, on_new_frame: Option, + on_subtitle_text: Option Message + 'a>>, on_error: Option Message + 'a>>, _phantom: PhantomData<(Theme, Renderer)>, } @@ -37,6 +38,7 @@ where height: iced::Length::Shrink, on_end_of_stream: None, on_new_frame: None, + on_subtitle_text: None, on_error: None, _phantom: Default::default(), } @@ -82,6 +84,18 @@ where } } + /// Message to send when the video receives a new frame. + pub fn on_subtitle_text(self, on_subtitle_text: F) -> Self + where + F: 'a + Fn(String) -> Message, + { + VideoPlayer { + on_subtitle_text: Some(Box::new(on_subtitle_text)), + ..self + } + } + + /// Message to send when the video playback encounters an error. pub fn on_error(self, on_error: F) -> Self where F: 'a + Fn(&glib::Error) -> Message, @@ -257,6 +271,17 @@ where shell .request_redraw(iced::window::RedrawRequest::At(std::time::Instant::now())); } + + if let Some(on_subtitle_text) = &self.on_subtitle_text { + if let Some(text) = inner + .subtitle_text + .try_lock() + .ok() + .and_then(|mut text| text.take()) + { + shell.publish(on_subtitle_text(text)); + } + } } Status::Captured } else {