Skip to content

Commit

Permalink
feat!: Add audio and subtitle stream parsing
Browse files Browse the repository at this point in the history
* Refactor code into common processing for all streams and spezialized
  functions for audio and video.
* Update code examples
* Change line breaks in lib.rs to LF
* Add handling of streams that contain additional identifiers in square
  brackets such as `Stream #0:1[0x2](eng):`. These are found in mov
  containers.
  • Loading branch information
Alexey Abel committed Oct 15, 2024
1 parent 701b7df commit 3e3d3db
Show file tree
Hide file tree
Showing 8 changed files with 383 additions and 140 deletions.
21 changes: 17 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -88,12 +88,12 @@ Read raw video frames.
```rust
use ffmpeg_sidecar::{command::FfmpegCommand, event::FfmpegEvent};

fn main() -> anyhow::Result<()> {
fn main() -> anyhow::Result<()> {
FfmpegCommand::new() // <- Builder API like `std::process::Command`
.testsrc() // <- Discoverable aliases for FFmpeg args
.testsrc() // <- Discoverable aliases for FFmpeg args
.rawvideo() // <- Convenient argument presets
.spawn()? // <- Uses an ordinary `std::process::Child`
.iter()? // <- Iterator over all log messages and video output
.spawn()? // <- Uses an ordinary `std::process::Child`
.iter()? // <- Iterator over all log messages and video output
.for_each(|event: FfmpegEvent| {
match event {
FfmpegEvent::OutputFrame(frame) => {
Expand All @@ -106,6 +106,19 @@ fn main() -> anyhow::Result<()> {
FfmpegEvent::Log(_level, msg) => {
eprintln!("[ffmpeg] {}", msg); // <- granular log message from stderr
}
FfmpegEvent::ParsedInputStream(stream) => {
if stream.is_video() {
let video_data = stream.video_data();
println!(
"Found video stream with index {} in input {} that has fps {}, width {}px, height {}px.",
stream.stream_index,
stream.parent_index,
video_data.fps,
video_data.width,
video_data.height
);
}
}
_ => {}
}
});
Expand Down
13 changes: 13 additions & 0 deletions examples/hello_world.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,19 @@ fn main() -> anyhow::Result<()> {
FfmpegEvent::Log(_level, msg) => {
eprintln!("[ffmpeg] {}", msg); // <- granular log message from stderr
}
FfmpegEvent::ParsedInputStream(stream) => {
if stream.is_video() {
let video_data = stream.video_data();
println!(
"Found video stream with index {} in input {} that has fps {}, width {}px, height {}px.",
stream.stream_index,
stream.parent_index,
video_data.fps,
video_data.width,
video_data.height
);
}
}
_ => {}
}
});
Expand Down
80 changes: 70 additions & 10 deletions src/event.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ pub enum FfmpegEvent {
ParsedStreamMapping(String),
ParsedInput(FfmpegInput),
ParsedOutput(FfmpegOutput),
ParsedInputStream(AVStream),
ParsedOutputStream(AVStream),
ParsedInputStream(Stream),
ParsedOutputStream(Stream),
ParsedDuration(FfmpegDuration),
Log(LogLevel, String),
LogEOF,
Expand Down Expand Up @@ -59,12 +59,76 @@ impl FfmpegOutput {
}
}

/// Represents metadata about a stream.
#[derive(Debug, Clone, PartialEq)]
pub struct AVStream {
/// Typically `video` or `audio`, but might be something else like `data` or `subtitle`.
pub stream_type: String,
/// Corresponds to stream `-f` parameter, e.g. `rawvideo`, `h264`, or `mpegts`
pub struct Stream {
/// Corresponds to stream `-f` parameter, e.g. `rawvideo`, `h264`, `opus` or `srt`.
pub format: String,
// The language of the stream as a three letter code such as `eng`, `ger` or `jpn`.
pub language: String,
/// The index of the input or output that this stream belongs to.
pub parent_index: usize,
/// The index of the stream inside the input.
pub stream_index: usize,
/// The stderr line that this stream was parsed from.
pub raw_log_message: String,
// Data that is specific to a certain stream type.
pub type_specific_data: TypeSpecificData,
}

impl Stream {
pub fn is_audio(&self) -> bool {
matches!(self.type_specific_data, TypeSpecificData::Audio(_))
}
pub fn is_subtitle(&self) -> bool {
matches!(self.type_specific_data, TypeSpecificData::Subtitle())
}
pub fn is_video(&self) -> bool {
matches!(self.type_specific_data, TypeSpecificData::Video(_))
}
pub fn is_other(&self) -> bool {
matches!(self.type_specific_data, TypeSpecificData::Other())
}

pub fn audio_data(&self) -> AudioStream {
if let TypeSpecificData::Audio(audio_stream) = &self.type_specific_data {
audio_stream.clone()
} else {
panic!("This is not an audio stream! Check `stream.is_audio()` before calling this method.");
}
}
pub fn video_data(&self) -> VideoStream {
if let TypeSpecificData::Video(video_stream) = &self.type_specific_data {
video_stream.clone()
} else {
panic!("This is not a video stream! Check `stream.is_video()` before calling this method.");
}
}
}

/// Represents metadata that is specific to a stream, e.g. fields that are only found in audio
/// streams or that are only found in video streams, etc. Storing this in an enum allows function to
/// accept the generic `Stream` type regardless of its actual type (audio, video, ...).
#[derive(Debug, Clone, PartialEq)]
pub enum TypeSpecificData {
Audio(AudioStream),
Video(VideoStream),
Subtitle(),
Other(),
}

/// Represents metadata that is specific to audio streams.
#[derive(Debug, Clone, PartialEq)]
pub struct AudioStream {
/// The sample rate of the audio stream, e.g. 48000 (Hz)
pub sample_rate: usize,
/// The number of channels of the audio stream, e.g. `stereo`, `5.1` or `7.1`
pub channels: String,
}

/// Represents metadata that is specific to video streams.
#[derive(Debug, Clone, PartialEq)]
pub struct VideoStream {
/// Corresponds to stream `-pix_fmt` parameter, e.g. `rgb24`
pub pix_fmt: String,
/// Width in pixels
Expand All @@ -73,10 +137,6 @@ pub struct AVStream {
pub height: u32,
/// Framerate in frames per second
pub fps: f32,
/// The index of the input or output that this stream belongs to
pub parent_index: usize,
/// The stderr line that this stream was parsed from
pub raw_log_message: String,
}

#[derive(Debug, Clone, PartialEq)]
Expand Down
37 changes: 21 additions & 16 deletions src/iter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use anyhow::Context;

use crate::{
child::FfmpegChild,
event::{AVStream, FfmpegEvent, FfmpegOutput, FfmpegProgress, LogLevel, OutputVideoFrame},
event::{FfmpegEvent, FfmpegOutput, FfmpegProgress, LogLevel, OutputVideoFrame, Stream},
log_parser::FfmpegLogParser,
metadata::FfmpegMetadata,
pix_fmt::get_bytes_per_frame,
Expand Down Expand Up @@ -181,34 +181,38 @@ impl Iterator for FfmpegIterator {
pub fn spawn_stdout_thread(
stdout: ChildStdout,
tx: SyncSender<FfmpegEvent>,
output_streams: Vec<AVStream>,
output_streams: Vec<Stream>,
outputs: Vec<FfmpegOutput>,
) -> JoinHandle<()> {
std::thread::spawn(move || {
// Filter streams which are sent to stdout
let stdout_output_streams = output_streams.iter().filter(|stream| {
outputs
.get(stream.parent_index)
.map(|o| o.is_stdout())
.unwrap_or(false)
});
let stdout_output_video_streams = output_streams
.iter()
.filter(|stream| stream.is_video())
.filter(|stream| {
outputs
.get(stream.parent_index)
.map(|o| o.is_stdout())
.unwrap_or(false)
});

// Error on mixing rawvideo and non-rawvideo streams
// TODO: Maybe just revert to chunk mode if this happens?
let any_rawvideo = stdout_output_streams
let any_rawvideo = stdout_output_video_streams
.clone()
.any(|s| s.format == "rawvideo");
let any_non_rawvideo = stdout_output_streams
let any_non_rawvideo = stdout_output_video_streams
.clone()
.any(|s| s.format != "rawvideo");
if any_rawvideo && any_non_rawvideo {
panic!("Cannot mix rawvideo and non-rawvideo streams");
}

// Prepare buffers
let mut buffers = stdout_output_streams
let mut buffers = stdout_output_video_streams
.map(|stream| {
let bytes_per_frame = get_bytes_per_frame(stream);
let video_data = stream.video_data();
let bytes_per_frame = get_bytes_per_frame(&video_data);
let buf_size = match stream.format.as_str() {
"rawvideo" => bytes_per_frame.expect("Should use a known pix_fmt") as usize,

Expand Down Expand Up @@ -240,9 +244,10 @@ pub fn spawn_stdout_thread(
loop {
let i = buffer_index.next().unwrap();
let stream = &output_streams[i];
let video_data = stream.video_data();
let buffer = &mut buffers[i];
let output_frame_num = frame_num / num_buffers;
let timestamp = output_frame_num as f32 / stream.fps;
let timestamp = output_frame_num as f32 / video_data.fps;
frame_num += 1;

// Handle two scenarios:
Expand All @@ -251,9 +256,9 @@ pub fn spawn_stdout_thread(
"rawvideo" => match reader.read_exact(buffer.as_mut_slice()) {
Ok(_) => tx
.send(FfmpegEvent::OutputFrame(OutputVideoFrame {
width: stream.width,
height: stream.height,
pix_fmt: stream.pix_fmt.clone(),
width: video_data.width,
height: video_data.height,
pix_fmt: video_data.pix_fmt.clone(),
output_index: i as u32,
data: buffer.clone(),
frame_num: output_frame_num as u32,
Expand Down
111 changes: 62 additions & 49 deletions src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,49 +1,62 @@
//! Wrap a standalone FFmpeg binary in an intuitive Iterator interface.
//!
//! ## Example
//!
//! ```rust
//! use ffmpeg_sidecar::{command::FfmpegCommand, event::FfmpegEvent};
//!
//! fn main() -> anyhow::Result<()> {
//! FfmpegCommand::new() // <- Builder API like `std::process::Command`
//! .testsrc() // <- Discoverable aliases for FFmpeg args
//! .rawvideo() // <- Convenient argument presets
//! .spawn()? // <- Uses an ordinary `std::process::Child`
//! .iter()? // <- Iterator over all log messages and video output
//! .for_each(|event: FfmpegEvent| {
//! match event {
//! FfmpegEvent::OutputFrame(frame) => {
//! println!("frame: {}x{}", frame.width, frame.height);
//! let _pixels: Vec<u8> = frame.data; // <- raw RGB pixels! 🎨
//! }
//! FfmpegEvent::Progress(progress) => {
//! eprintln!("Current speed: {}x", progress.speed); // <- parsed progress updates
//! }
//! FfmpegEvent::Log(_level, msg) => {
//! eprintln!("[ffmpeg] {}", msg); // <- granular log message from stderr
//! }
//! _ => {}
//! }
//! });
//! Ok(())
//! }
//! ```
//!

#[cfg(test)]
mod test;

pub mod child;
pub mod comma_iter;
pub mod command;
pub mod download;
pub mod event;
pub mod ffprobe;
pub mod iter;
pub mod log_parser;
pub mod metadata;
pub mod paths;
pub mod pix_fmt;
pub mod read_until_any;
pub mod version;
//! Wrap a standalone FFmpeg binary in an intuitive Iterator interface.
//!
//! ## Example
//!
//! ```rust
//! use ffmpeg_sidecar::{command::FfmpegCommand, event::FfmpegEvent};
//!
//!fn main() -> anyhow::Result<()> {
//! FfmpegCommand::new() // <- Builder API like `std::process::Command`
//! .testsrc() // <- Discoverable aliases for FFmpeg args
//! .rawvideo() // <- Convenient argument presets
//! .spawn()? // <- Uses an ordinary `std::process::Child`
//! .iter()? // <- Iterator over all log messages and video output
//! .for_each(|event: FfmpegEvent| {
//! match event {
//! FfmpegEvent::OutputFrame(frame) => {
//! println!("frame: {}x{}", frame.width, frame.height);
//! let _pixels: Vec<u8> = frame.data; // <- raw RGB pixels! 🎨
//! }
//! FfmpegEvent::Progress(progress) => {
//! eprintln!("Current speed: {}x", progress.speed); // <- parsed progress updates
//! }
//! FfmpegEvent::Log(_level, msg) => {
//! eprintln!("[ffmpeg] {}", msg); // <- granular log message from stderr
//! }
//! FfmpegEvent::ParsedInputStream(stream) => {
//! if stream.is_video() {
//! let video_data = stream.video_data();
//! println!(
//! "Found video stream with index {} in input {} that has fps {}, width {}px, height {}px.",
//! stream.stream_index,
//! stream.parent_index,
//! video_data.fps,
//! video_data.width,
//! video_data.height
//! );
//! }
//! }
//! _ => {}
//! }
//! });
//! Ok(())
//! }
//! ```
//!

#[cfg(test)]
mod test;

pub mod child;
pub mod comma_iter;
pub mod command;
pub mod download;
pub mod event;
pub mod ffprobe;
pub mod iter;
pub mod log_parser;
pub mod metadata;
pub mod paths;
pub mod pix_fmt;
pub mod read_until_any;
pub mod version;
Loading

0 comments on commit 3e3d3db

Please sign in to comment.