From 12a11e9dfd134cefcd2a4dfadbb8282bc4724477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Grzesik?= Date: Tue, 23 Apr 2024 13:05:21 +0200 Subject: [PATCH] encoder/v4l2: Add VP8 stateful encoder This commit adds VP8 stateful encoder with additional necessary v4l2 controls. --- src/backend/v4l2/controls.rs | 22 ++ src/encoder.rs | 1 + src/encoder/stateful.rs | 1 + src/encoder/stateful/vp8.rs | 6 + src/encoder/stateful/vp8/v4l2.rs | 369 +++++++++++++++++++++++++++++++ src/encoder/vp8.rs | 27 +++ src/utils.rs | 1 + 7 files changed, 427 insertions(+) create mode 100644 src/encoder/stateful/vp8.rs create mode 100644 src/encoder/stateful/vp8/v4l2.rs create mode 100644 src/encoder/vp8.rs diff --git a/src/backend/v4l2/controls.rs b/src/backend/v4l2/controls.rs index 5c002a47..d36189aa 100644 --- a/src/backend/v4l2/controls.rs +++ b/src/backend/v4l2/controls.rs @@ -458,6 +458,28 @@ impl From for i32 { } } +/// Safe wrapper over [`v4l2r::bindings::V4L2_CID_MPEG_VIDEO_VP8_PROFILE`] +#[allow(unused)] +#[repr(i32)] +#[derive(N, Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum VideoVP8Profile { + Profile0 = v4l2r::bindings::v4l2_mpeg_video_vp8_profile_V4L2_MPEG_VIDEO_VP8_PROFILE_0 as i32, + Profile1 = v4l2r::bindings::v4l2_mpeg_video_vp8_profile_V4L2_MPEG_VIDEO_VP8_PROFILE_1 as i32, + Profile2 = v4l2r::bindings::v4l2_mpeg_video_vp8_profile_V4L2_MPEG_VIDEO_VP8_PROFILE_2 as i32, + Profile3 = v4l2r::bindings::v4l2_mpeg_video_vp8_profile_V4L2_MPEG_VIDEO_VP8_PROFILE_3 as i32, +} + +impl ExtControlTrait for VideoVP8Profile { + const ID: u32 = v4l2r::bindings::V4L2_CID_MPEG_VIDEO_VP8_PROFILE; + type PAYLOAD = i32; +} + +impl From for i32 { + fn from(value: VideoVP8Profile) -> Self { + value as i32 + } +} + /// Safe wrapper over [`v4l2r::bindings::V4L2_CID_MPEG_VIDEO_VP9_PROFILE`] #[allow(unused)] #[repr(i32)] diff --git a/src/encoder.rs b/src/encoder.rs index dff69c12..222899f7 100644 --- a/src/encoder.rs +++ b/src/encoder.rs @@ -7,6 +7,7 @@ use thiserror::Error; pub mod av1; pub mod h264; pub mod h265; +pub mod vp8; pub mod vp9; pub mod stateful; diff --git a/src/encoder/stateful.rs b/src/encoder/stateful.rs index 6c7f4279..cd67af05 100644 --- a/src/encoder/stateful.rs +++ b/src/encoder/stateful.rs @@ -16,6 +16,7 @@ use crate::encoder::VideoEncoder; pub mod h264; pub mod h265; +pub mod vp8; pub mod vp9; #[derive(Debug, Error)] diff --git a/src/encoder/stateful/vp8.rs b/src/encoder/stateful/vp8.rs new file mode 100644 index 00000000..e82a3308 --- /dev/null +++ b/src/encoder/stateful/vp8.rs @@ -0,0 +1,6 @@ +// Copyright 2024 The ChromiumOS Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#[cfg(feature = "v4l2")] +pub mod v4l2; diff --git a/src/encoder/stateful/vp8/v4l2.rs b/src/encoder/stateful/vp8/v4l2.rs new file mode 100644 index 00000000..626cfd2f --- /dev/null +++ b/src/encoder/stateful/vp8/v4l2.rs @@ -0,0 +1,369 @@ +// Copyright 2024 The ChromiumOS Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +use std::sync::Arc; + +use v4l2r::controls::codec::VideoGopSize; +use v4l2r::controls::codec::VideoVP8Profile; +use v4l2r::controls::codec::VideoVPXMaxQp; +use v4l2r::controls::codec::VideoVPXMinQp; +use v4l2r::device::Device; + +use crate::backend::v4l2::encoder::CaptureBuffers; +use crate::backend::v4l2::encoder::ControlError; +use crate::backend::v4l2::encoder::EncoderCodec; +use crate::backend::v4l2::encoder::InitializationError; +use crate::backend::v4l2::encoder::OutputBufferHandle; +use crate::backend::v4l2::encoder::V4L2Backend; +use crate::encoder::stateful::StatefulEncoder; +use crate::encoder::vp8::EncoderConfig; +use crate::encoder::vp8::VP8; +use crate::encoder::PredictionStructure; +use crate::encoder::Tunings; +use crate::Fourcc; +use crate::Resolution; + +const PIXEL_FORMAT_VP8: v4l2r::PixelFormat = v4l2r::PixelFormat::from_fourcc(b"VP80"); + +pub type V4L2VP8Backend = V4L2Backend; + +pub type V4L2StatefulVP8Encoder = + StatefulEncoder>; + +impl EncoderCodec for V4L2VP8Backend +where + Handle: OutputBufferHandle, + CaptureBufferz: CaptureBuffers, +{ + fn apply_tunings(device: &Device, tunings: &Tunings) -> Result<(), ControlError> { + let min_qp = VideoVPXMinQp(tunings.min_quality.clamp(0, 127) as i32); + Self::apply_ctrl(device, "vpx min qp", min_qp)?; + + let max_qp = VideoVPXMaxQp(tunings.max_quality.clamp(0, 127) as i32); + Self::apply_ctrl(device, "vpx max qp", max_qp)?; + + Ok(()) + } +} + +impl V4L2VP8Backend +where + Handle: OutputBufferHandle, + CaptureBufferz: CaptureBuffers, +{ + pub fn new( + device: Arc, + capture_buffers: CaptureBufferz, + config: EncoderConfig, + fourcc: Fourcc, + coded_size: Resolution, + tunings: Tunings, + ) -> Result { + match config.pred_structure { + PredictionStructure::LowDelay { limit } => { + let limit = limit as i32; + + Self::apply_ctrl(&device, "gop size", VideoGopSize(limit))?; + } + } + + // TODO: allow picking profile + Self::apply_ctrl(&device, "vp8 profile", VideoVP8Profile::Profile0)?; + + Self::create( + device, + capture_buffers, + fourcc, + coded_size, + config.resolution, + PIXEL_FORMAT_VP8, + tunings, + ) + } +} + +impl V4L2StatefulVP8Encoder +where + Handle: OutputBufferHandle, + CaptureBufferz: CaptureBuffers, +{ + pub fn new( + device: Arc, + capture_buffers: CaptureBufferz, + config: EncoderConfig, + fourcc: Fourcc, + coded_size: Resolution, + tunings: Tunings, + ) -> Result { + Ok(Self::create( + tunings.clone(), + V4L2VP8Backend::new(device, capture_buffers, config, fourcc, coded_size, tunings)?, + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use std::path::PathBuf; + use std::sync::Arc; + + use v4l2r::device::Device; + use v4l2r::device::DeviceConfig; + + use crate::backend::v4l2::encoder::tests::find_device_with_capture; + use crate::backend::v4l2::encoder::tests::perform_v4l2_encoder_dmabuf_test; + use crate::backend::v4l2::encoder::tests::perform_v4l2_encoder_mmap_test; + use crate::backend::v4l2::encoder::tests::v4l2_format_to_frame_layout; + use crate::backend::v4l2::encoder::tests::BoPoolAllocator; + use crate::backend::v4l2::encoder::tests::GbmDevice; + use crate::backend::v4l2::encoder::MmapingCapture; + use crate::encoder::simple_encode_loop; + use crate::encoder::tests::userptr_test_frame_generator; + use crate::encoder::RateControl; + use crate::utils::DmabufFrame; + use crate::utils::IvfFileHeader; + use crate::utils::IvfFrameHeader; + use crate::Resolution; + + #[ignore] + // Ignore this test by default as it requires v4l2m2m-compatible hardware. + #[test] + fn test_v4l2_encoder_userptr() { + const VISIBLE_SIZE: Resolution = Resolution { + width: 500, + height: 500, + }; + const CODED_SIZE: Resolution = Resolution { + width: 512, + height: 512, + }; + const FRAME_COUNT: u64 = 100; + + let _ = env_logger::try_init(); + + let device = find_device_with_capture(PIXEL_FORMAT_VP8).expect("no VP8 encoder found"); + let device = Device::open(&device, DeviceConfig::new().non_blocking_dqbuf()).expect("open"); + let device = Arc::new(device); + + let mut encoder = V4L2StatefulVP8Encoder::new( + device, + MmapingCapture, + EncoderConfig { + resolution: VISIBLE_SIZE, + ..Default::default() + }, + Fourcc::from(b"NM12"), + CODED_SIZE, + Tunings { + rate_control: RateControl::ConstantBitrate(400_000), + ..Default::default() + }, + ) + .unwrap(); + + let format: v4l2r::Format = encoder.backend().output_format().unwrap(); + let layout = v4l2_format_to_frame_layout(&format); + + let mut bitstream = Vec::new(); + + let file_header = IvfFileHeader::new( + IvfFileHeader::CODEC_VP8, + VISIBLE_SIZE.width as u16, + VISIBLE_SIZE.height as u16, + 30, + FRAME_COUNT as u32, + ); + + file_header.writo_into(&mut bitstream).unwrap(); + + let buffer_size = format + .plane_fmt + .iter() + .map(|plane| plane.sizeimage) + .max() + .unwrap() as usize; + let mut frame_producer = userptr_test_frame_generator(FRAME_COUNT, layout, buffer_size); + + simple_encode_loop(&mut encoder, &mut frame_producer, |coded| { + // Skip Duck IVF frame header from v4l2 stream, and replace with ours with correct + // timestamp. + let frame_bitstream = &coded.bitstream[4 + 8..]; + + let header = IvfFrameHeader { + timestamp: coded.metadata.timestamp, + frame_size: frame_bitstream.len() as u32, + }; + + header.writo_into(&mut bitstream).unwrap(); + bitstream.extend(frame_bitstream); + }) + .expect("encode loop"); + + let write_to_file = std::option_env!("CROS_CODECS_TEST_WRITE_TO_FILE") == Some("true"); + if write_to_file { + use std::io::Write; + let mut out = std::fs::File::create("test_v4l2_encoder_userptr.vp8.ivf").unwrap(); + out.write_all(&bitstream).unwrap(); + out.flush().unwrap(); + } + } + + #[ignore] + // Ignore this test by default as it requires v4l2m2m-compatible hardware. + #[test] + fn test_v4l2_encoder_mmap() { + const VISIBLE_SIZE: Resolution = Resolution { + width: 500, + height: 500, + }; + const CODED_SIZE: Resolution = Resolution { + width: 512, + height: 512, + }; + const FRAME_COUNT: u64 = 100; + + let _ = env_logger::try_init(); + + let device = find_device_with_capture(PIXEL_FORMAT_VP8).expect("no VP8 encoder found"); + let device = Device::open(&device, DeviceConfig::new().non_blocking_dqbuf()).expect("open"); + let device = Arc::new(device); + + let encoder = V4L2StatefulVP8Encoder::new( + device, + MmapingCapture, + EncoderConfig { + resolution: VISIBLE_SIZE, + ..Default::default() + }, + Fourcc::from(b"NM12"), + CODED_SIZE, + Tunings { + rate_control: RateControl::ConstantBitrate(2_000_000), + ..Default::default() + }, + ) + .unwrap(); + + let mut bitstream = Vec::new(); + + let file_header = IvfFileHeader::new( + IvfFileHeader::CODEC_VP8, + VISIBLE_SIZE.width as u16, + VISIBLE_SIZE.height as u16, + 30, + FRAME_COUNT as u32, + ); + + file_header.writo_into(&mut bitstream).unwrap(); + + perform_v4l2_encoder_mmap_test(FRAME_COUNT, encoder, |coded| { + // Skip Duck IVF frame header from v4l2 stream, and replace with ours with correct + // timestamp. + let frame_bitstream = &coded.bitstream[4 + 8..]; + + let header = IvfFrameHeader { + timestamp: coded.metadata.timestamp, + frame_size: frame_bitstream.len() as u32, + }; + + header.writo_into(&mut bitstream).unwrap(); + bitstream.extend(frame_bitstream); + }); + + let write_to_file = std::option_env!("CROS_CODECS_TEST_WRITE_TO_FILE") == Some("true"); + if write_to_file { + use std::io::Write; + let mut out = std::fs::File::create("test_v4l2_encoder_mmap.vp8.ivf").unwrap(); + out.write_all(&bitstream).unwrap(); + out.flush().unwrap(); + } + } + + #[ignore] + // Ignore this test by default as it requires v4l2m2m-compatible hardware. + #[test] + fn test_v4l2_encoder_dmabuf() { + const VISIBLE_SIZE: Resolution = Resolution { + width: 500, + height: 500, + }; + const CODED_SIZE: Resolution = Resolution { + width: 512, + height: 512, + }; + const FRAME_COUNT: u64 = 100; + + let _ = env_logger::try_init(); + + let device = find_device_with_capture(PIXEL_FORMAT_VP8).expect("no VP8 encoder found"); + let device = Device::open(&device, DeviceConfig::new().non_blocking_dqbuf()).expect("open"); + let device = Arc::new(device); + + let gbm = GbmDevice::open(PathBuf::from("/dev/dri/renderD128")) + .and_then(gbm::Device::new) + .expect("failed to create GBM device"); + + let gbm = Arc::new(gbm); + + let encoder = V4L2StatefulVP8Encoder::::new( + device.clone(), + BoPoolAllocator::new(gbm.clone()), + EncoderConfig { + resolution: VISIBLE_SIZE, + ..Default::default() + }, + Fourcc::from(b"NV12"), + CODED_SIZE, + Tunings { + framerate: 30, + rate_control: RateControl::ConstantBitrate(400_000), + ..Default::default() + }, + ) + .unwrap(); + + let mut bitstream = Vec::new(); + + let file_header = IvfFileHeader::new( + IvfFileHeader::CODEC_VP8, + VISIBLE_SIZE.width as u16, + VISIBLE_SIZE.height as u16, + 30, + FRAME_COUNT as u32, + ); + + file_header.writo_into(&mut bitstream).unwrap(); + + perform_v4l2_encoder_dmabuf_test( + CODED_SIZE, + VISIBLE_SIZE, + FRAME_COUNT, + gbm, + encoder, + |coded| { + // Skip Duck IVF frame header from v4l2 stream, and replace with ours with correct + // timestamp. + let frame_bitstream = &coded.bitstream[4 + 8..]; + + let header = IvfFrameHeader { + timestamp: coded.metadata.timestamp, + frame_size: frame_bitstream.len() as u32, + }; + + header.writo_into(&mut bitstream).unwrap(); + bitstream.extend(frame_bitstream); + }, + ); + + let write_to_file = std::option_env!("CROS_CODECS_TEST_WRITE_TO_FILE") == Some("true"); + if write_to_file { + use std::io::Write; + let mut out = std::fs::File::create("test_v4l2_encoder_dmabuf.vp8.ivf").unwrap(); + out.write_all(&bitstream).unwrap(); + out.flush().unwrap(); + } + } +} diff --git a/src/encoder/vp8.rs b/src/encoder/vp8.rs new file mode 100644 index 00000000..fcc827dd --- /dev/null +++ b/src/encoder/vp8.rs @@ -0,0 +1,27 @@ +// Copyright 2024 The ChromiumOS Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +use crate::encoder::PredictionStructure; +use crate::Resolution; + +pub struct VP8; + +#[derive(Clone)] +pub struct EncoderConfig { + pub resolution: Resolution, + pub pred_structure: PredictionStructure, +} + +impl Default for EncoderConfig { + fn default() -> Self { + // Artificially encoder configuration with intent to be widely supported. + Self { + resolution: Resolution { + width: 320, + height: 240, + }, + pred_structure: PredictionStructure::LowDelay { limit: 2048 }, + } + } +} diff --git a/src/utils.rs b/src/utils.rs index cbd66252..72f0bf58 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -108,6 +108,7 @@ impl Default for IvfFileHeader { impl IvfFileHeader { pub const MAGIC: [u8; 4] = *b"DKIF"; + pub const CODEC_VP8: [u8; 4] = *b"VP80"; pub const CODEC_VP9: [u8; 4] = *b"VP90"; pub const CODEC_AV1: [u8; 4] = *b"AV01";