From 2bb5833486b8ef57f44f9e7e480882b806969604 Mon Sep 17 00:00:00 2001 From: Chris Spencer Date: Mon, 31 Jul 2023 14:57:18 +0100 Subject: [PATCH] media_codec: Add support for asynchronous notification callbacks --- ndk/CHANGELOG.md | 1 + ndk/src/media/media_codec.rs | 352 +++++++++++++++++++++++++++++++---- 2 files changed, 321 insertions(+), 32 deletions(-) diff --git a/ndk/CHANGELOG.md b/ndk/CHANGELOG.md index 1ed88f93..8f06daa1 100644 --- a/ndk/CHANGELOG.md +++ b/ndk/CHANGELOG.md @@ -15,6 +15,7 @@ - native_window: Add `lock()` to blit raw pixel data. (#404) - hardware_buffer_format: Add `YCbCr_P010` and `R8_UNORM` variants. (#405) - **Breaking:** hardware_buffer_format: Add catch-all variant. (#407) +- **Breaking:** media_codec: Add support for asynchronous notification callbacks. (#410) - Add panic guards to callbacks. (#412) - looper: Add `remove_fd()` to unregister events/callbacks for a file descriptor. (#416) diff --git a/ndk/src/media/media_codec.rs b/ndk/src/media/media_codec.rs index 22871769..051f3dc8 100644 --- a/ndk/src/media/media_codec.rs +++ b/ndk/src/media/media_codec.rs @@ -5,10 +5,12 @@ use crate::media_error::{MediaError, Result}; use crate::native_window::NativeWindow; +use crate::utils::abort_on_panic; use std::{ - ffi::{CStr, CString}, + ffi::{c_char, c_void, CStr, CString}, fmt::Display, mem::MaybeUninit, + pin::Pin, ptr::{self, NonNull}, slice, time::Duration, @@ -42,6 +44,14 @@ impl Default for MediaFormat { } impl MediaFormat { + /// Assumes ownership of `ptr` + /// + /// # Safety + /// `ptr` must be a valid pointer to an Android [`ffi::AMediaFormat`]. + pub unsafe fn from_ptr(ptr: NonNull) -> Self { + Self { inner: ptr } + } + fn as_ptr(&self) -> *mut ffi::AMediaFormat { self.inner.as_ptr() } @@ -216,8 +226,76 @@ impl Drop for MediaFormat { #[derive(Debug)] pub struct MediaCodec { inner: NonNull, + async_notify_callback: Option>>, +} + +pub struct AsyncNotifyCallback { + /// Called when an input buffer becomes available. + /// + /// The specified index is the index of the available input buffer. + pub on_input_available: Option, + + /// Called when an output buffer becomes available. + /// + /// The specified index is the index of the available output buffer. The specified + /// [`BufferInfo`] contains information regarding the available output buffer. + pub on_output_available: Option, + + /// Called when the output format has changed. + /// + /// The specified format contains the new output format. + pub on_format_changed: Option, + + /// Called when the [`MediaCodec`] encountered an error. + /// + /// The specified [`ActionCode`] indicates the possible actions that client can take, and it can + /// be checked by calling [`ActionCode::is_recoverable`] or [`ActionCode::is_transient`]. If + /// both [`ActionCode::is_recoverable`] and [`ActionCode::is_transient`] return [`false`], then + /// the codec error is fatal and the codec must be deleted. The specified detail string may + /// contain more detailed messages about this error. + pub on_error: Option, +} + +impl std::fmt::Debug for AsyncNotifyCallback { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AsyncNotifyCallback") + .field( + "on_input_available", + match &self.on_input_available { + Some(_) => &"Some(_)", + None => &"None", + }, + ) + .field( + "on_output_available", + match &self.on_output_available { + Some(_) => &"Some(_)", + None => &"None", + }, + ) + .field( + "on_format_changed", + match &self.on_format_changed { + Some(_) => &"Some(_)", + None => &"None", + }, + ) + .field( + "on_error", + match &self.on_error { + Some(_) => &"Some(_)", + None => &"None", + }, + ) + .finish() + } } +pub type InputAvailableCallback = Box; +pub type OutputAvailableCallback = Box; +pub type FormatChangedCallback = Box; +pub type ErrorCallback = Box; + impl MediaCodec { fn as_ptr(&self) -> *mut ffi::AMediaCodec { self.inner.as_ptr() @@ -227,6 +305,7 @@ impl MediaCodec { let c_string = CString::new(name).unwrap(); Some(Self { inner: NonNull::new(unsafe { ffi::AMediaCodec_createCodecByName(c_string.as_ptr()) })?, + async_notify_callback: None, }) } @@ -236,6 +315,7 @@ impl MediaCodec { inner: NonNull::new(unsafe { ffi::AMediaCodec_createDecoderByType(c_string.as_ptr()) })?, + async_notify_callback: None, }) } @@ -245,9 +325,150 @@ impl MediaCodec { inner: NonNull::new(unsafe { ffi::AMediaCodec_createEncoderByType(c_string.as_ptr()) })?, + async_notify_callback: None, }) } + /// Set an asynchronous callback for actionable [`MediaCodec`] events. + /// + /// When asynchronous callback is enabled, it is an error for the client to call + /// [`MediaCodec::dequeue_input_buffer()`] or [`MediaCodec::dequeue_output_buffer()`]. + /// + /// [`MediaCodec::flush()`] behaves differently in asynchronous mode. After calling + /// [`MediaCodec::flush()`], the client must call [`MediaCodec::start()`] to "resume" receiving + /// input buffers. Even if the client does not receive + /// [`AsyncNotifyCallback::on_input_available`] callbacks from video encoders configured with an + /// input surface, the client still needs to call [`MediaCodec::start()`] to resume the input + /// surface to send buffers to the encoders. + /// + /// When called with [`None`] callback, this method unregisters any previously set callback. + /// + /// Refer to the definition of [`AsyncNotifyCallback`] on how each callback function is called + /// and what are specified. + /// + /// Once the callback is unregistered or the codec is reset / released, the previously + /// registered callback will not be called. + /// + /// All callbacks are fired on one NDK internal thread. + /// [`MediaCodec::set_async_notify_callback`] should not be called on the callback thread. No + /// heavy duty task should be performed on callback thread. + #[cfg(feature = "api-level-28")] + pub fn set_async_notify_callback( + &mut self, + callback: Option, + ) -> Result<()> { + unsafe extern "C" fn ffi_on_input_available( + _codec: *mut ffi::AMediaCodec, + user_data: *mut c_void, + index: i32, + ) { + abort_on_panic(|| { + let callback = &mut *(user_data as *mut AsyncNotifyCallback); + if let Some(f) = callback.on_input_available.as_mut() { + f(index as usize); + } + }) + } + + unsafe extern "C" fn ffi_on_output_available( + _codec: *mut ffi::AMediaCodec, + user_data: *mut c_void, + index: i32, + buffer_info: *mut ffi::AMediaCodecBufferInfo, + ) { + abort_on_panic(|| { + let callback = &mut *(user_data as *mut AsyncNotifyCallback); + if let Some(f) = callback.on_output_available.as_mut() { + let buffer_info = BufferInfo { + inner: *buffer_info, + }; + f(index as usize, &buffer_info); + } + }) + } + + unsafe extern "C" fn ffi_on_format_changed( + _codec: *mut ffi::AMediaCodec, + user_data: *mut c_void, + format: *mut ffi::AMediaFormat, + ) { + abort_on_panic(|| { + // Ownership of the format is not documented, but the implementation allocates a new instance and does + // not free it, so assume it is ok for us to do so + // https://cs.android.com/android/platform/superproject/main/+/refs/heads/main:frameworks/av/media/ndk/NdkMediaCodec.cpp;l=248-254;drc=5e15c3e22f3fa32d64e57302201123ce41589adf + let format = MediaFormat::from_ptr(NonNull::new_unchecked(format)); + + let callback = &mut *(user_data as *mut AsyncNotifyCallback); + if let Some(f) = callback.on_format_changed.as_mut() { + f(&format); + } + }) + } + + unsafe extern "C" fn ffi_on_error( + _codec: *mut ffi::AMediaCodec, + user_data: *mut c_void, + error: ffi::media_status_t, + action_code: i32, + detail: *const c_char, + ) { + abort_on_panic(|| { + let callback = &mut *(user_data as *mut AsyncNotifyCallback); + if let Some(f) = callback.on_error.as_mut() { + f( + MediaError::from_status(error).unwrap_err(), + ActionCode(action_code), + CStr::from_ptr(detail), + ); + } + }) + } + + let (callback, ffi_callback, user_data) = if let Some(callback) = callback { + // On Android 12 and earlier, due to faulty null checks, if a callback is not set, but at least one other + // callback *is* set, then it will segfault in when trying to invoke the unset callback. See for example: + // https://cs.android.com/android/platform/superproject/+/android-12.0.0_r34:frameworks/av/media/ndk/NdkMediaCodec.cpp;l=161-162;drc=ef058464777739e2d9ffad5f00d0e57b186d9a13 + // To work around this we just enable all callbacks and do nothing if the corresponding callback is not set + // in AsyncNotifyCallback + let ffi_callback = ffi::AMediaCodecOnAsyncNotifyCallback { + onAsyncInputAvailable: Some(ffi_on_input_available), + onAsyncOutputAvailable: Some(ffi_on_output_available), + onAsyncFormatChanged: Some(ffi_on_format_changed), + onAsyncError: Some(ffi_on_error), + }; + + let mut boxed = Box::pin(callback); + let ptr: *mut AsyncNotifyCallback = &mut *boxed; + + (Some(boxed), ffi_callback, ptr as *mut c_void) + } else { + let ffi_callback = ffi::AMediaCodecOnAsyncNotifyCallback { + onAsyncInputAvailable: None, + onAsyncOutputAvailable: None, + onAsyncFormatChanged: None, + onAsyncError: None, + }; + + (None, ffi_callback, ptr::null_mut()) + }; + + let status = unsafe { + ffi::AMediaCodec_setAsyncNotifyCallback(self.as_ptr(), ffi_callback, user_data) + }; + let result = MediaError::from_status(status); + + // This behavior is not documented, but the implementation always clears the callback on failure, so we must + // clear any callback that may have been previously registered + // https://cs.android.com/android/platform/superproject/main/+/main:frameworks/av/media/ndk/NdkMediaCodec.cpp;l=559;drc=8c4e619c7461ac1a8c20c55364643662e9185e4d + if result.is_ok() { + self.async_notify_callback = callback; + } else { + self.async_notify_callback = None; + } + + result + } + pub fn configure( &self, format: &MediaFormat, @@ -341,7 +562,9 @@ impl MediaCodec { Ok(DequeuedOutputBufferInfoResult::Buffer(OutputBuffer { codec: self, index: result as usize, - info: unsafe { info.assume_init() }, + info: BufferInfo { + inner: unsafe { info.assume_init() }, + }, })) } else { Err(MediaError::from_status(ffi::media_status_t(result as _)).unwrap_err()) @@ -353,6 +576,28 @@ impl MediaCodec { MediaError::from_status(status) } + pub fn input_buffer(&self, index: usize) -> Option<&mut [MaybeUninit]> { + unsafe { + let mut out_size = 0; + let buffer_ptr = ffi::AMediaCodec_getInputBuffer(self.as_ptr(), index, &mut out_size); + if buffer_ptr.is_null() { + return None; + } + Some(slice::from_raw_parts_mut(buffer_ptr.cast(), out_size)) + } + } + + pub fn output_buffer(&self, index: usize) -> Option<&[u8]> { + unsafe { + let mut out_size = 0; + let buffer_ptr = ffi::AMediaCodec_getOutputBuffer(self.as_ptr(), index, &mut out_size); + if buffer_ptr.is_null() { + return None; + } + Some(slice::from_raw_parts(buffer_ptr, out_size)) + } + } + #[cfg(feature = "api-level-28")] pub fn input_format(&self) -> MediaFormat { let inner = NonNull::new(unsafe { ffi::AMediaCodec_getInputFormat(self.as_ptr()) }) @@ -385,11 +630,23 @@ impl MediaCodec { size: usize, time: u64, flags: u32, + ) -> Result<()> { + debug_assert!(ptr::eq(self, buffer.codec)); + self.queue_input_buffer_by_index(buffer.index, offset, size, time, flags) + } + + pub fn queue_input_buffer_by_index( + &self, + buffer_index: usize, + offset: usize, + size: usize, + time: u64, + flags: u32, ) -> Result<()> { let status = unsafe { ffi::AMediaCodec_queueInputBuffer( self.as_ptr(), - buffer.index, + buffer_index, offset as ffi::off_t, size, time, @@ -400,8 +657,12 @@ impl MediaCodec { } pub fn release_output_buffer(&self, buffer: OutputBuffer<'_>, render: bool) -> Result<()> { + self.release_output_buffer_by_index(buffer.index, render) + } + + pub fn release_output_buffer_by_index(&self, buffer_index: usize, render: bool) -> Result<()> { let status = - unsafe { ffi::AMediaCodec_releaseOutputBuffer(self.as_ptr(), buffer.index, render) }; + unsafe { ffi::AMediaCodec_releaseOutputBuffer(self.as_ptr(), buffer_index, render) }; MediaError::from_status(status) } @@ -409,9 +670,17 @@ impl MediaCodec { &self, buffer: OutputBuffer<'_>, timestamp_ns: i64, + ) -> Result<()> { + self.release_output_buffer_at_time_by_index(buffer.index, timestamp_ns) + } + + pub fn release_output_buffer_at_time_by_index( + &self, + buffer_index: usize, + timestamp_ns: i64, ) -> Result<()> { let status = unsafe { - ffi::AMediaCodec_releaseOutputBufferAtTime(self.as_ptr(), buffer.index, timestamp_ns) + ffi::AMediaCodec_releaseOutputBufferAtTime(self.as_ptr(), buffer_index, timestamp_ns) }; MediaError::from_status(status) } @@ -467,17 +736,12 @@ pub struct InputBuffer<'a> { impl InputBuffer<'_> { pub fn buffer_mut(&mut self) -> &mut [MaybeUninit] { - unsafe { - let mut out_size = 0; - let buffer_ptr = - ffi::AMediaCodec_getInputBuffer(self.codec.as_ptr(), self.index, &mut out_size); - assert!( - !buffer_ptr.is_null(), + self.codec.input_buffer(self.index).unwrap_or_else(|| { + panic!( "AMediaCodec_getInputBuffer returned NULL for index {}", self.index - ); - slice::from_raw_parts_mut(buffer_ptr.cast(), out_size) - } + ) + }) } } @@ -491,25 +755,17 @@ pub enum DequeuedInputBufferResult<'a> { pub struct OutputBuffer<'a> { codec: &'a MediaCodec, index: usize, - info: ffi::AMediaCodecBufferInfo, + info: BufferInfo, } impl OutputBuffer<'_> { pub fn buffer(&self) -> &[u8] { - unsafe { - let mut _out_size = 0; - let buffer_ptr = - ffi::AMediaCodec_getOutputBuffer(self.codec.as_ptr(), self.index, &mut _out_size); - assert!( - !buffer_ptr.is_null(), + self.codec.output_buffer(self.index).unwrap_or_else(|| { + panic!( "AMediaCodec_getOutputBuffer returned NULL for index {}", self.index - ); - slice::from_raw_parts( - buffer_ptr.add(self.info.offset as usize), - self.info.size as usize, ) - } + }) } #[cfg(feature = "api-level-28")] @@ -521,12 +777,8 @@ impl OutputBuffer<'_> { MediaFormat { inner } } - pub fn flags(&self) -> u32 { - self.info.flags - } - - pub fn presentation_time_us(&self) -> i64 { - self.info.presentationTimeUs + pub fn info(&self) -> &BufferInfo { + &self.info } } @@ -537,3 +789,39 @@ pub enum DequeuedOutputBufferInfoResult<'a> { OutputFormatChanged, OutputBuffersChanged, } + +#[derive(Copy, Clone, Debug)] +pub struct BufferInfo { + inner: ffi::AMediaCodecBufferInfo, +} + +impl BufferInfo { + pub fn offset(&self) -> i32 { + self.inner.offset + } + + pub fn size(&self) -> i32 { + self.inner.size + } + + pub fn presentation_time_us(&self) -> i64 { + self.inner.presentationTimeUs + } + + pub fn flags(&self) -> u32 { + self.inner.flags + } +} + +#[derive(Copy, Clone, Debug)] +pub struct ActionCode(pub i32); + +impl ActionCode { + pub fn is_recoverable(self) -> bool { + unsafe { ffi::AMediaCodecActionCode_isRecoverable(self.0) } + } + + pub fn is_transient(self) -> bool { + unsafe { ffi::AMediaCodecActionCode_isTransient(self.0) } + } +}