From 0ee7557288463d800e71dfe8051844e3891719fe Mon Sep 17 00:00:00 2001 From: Chris Pearce Date: Thu, 7 Dec 2017 14:51:13 +1300 Subject: [PATCH 1/4] Make PlanarYCbCrImage hold pixel data in an Arc>. This patch copies video frames coming out of Gecko/C++ and stores the result in an Arc>. In a later commit, I'll make the pixel data shared with C++, instead of being copied on every update_current_images() call. --- examples/src/bin/test-player.rs | 2 +- gecko-media/gecko/glue/ImageContainer.cpp | 24 ++++++++ gecko-media/gecko/glue/include/Player.h | 6 ++ gecko-media/src/player.rs | 73 +++++++++++------------ 4 files changed, 65 insertions(+), 40 deletions(-) diff --git a/examples/src/bin/test-player.rs b/examples/src/bin/test-player.rs index 36066e5..24d8c90 100644 --- a/examples/src/bin/test-player.rs +++ b/examples/src/bin/test-player.rs @@ -272,7 +272,7 @@ impl webrender::ExternalImageHandler for ImageGenerator { source: webrender::ExternalImageSource::RawData( self.current_image .as_ref() - .map(|p| p.pixel_data(channel_index)) + .map(|img| img.plane_for_channel(channel_index).pixels) .unwrap(), ), } diff --git a/gecko-media/gecko/glue/ImageContainer.cpp b/gecko-media/gecko/glue/ImageContainer.cpp index 26ce142..296da88 100644 --- a/gecko-media/gecko/glue/ImageContainer.cpp +++ b/gecko-media/gecko/glue/ImageContainer.cpp @@ -430,6 +430,29 @@ PlanarYCbCrImage_GetPixelData(uint32_t aFrameID, PlaneType aPlaneType) return nullptr; } +ByteSlice +PlanarYCbCrImage_GetSliceData(uint32_t aFrameID) +{ + RefPtr img; + { + StaticMutexAutoLock lock(sImageMutex); + ExportedImage* i = sImages.Get(aFrameID); + if (!i) { + return ByteSlice { nullptr, 0 }; + } + img = i->mImage; + } + PlanarYCbCrImage* planarImage = img->AsPlanarYCbCrImage(); + MOZ_ASSERT(planarImage); + if (!planarImage) { + return ByteSlice { nullptr, 0 }; + } + const PlanarYCbCrData* data = planarImage->GetData(); + const uint8_t* ptr = data->mYChannel; + size_t length = planarImage->GetDataSize(); + return ByteSlice { ptr, length }; +} + void ImageContainer::DeallocateExportedImages() { @@ -509,6 +532,7 @@ ImageContainer::NotifyOwnerOfNewImages() img->mAddRefPixelData = &PlanarYCbCrImage_AddRefPixelData; img->mFreePixelData = &PlanarYCbCrImage_FreeData; img->mGetPixelData = &PlanarYCbCrImage_GetPixelData; + img->mGetSliceData = &PlanarYCbCrImage_GetSliceData; } RefPtr task = new UpdateCurrentImagesRunnable(mOwner, Move(images)); diff --git a/gecko-media/gecko/glue/include/Player.h b/gecko-media/gecko/glue/include/Player.h index c67e58a..570009b 100644 --- a/gecko-media/gecko/glue/include/Player.h +++ b/gecko-media/gecko/glue/include/Player.h @@ -32,6 +32,11 @@ enum PlaneType Cr }; +struct ByteSlice { + const uint8_t* mData; + size_t mLength; +}; + struct GeckoPlanarYCbCrImage { // Luminance buffer @@ -59,6 +64,7 @@ struct GeckoPlanarYCbCrImage void (*mAddRefPixelData)(uint32_t aFrameID); void (*mFreePixelData)(uint32_t aFrameID); const uint8_t* (*mGetPixelData)(uint32_t aFrameID, PlaneType aPlaneType); + ByteSlice (*mGetSliceData)(uint32_t aFrameID); }; struct PlayerCallbackObject diff --git a/gecko-media/src/player.rs b/gecko-media/src/player.rs index 21b10a2..58b3676 100644 --- a/gecko-media/src/player.rs +++ b/gecko-media/src/player.rs @@ -16,6 +16,7 @@ use std::mem; use std::ops::Range; use std::os::raw::c_char; use std::slice; +use std::sync::Arc; use timestamp::TimeStamp; /// Holds useful metadata extracted from a media resource during loading. @@ -28,10 +29,10 @@ pub struct Metadata { pub video_dimensions: Option<(i32, i32)>, } -/// Holds pixel data and coordinates of a non-interleaved plane of data. -/// -pub struct Plane { - pub pixels: *const u8, +/// Holds pixel data and geometry of a non-interleaved plane of data. +pub struct Plane<'a> { + /// Pixel sample data. One byte per sample. Will contain stride*height byte. + pub pixels: &'a [u8], /// The width of a line of pixels in bytes. pub width: i32, /// The stride of a line of pixels in bytes. @@ -40,16 +41,6 @@ pub struct Plane { pub height: i32, } -impl Plane { - /// Returns a slice storing the raw pixel data. - pub fn data<'a>(&'a self) -> &'a [u8] { - unsafe { - let size = self.stride as usize * self.height as usize; - slice::from_raw_parts(self.pixels, size) - } - } -} - /// A subregion of an image buffer. #[derive(Clone)] pub struct Region { @@ -79,11 +70,12 @@ pub struct Region { pub struct PlanarYCbCrImage { /// The sub-region of the buffer which contains the image to be rendered. pub picture: Region, - /// The time at which this image should be renderd. + /// The time at which this image should be rendered. pub time_stamp: TimeStamp, /// A stream-unique identifier. pub frame_id: u32, - pub gecko_image: GeckoPlanarYCbCrImage, + gecko_image: GeckoPlanarYCbCrImage, + pixel_buffer: Arc>, } // When cloning, we need to ensure we increment the reference count on @@ -99,41 +91,38 @@ impl Clone for PlanarYCbCrImage { time_stamp: self.time_stamp.clone(), frame_id: self.frame_id, gecko_image: self.gecko_image, + pixel_buffer: self.pixel_buffer.clone(), } } } impl PlanarYCbCrImage { - /// Returns a slice storing the raw pixel data. - pub fn pixel_data<'a>(&'a self, channel_index: u8) -> &'a [u8] { + fn pixel_slice_for_channel<'a>(&'a self, channel_index: u8) -> &'a [u8] { let img = &self.gecko_image; - let (pixels, size) = match channel_index { - 0 => ( - self.get_pixels(PlaneType_Y), - img.mYStride as usize * img.mYHeight as usize, - ), - 1 => ( - self.get_pixels(PlaneType_Cb), - img.mCbCrStride as usize * img.mCbCrHeight as usize, - ), - 2 => ( - self.get_pixels(PlaneType_Cr), - img.mCbCrStride as usize * img.mCbCrHeight as usize, - ), + let y_plane_size = (img.mYStride * img.mYHeight) as usize; + let cbcr_plane_size = (img.mCbCrStride * img.mCbCrHeight) as usize; + let (start, size) = match channel_index { + 0 => (0, y_plane_size), + 1 => (y_plane_size, cbcr_plane_size), + 2 => (y_plane_size + cbcr_plane_size, cbcr_plane_size), _ => panic!("Invalid channel_index"), }; - unsafe { slice::from_raw_parts(pixels, size) } + &self.pixel_buffer[start..(start + size)] } - fn get_pixels(&self, plane: u32) -> *const u8 { - let img = &self.gecko_image; - let f = img.mGetPixelData.unwrap(); - unsafe { f(img.mFrameID, plane) as *const u8 } + pub fn plane_for_channel(&self, channel_index: u8) -> Plane { + match channel_index { + 0 => self.y_plane(), + 1 => self.cb_plane(), + 2 => self.cr_plane(), + _ => panic!("Invalid planar index"), + } } + pub fn y_plane(&self) -> Plane { let img = &self.gecko_image; Plane { - pixels: self.get_pixels(PlaneType_Y), + pixels: self.pixel_slice_for_channel(0), width: img.mYWidth, stride: img.mYStride, height: img.mYHeight, @@ -143,7 +132,7 @@ impl PlanarYCbCrImage { pub fn cb_plane(&self) -> Plane { let img = &self.gecko_image; Plane { - pixels: self.get_pixels(PlaneType_Cb), + pixels: self.pixel_slice_for_channel(1), width: img.mCbCrWidth, stride: img.mCbCrStride, height: img.mCbCrHeight, @@ -153,7 +142,7 @@ impl PlanarYCbCrImage { pub fn cr_plane(&self) -> Plane { let img = &self.gecko_image; Plane { - pixels: self.get_pixels(PlaneType_Cr), + pixels: self.pixel_slice_for_channel(2), width: img.mCbCrWidth, stride: img.mCbCrStride, height: img.mCbCrHeight, @@ -529,6 +518,11 @@ fn to_ffi_planar_ycbycr_images(size: usize, elements: *mut GeckoPlanarYCbCrImage elements .iter() .map(|&img| -> PlanarYCbCrImage { + let byte_slice = unsafe { + let f = img.mGetSliceData.unwrap(); + f(img.mFrameID) + }; + let pixels = unsafe { slice::from_raw_parts(byte_slice.mData, byte_slice.mLength) }; PlanarYCbCrImage { picture: Region { x: img.mPicX, @@ -539,6 +533,7 @@ fn to_ffi_planar_ycbycr_images(size: usize, elements: *mut GeckoPlanarYCbCrImage time_stamp: TimeStamp(img.mTimeStamp), frame_id: img.mFrameID, gecko_image: img, + pixel_buffer: Arc::new(pixels.to_vec()), } }) .collect::>() From 92b329680d4a70a5905334ee7776400c8a445813 Mon Sep 17 00:00:00 2001 From: Chris Pearce Date: Tue, 19 Dec 2017 16:18:02 +1300 Subject: [PATCH 2/4] Use Rust allocated Vec to store images. We create a VideoFrameAllocator that stores a list of refcounted Vecs in which to store data. Each has a handle. When C++ wants to create a new image, it calls into Rust, which creates a new Vec and copies the image into it, and returns the handle to C++. The C++ code will associate the handle with other frame metadata such as timestamp, and pass the handle back to Rust code when the image is to be composited. This should mean that Rust controls the lifetimes of images, as they're stored in ref counted Rust memory. --- examples/src/bin/test-player.rs | 15 +- .../gecko/glue/GeckoMediaDecoderOwner.cpp | 27 +- gecko-media/gecko/glue/ImageContainer.cpp | 339 ++++++++--------- gecko-media/gecko/glue/MediaData.cpp | 7 + gecko-media/gecko/glue/Player.cpp | 29 +- .../glue/include/GeckoMediaDecoderOwner.h | 5 +- .../gecko/glue/include/ImageContainer.h | 13 +- gecko-media/gecko/glue/include/Player.h | 69 ++-- gecko-media/src/player.rs | 340 +++++++++++------- 9 files changed, 458 insertions(+), 386 deletions(-) diff --git a/examples/src/bin/test-player.rs b/examples/src/bin/test-player.rs index 24d8c90..5317d0f 100644 --- a/examples/src/bin/test-player.rs +++ b/examples/src/bin/test-player.rs @@ -272,7 +272,7 @@ impl webrender::ExternalImageHandler for ImageGenerator { source: webrender::ExternalImageSource::RawData( self.current_image .as_ref() - .map(|img| img.plane_for_channel(channel_index).pixels) + .map(|img| &img.plane_for_channel(channel_index).pixels) .unwrap(), ), } @@ -320,7 +320,7 @@ impl App { api: &RenderApi, resources: &mut ResourceUpdates, image_key: Option, - plane: Plane, + plane: &Plane, channel_index: u8, ) -> Option { match image_key { @@ -452,13 +452,13 @@ impl ui::Example for App { } let time_now = TimeStamp(time::precise_time_ns()); - while self.frame_queue.len() > 1 && self.frame_queue[0].time_stamp > time_now { + while self.frame_queue.len() > 1 && self.frame_queue[0].timestamp() > time_now { self.frame_queue.remove(0); } if let Some(first_frame) = self.frame_queue.first() { - if self.last_frame_id != first_frame.frame_id { - self.last_frame_id = first_frame.frame_id; + if self.last_frame_id != first_frame.frame_id() { + self.last_frame_id = first_frame.frame_id(); self.current_frame_sender.as_ref().map(|sender| { sender.send(first_frame.clone()).unwrap(); }); @@ -513,7 +513,10 @@ impl ui::Example for App { // intrinsic size, we'll just use the frame size. let (width, height) = match self.video_dimensions() { Some((width, height)) => (width, height), - None => (frame.picture.width, frame.picture.height), + None => ( + frame.picture_region().width as i32, + frame.picture_region().height as i32, + ), }; // Resize so that the video is rendered as wide as the window. diff --git a/gecko-media/gecko/glue/GeckoMediaDecoderOwner.cpp b/gecko-media/gecko/glue/GeckoMediaDecoderOwner.cpp index 5fa4db1..9ae8d9c 100644 --- a/gecko-media/gecko/glue/GeckoMediaDecoderOwner.cpp +++ b/gecko-media/gecko/glue/GeckoMediaDecoderOwner.cpp @@ -5,20 +5,21 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "GeckoMediaDecoderOwner.h" -#include "mozilla/AbstractThread.h" -#include "VideoFrameContainer.h" #include "ImageContainer.h" +#include "VideoFrameContainer.h" +#include "mozilla/AbstractThread.h" namespace mozilla { -GeckoMediaDecoderOwner::GeckoMediaDecoderOwner(PlayerCallbackObject aCallback) +GeckoMediaDecoderOwner::GeckoMediaDecoderOwner(PlayerCallbackObject aCallback, + FrameAllocatorObject aAllocator) : mCallback(aCallback) + , mAllocator(aAllocator) { } GeckoMediaDecoderOwner::GeckoMediaDecoderOwner() { - } void @@ -213,9 +214,9 @@ VideoFrameContainer* GeckoMediaDecoderOwner::GetVideoFrameContainer() { RefPtr container = - new layers::ImageContainer(this); - mVideoFrameContainer = - new VideoFrameContainer(this, container.forget()); + new layers::ImageContainer(this, mAllocator); + mAllocator = { 0 }; + mVideoFrameContainer = new VideoFrameContainer(this, container.forget()); return mVideoFrameContainer; } @@ -285,7 +286,7 @@ GeckoMediaDecoderOwner::NotifySeekable() const i++; } (*mCallback.mNotifySeekable)(mCallback.mContext, size, ranges); - delete [] ranges; + delete[] ranges; } } @@ -293,14 +294,6 @@ void GeckoMediaDecoderOwner::Shutdown() { if (mVideoFrameContainer) { - // The ImageContainer keeps a list of the images that it sends out to Rust, - // so that if we shutdown, we can deallocate and neuter the images - // proactively. If we don't do this, we can end up with crashes if Rust - // code on another thread tries to use images after we've shutdown. - auto imageContainer = mVideoFrameContainer->GetImageContainer(); - if (imageContainer) { - imageContainer->DeallocateExportedImages(); - } mVideoFrameContainer->ForgetElement(); mVideoFrameContainer = nullptr; } @@ -308,7 +301,7 @@ GeckoMediaDecoderOwner::Shutdown() if (mCallback.mContext && mCallback.mFree) { (*mCallback.mFree)(mCallback.mContext); } - mCallback = {0}; + mCallback = { 0 }; } } // namespace mozilla diff --git a/gecko-media/gecko/glue/ImageContainer.cpp b/gecko-media/gecko/glue/ImageContainer.cpp index 296da88..73ef9a8 100644 --- a/gecko-media/gecko/glue/ImageContainer.cpp +++ b/gecko-media/gecko/glue/ImageContainer.cpp @@ -38,11 +38,10 @@ // #include "mozilla/layers/D3D11YCbCrImage.h" // #endif -#include "mozilla/AbstractThread.h" #include "GeckoMediaDecoderOwner.h" -#include "RustServices.h" -#include "nsClassHashtable.h" +#include "mozilla/AbstractThread.h" #include "mozilla/Assertions.h" +#include "RustServices.h" namespace mozilla { @@ -201,71 +200,159 @@ BufferRecycleBin::ClearRecycledBuffers() // } // } +class RustBufferAllocator { + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(RustBufferAllocator) +public: + explicit RustBufferAllocator(FrameAllocatorObject aAllocator) + : mAllocator(aAllocator) + { + } + size_t Allocate(GeckoPlanarYCbCrImageData* aImage) { + if (!mAllocator.mContext || !mAllocator.mAllocateFrame) { + return 0; + } + (*mAllocator.mAllocateFrame)(mAllocator.mContext, aImage); + } + void DropFrame(size_t aHandle) { + if (aHandle && mAllocator.mContext && mAllocator.mDropFrame) { + mAllocator.mDropFrame(mAllocator.mContext, aHandle); + } + } +private: + ~RustBufferAllocator() { + if (mAllocator.mContext && mAllocator.mFree) { + (*mAllocator.mFree)(mAllocator.mContext); + mAllocator = { 0 }; + } + } + FrameAllocatorObject mAllocator; +}; + +class RustPlanarYCbCrImage : public PlanarYCbCrImage { +public: + explicit RustPlanarYCbCrImage(RustBufferAllocator* aContainer) + : mAllocator(aContainer) + { + } + + ~RustPlanarYCbCrImage() override { + if (mAllocator && mHandle) { + mAllocator->DropFrame(mHandle); + } + } + + uint8_t* AllocateAndGetNewBuffer(uint32_t aSize) override { + return nullptr; + } + + size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override { + return 0; + } + + RustPlanarYCbCrImage* AsRustPlanarYCbCrImage() override { return this; } + + bool IsValid() { return mHandle != 0; } + + bool CopyData(const Data& aData) override + { + // We can only handle non-interleaved image formats. + // Fortunately, our decoders only output planar i420 + // images. + MOZ_ASSERT(aData.mYSkip == 0); + MOZ_ASSERT(aData.mCbSkip == 0); + MOZ_ASSERT(aData.mCrSkip == 0); + MOZ_ASSERT(aData.mPicSize.width > 0); + MOZ_ASSERT(aData.mPicSize.height > 0); + + GeckoPlanarYCbCrImageData img; + + img.mYPlane.mStride = aData.mYStride; + img.mYPlane.mWidth = aData.mYSize.width; + img.mYPlane.mHeight = aData.mYSize.height; + img.mYPlane.mPixelData = aData.mYChannel; + + img.mCbPlane.mStride = aData.mCbCrStride; + img.mCbPlane.mWidth = aData.mCbCrSize.width; + img.mCbPlane.mHeight = aData.mCbCrSize.height; + img.mCbPlane.mPixelData = aData.mCbChannel; + + img.mCrPlane.mStride = aData.mCbCrStride; + img.mCrPlane.mWidth = aData.mCbCrSize.width; + img.mCrPlane.mHeight = aData.mCbCrSize.height; + img.mCrPlane.mPixelData = aData.mCrChannel; + + img.mPicX = aData.mPicX; + img.mPicY = aData.mPicY; + img.mPicWidth = aData.mPicSize.width; + img.mPicHeight = aData.mPicSize.height; + + mHandle = mAllocator->Allocate(&img); + + mOrigin = gfx::IntPoint(aData.mPicX, aData.mPicY); + mSize = gfx::IntSize(aData.mPicSize.width, aData.mPicSize.height); + + return mHandle != 0; + } + size_t Handle() const { return mHandle; } +private: + size_t mHandle = 0; + RefPtr mAllocator; +}; + +class RustImageFactory : public ImageFactory +{ +protected: + friend class ImageContainer; + + explicit RustImageFactory(FrameAllocatorObject aAllocator) + : mAllocator(new RustBufferAllocator(aAllocator)) + {} + + RefPtr CreatePlanarYCbCrImage( + const gfx::IntSize& aScaleHint, + BufferRecycleBin *aRecycleBin) + { + return new RustPlanarYCbCrImage(mAllocator); + } +private: + RefPtr mAllocator; +}; + +ImageContainer::ImageContainer(GeckoMediaDecoderOwner* aOwner, + ImageFactory* aImageFactory) + : mRecursiveMutex("ImageContainer.mRecursiveMutex") + , mGenerationCounter(++sGenerationCounter) + , mPaintCount(0) + , mDroppedImageCount(0) + , mImageFactory(aImageFactory) + , mRecycleBin(new BufferRecycleBin()) + , mCurrentProducerID(-1) + , mOwner(aOwner) +{ +} + ImageContainer::ImageContainer(GeckoMediaDecoderOwner* aOwner) -: mRecursiveMutex("ImageContainer.mRecursiveMutex"), - mGenerationCounter(++sGenerationCounter), - mPaintCount(0), - mDroppedImageCount(0), - mImageFactory(new ImageFactory()), - mRecycleBin(new BufferRecycleBin()), - // mIsAsync(flag == ASYNCHRONOUS), - mCurrentProducerID(-1), - mOwner(aOwner) + : ImageContainer(aOwner, new ImageFactory()) { - // if (flag == ASYNCHRONOUS) { - // mNotifyCompositeListener = new ImageContainerListener(this); - // EnsureImageClient(); - // } } -// ImageContainer::ImageContainer(const CompositableHandle& aHandle) -// : mRecursiveMutex("ImageContainer.mRecursiveMutex"), -// mGenerationCounter(++sGenerationCounter), -// mPaintCount(0), -// mDroppedImageCount(0), -// mImageFactory(nullptr), -// mRecycleBin(nullptr), -// mIsAsync(true), -// // mAsyncContainerHandle(aHandle), -// mCurrentProducerID(-1) -// { -// // MOZ_ASSERT(mAsyncContainerHandle); -// } +ImageContainer::ImageContainer(GeckoMediaDecoderOwner* aOwner, + FrameAllocatorObject aAllocator) + : ImageContainer(aOwner, new RustImageFactory(aAllocator)) +{ +} ImageContainer::~ImageContainer() { - // if (mNotifyCompositeListener) { - // mNotifyCompositeListener->ClearImageContainer(); - // } - // if (mAsyncContainerHandle) { - // if (RefPtr imageBridge = ImageBridgeChild::GetSingleton()) { - // imageBridge->ForgetImageContainer(mAsyncContainerHandle); - // } - // } } RefPtr ImageContainer::CreatePlanarYCbCrImage() { RecursiveMutexAutoLock lock(mRecursiveMutex); - // EnsureImageClient(); - // if (mImageClient && mImageClient->AsImageClientSingle()) { - // return new SharedPlanarYCbCrImage(mImageClient); - // } return mImageFactory->CreatePlanarYCbCrImage(mScaleHint, mRecycleBin); } -// RefPtr -// ImageContainer::CreateSharedRGBImage() -// { -// RecursiveMutexAutoLock lock(mRecursiveMutex); -// EnsureImageClient(); -// if (!mImageClient || !mImageClient->AsImageClientSingle()) { -// return nullptr; -// } -// return new SharedRGBImage(mImageClient); -// } - void ImageContainer::SetCurrentImageInternal(const nsTArray& aImages) { @@ -360,121 +447,6 @@ class UpdateCurrentImagesRunnable : public Runnable RefPtr mOwner; }; -struct ExportedImage { - ExportedImage(Image* aImage, ImageContainer* aContainer) - : mImage(aImage) - , mContainer(aContainer) - { - MOZ_ASSERT(mImage); - MOZ_ASSERT(mContainer); - } - ExportedImage(const ExportedImage& aOther) - : mImage(aOther.mImage) - , mContainer(aOther.mContainer) - { - } - ExportedImage(ExportedImage&& aOther) - : mImage(Move(aOther.mImage)) - , mContainer(Move(aOther.mContainer)) - { - } - RefPtr mImage; - RefPtr mContainer; - uint32_t mRefCount = 0; -}; - -StaticMutex sImageMutex; -static nsClassHashtable sImages; - -void PlanarYCbCrImage_AddRefPixelData(uint32_t aFrameID) -{ - StaticMutexAutoLock lock(sImageMutex); - ExportedImage* img = sImages.Get(aFrameID); - if (img) { - img->mRefCount += 1; - } -} - -void PlanarYCbCrImage_FreeData(uint32_t aFrameID) { - StaticMutexAutoLock lock(sImageMutex); - ExportedImage* img = sImages.Get(aFrameID); - if (img && img->mRefCount == 0) { - img->mContainer->RecordImageDropped(aFrameID); - sImages.Remove(aFrameID); - } -} - -const uint8_t* -PlanarYCbCrImage_GetPixelData(uint32_t aFrameID, PlaneType aPlaneType) -{ - RefPtr img; - { - StaticMutexAutoLock lock(sImageMutex); - ExportedImage* i = sImages.Get(aFrameID); - if (!i) { - return nullptr; - } - img = i->mImage; - } - PlanarYCbCrImage* planarImage = img->AsPlanarYCbCrImage(); - MOZ_ASSERT(planarImage); - if (!planarImage) { - return nullptr; - } - const PlanarYCbCrData* data = planarImage->GetData(); - switch (aPlaneType) { - case PlaneType::Y: return data->mYChannel; - case PlaneType::Cb: return data->mCbChannel; - case PlaneType::Cr: return data->mCrChannel; - } - return nullptr; -} - -ByteSlice -PlanarYCbCrImage_GetSliceData(uint32_t aFrameID) -{ - RefPtr img; - { - StaticMutexAutoLock lock(sImageMutex); - ExportedImage* i = sImages.Get(aFrameID); - if (!i) { - return ByteSlice { nullptr, 0 }; - } - img = i->mImage; - } - PlanarYCbCrImage* planarImage = img->AsPlanarYCbCrImage(); - MOZ_ASSERT(planarImage); - if (!planarImage) { - return ByteSlice { nullptr, 0 }; - } - const PlanarYCbCrData* data = planarImage->GetData(); - const uint8_t* ptr = data->mYChannel; - size_t length = planarImage->GetDataSize(); - return ByteSlice { ptr, length }; -} - -void -ImageContainer::DeallocateExportedImages() -{ - StaticMutexAutoLock lock(sImageMutex); - for (size_t frameId : mExportedImages) { - sImages.Remove(frameId); - } - mExportedImages.Clear(); -} - -void -ImageContainer::RecordImageExported(size_t aFrameID) -{ - mExportedImages.AppendElement(aFrameID); -} - -void -ImageContainer::RecordImageDropped(size_t aFrameID) -{ - mExportedImages.RemoveElement(aFrameID); -} - void ImageContainer::NotifyOwnerOfNewImages() { @@ -485,29 +457,15 @@ ImageContainer::NotifyOwnerOfNewImages() nsTArray images; for (const OwningImage& owningImage : mCurrentImages) { - PlanarYCbCrImage* planarImage = owningImage.mImage->AsPlanarYCbCrImage(); + RustPlanarYCbCrImage* planarImage = owningImage.mImage->AsRustPlanarYCbCrImage(); MOZ_ASSERT(planarImage); if (!planarImage) { continue; } - const PlanarYCbCrData* data = planarImage->GetData(); GeckoPlanarYCbCrImage* img = images.AppendElement(); - - img->mYStride = data->mYStride; - img->mYWidth = data->mYSize.width; - img->mYHeight = data->mYSize.height; - img->mYSkip = data->mYSkip; - img->mCbCrStride = data->mCbCrStride; - img->mCbCrWidth = data->mCbCrSize.width; - img->mCbCrHeight = data->mCbCrSize.height; - img->mCbSkip = data->mCbSkip; - img->mCrSkip = data->mCrSkip; - img->mPicX = data->mPicX; - img->mPicY = data->mPicY; - img->mPicWidth = data->mPicSize.width; - img->mPicHeight = data->mPicSize.height; img->mFrameID = owningImage.mFrameID; + img->mImageHandle = planarImage->Handle(); // Calculate a TimeStamp in the external frame of reference. // Note the OwningImage can have a null TimeStamp if we're @@ -518,21 +476,6 @@ ImageContainer::NotifyOwnerOfNewImages() TimeStamp now = TimeStamp::Now(); TimeStamp frameTime = !owningImage.mTimeStamp.IsNull() ? owningImage.mTimeStamp : now; img->mTimeStamp = rustTime + (frameTime - now).ToMicroseconds() * 1000.0; - - { - StaticMutexAutoLock lock(sImageMutex); - if (!sImages.Contains(img->mFrameID)) { - sImages.Put(img->mFrameID, new ExportedImage(owningImage.mImage, this)); - RecordImageExported(img->mFrameID); - } else { - sImages.Get(img->mFrameID)->mRefCount += 1; - } - } - - img->mAddRefPixelData = &PlanarYCbCrImage_AddRefPixelData; - img->mFreePixelData = &PlanarYCbCrImage_FreeData; - img->mGetPixelData = &PlanarYCbCrImage_GetPixelData; - img->mGetSliceData = &PlanarYCbCrImage_GetSliceData; } RefPtr task = new UpdateCurrentImagesRunnable(mOwner, Move(images)); diff --git a/gecko-media/gecko/glue/MediaData.cpp b/gecko-media/gecko/glue/MediaData.cpp index 962d8c7..9194cec 100644 --- a/gecko-media/gecko/glue/MediaData.cpp +++ b/gecko-media/gecko/glue/MediaData.cpp @@ -252,6 +252,10 @@ ConstructPlanarYCbCrData(const VideoInfo& aInfo, data.mPicX = aPicture.x; data.mPicY = aPicture.y; data.mPicSize = aPicture.Size(); + MOZ_ASSERT(aPicture.width > 0); + MOZ_ASSERT(aPicture.height > 0); + MOZ_ASSERT(data.mPicSize.width > 0); + MOZ_ASSERT(data.mPicSize.height > 0); data.mStereoMode = aInfo.mStereoMode; data.mYUVColorSpace = aBuffer.mYUVColorSpace; // data.mBitDepth = aBuffer.mBitDepth; @@ -309,6 +313,9 @@ VideoData::CreateAndCopyData(const VideoInfo& aInfo, return nullptr; } + MOZ_ASSERT(aPicture.width > 0); + MOZ_ASSERT(aPicture.height > 0); + RefPtr v(new VideoData(aOffset, aTime, aDuration, diff --git a/gecko-media/gecko/glue/Player.cpp b/gecko-media/gecko/glue/Player.cpp index dfa3782..9b89cf0 100644 --- a/gecko-media/gecko/glue/Player.cpp +++ b/gecko-media/gecko/glue/Player.cpp @@ -10,10 +10,10 @@ #include "GeckoMediaDecoder.h" #include "GeckoMediaDecoderOwner.h" #include "GeckoMediaMacros.h" -#include "mozilla/RefPtr.h" -#include "nsTArray.h" #include "RustMediaResource.h" #include "UniquePtr.h" +#include "mozilla/RefPtr.h" +#include "nsTArray.h" using namespace mozilla; @@ -36,8 +36,10 @@ class RustVecU8BufferMediaResource : public BufferMediaResource struct Player { - Player(size_t aId, PlayerCallbackObject aCallback) - : mDecoderOwner(new GeckoMediaDecoderOwner(aCallback)) + Player(size_t aId, + PlayerCallbackObject aCallback, + FrameAllocatorObject aAllocator) + : mDecoderOwner(new GeckoMediaDecoderOwner(aCallback, aAllocator)) , mId(aId) { } @@ -59,14 +61,16 @@ static void CreatePlayer(size_t aId, MediaResource* aResource, const char* aMimeType, - PlayerCallbackObject aCallback) + PlayerCallbackObject aCallback, + FrameAllocatorObject aAllocator) { mozilla::Maybe mime = MakeMediaContainerType(aMimeType); if (!mime) { return; } - Player* reflector = sReflectors.AppendElement(Player(aId, aCallback)); + Player* reflector = + sReflectors.AppendElement(Player(aId, aCallback, aAllocator)); MOZ_ASSERT(GetReflector(aId) == reflector); MediaDecoderInit decoderInit(reflector->mDecoderOwner.get(), @@ -86,22 +90,23 @@ void GeckoMedia_Player_CreateBlobPlayer(size_t aId, RustVecU8Object aMediaData, const char* aMimeType, - PlayerCallbackObject aCallback) + PlayerCallbackObject aCallback, + FrameAllocatorObject aAllocator) { RefPtr resource = new RustVecU8BufferMediaResource(aMediaData); - CreatePlayer(aId, resource, aMimeType, aCallback); + CreatePlayer(aId, resource, aMimeType, aCallback, aAllocator); } void GeckoMedia_Player_CreateNetworkPlayer(size_t aId, NetworkResourceObject aNetworkResource, const char* aMimeType, - PlayerCallbackObject aCallback) + PlayerCallbackObject aCallback, + FrameAllocatorObject aAllocator) { - RefPtr resource = - new RustMediaResource(aNetworkResource); - CreatePlayer(aId, resource, aMimeType, aCallback); + RefPtr resource = new RustMediaResource(aNetworkResource); + CreatePlayer(aId, resource, aMimeType, aCallback, aAllocator); IMPL_GECKO_MEDIA_REFLECTOR_GET(Player) resource->SetDecoder(reflector->mDecoder); } diff --git a/gecko-media/gecko/glue/include/GeckoMediaDecoderOwner.h b/gecko-media/gecko/glue/include/GeckoMediaDecoderOwner.h index 17515ae..fd8e7bb 100644 --- a/gecko-media/gecko/glue/include/GeckoMediaDecoderOwner.h +++ b/gecko-media/gecko/glue/include/GeckoMediaDecoderOwner.h @@ -30,10 +30,10 @@ class HTMLMediaElement; class GeckoMediaDecoderOwner : public MediaDecoderOwner { public: - NS_INLINE_DECL_THREADSAFE_REFCOUNTING(GeckoMediaDecoderOwner) - GeckoMediaDecoderOwner(PlayerCallbackObject aCallback); + GeckoMediaDecoderOwner(PlayerCallbackObject aCallback, + FrameAllocatorObject aAllocator); GeckoMediaDecoderOwner(); // Called by the media decoder to indicate that the download is progressing. @@ -208,6 +208,7 @@ class GeckoMediaDecoderOwner : public MediaDecoderOwner bool mHasError = false; PlayerCallbackObject mCallback = { 0 }; + FrameAllocatorObject mAllocator = { 0 }; RefPtr mDecoder; RefPtr mVideoFrameContainer; }; diff --git a/gecko-media/gecko/glue/include/ImageContainer.h b/gecko-media/gecko/glue/include/ImageContainer.h index 862b9d1..1de589a 100644 --- a/gecko-media/gecko/glue/include/ImageContainer.h +++ b/gecko-media/gecko/glue/include/ImageContainer.h @@ -34,6 +34,7 @@ #include "nsDataHashtable.h" #include "mozilla/EnumeratedArray.h" #include "mozilla/UniquePtr.h" +#include "Player.h" #ifndef XPCOM_GLUE_AVOID_NSPR /** @@ -163,6 +164,7 @@ class PlanarYCbCrImage; class TextureClient; class KnowsCompositor; class NVImage; +class RustPlanarYCbCrImage; #ifdef XP_WIN class D3D11YCbCrRecycleAllocator; #endif @@ -243,6 +245,7 @@ class Image { // virtual MacIOSurfaceImage* AsMacIOSurfaceImage() { return nullptr; } // #endif virtual PlanarYCbCrImage* AsPlanarYCbCrImage() { return nullptr; } + virtual RustPlanarYCbCrImage* AsRustPlanarYCbCrImage() { return nullptr; } // virtual NVImage* AsNVImage() { return nullptr; } @@ -355,6 +358,7 @@ class ImageFactory // ImageContainer* mImageContainer; // }; + /** * A class that manages Images for an ImageLayer. The only reason * we need a separate class here is that ImageLayers aren't threadsafe @@ -383,6 +387,9 @@ class ImageContainer final : public SupportsWeakPtr NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ImageContainer) + ImageContainer(GeckoMediaDecoderOwner* aOwner, + ImageFactory* aImageFactory); + public: MOZ_DECLARE_WEAKREFERENCE_TYPENAME(ImageContainer) @@ -392,6 +399,9 @@ class ImageContainer final : public SupportsWeakPtr explicit ImageContainer(GeckoMediaDecoderOwner* aOwner); + ImageContainer(GeckoMediaDecoderOwner* aOwner, + FrameAllocatorObject aAllocator); + /** * Create ImageContainer just to hold another ASYNCHRONOUS ImageContainer's * async container ID. @@ -622,9 +632,6 @@ class ImageContainer final : public SupportsWeakPtr // void DropImageClient(); - void DeallocateExportedImages(); - void RecordImageDropped(size_t aFrameID); - private: typedef mozilla::RecursiveMutex RecursiveMutex; diff --git a/gecko-media/gecko/glue/include/Player.h b/gecko-media/gecko/glue/include/Player.h index 570009b..f8bf7d6 100644 --- a/gecko-media/gecko/glue/include/Player.h +++ b/gecko-media/gecko/glue/include/Player.h @@ -32,39 +32,32 @@ enum PlaneType Cr }; -struct ByteSlice { - const uint8_t* mData; - size_t mLength; +struct GeckoImagePlane +{ + int32_t mStride; + int32_t mWidth; + int32_t mHeight; + const uint8_t* mPixelData; }; -struct GeckoPlanarYCbCrImage +struct GeckoPlanarYCbCrImageData { - // Luminance buffer - int32_t mYStride; - int32_t mYWidth; - int32_t mYHeight; - int32_t mYSkip; - - // Chroma buffers - int32_t mCbCrStride; - int32_t mCbCrWidth; - int32_t mCbCrHeight; - int32_t mCbSkip; - int32_t mCrSkip; + GeckoImagePlane mYPlane; + GeckoImagePlane mCbPlane; + GeckoImagePlane mCrPlane; // Picture region uint32_t mPicX; uint32_t mPicY; int32_t mPicWidth; int32_t mPicHeight; +}; +struct GeckoPlanarYCbCrImage +{ + size_t mImageHandle; uint64_t mTimeStamp; uint32_t mFrameID; - - void (*mAddRefPixelData)(uint32_t aFrameID); - void (*mFreePixelData)(uint32_t aFrameID); - const uint8_t* (*mGetPixelData)(uint32_t aFrameID, PlaneType aPlaneType); - ByteSlice (*mGetSliceData)(uint32_t aFrameID); }; struct PlayerCallbackObject @@ -85,23 +78,41 @@ struct PlayerCallbackObject void (*mFree)(void*); }; -struct GeckoMediaByteRange { +struct FrameAllocatorObject +{ + void* mContext; + size_t (*mAllocateFrame)(void*, const GeckoPlanarYCbCrImageData* aImage); + void (*mDropFrame)(void*, size_t); + void (*mFree)(void*); +}; + +struct GeckoMediaByteRange +{ uint64_t mStart; uint64_t mEnd; }; -struct CachedRangesObserverObject { +struct CachedRangesObserverObject +{ void (*mUpdate)(uint32_t, const GeckoMediaByteRange*, size_t); uint32_t mResourceID; }; -struct NetworkResourceObject { +struct NetworkResourceObject +{ void (*mSetRangesObserver)(void*, CachedRangesObserverObject); - bool (*mReadAt)(void* aData, uint64_t aOffset, uint8_t* aBytes, uint32_t aLength, uint32_t* aOutBytesRead); + bool (*mReadAt)(void* aData, + uint64_t aOffset, + uint8_t* aBytes, + uint32_t aLength, + uint32_t* aOutBytesRead); void (*mPin)(void* aData); void (*mUnPin)(void* aData); int64_t (*mLength)(void* aData); - bool (*mReadFromCache)(void* aData, uint64_t aOffset, uint8_t* aBytes, uint32_t aLength); + bool (*mReadFromCache)(void* aData, + uint64_t aOffset, + uint8_t* aBytes, + uint32_t aLength); void (*mFree)(void*); void* mData; }; @@ -111,13 +122,15 @@ void GeckoMedia_Player_CreateBlobPlayer(size_t aId, RustVecU8Object aMediaData, const char* aMimeType, - PlayerCallbackObject aCallback); + PlayerCallbackObject aCallback, + FrameAllocatorObject aAllocator); void GeckoMedia_Player_CreateNetworkPlayer(size_t aId, NetworkResourceObject aMediaData, const char* aMimeType, - PlayerCallbackObject aCallback); + PlayerCallbackObject aCallback, + FrameAllocatorObject aAllocator); void GeckoMedia_Player_Play(size_t aId); diff --git a/gecko-media/src/player.rs b/gecko-media/src/player.rs index 58b3676..679ea04 100644 --- a/gecko-media/src/player.rs +++ b/gecko-media/src/player.rs @@ -2,13 +2,13 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. -use bindings::CachedRangesObserverObject; -use bindings::{GeckoMediaByteRange, GeckoPlanarYCbCrImage}; +use bindings::{CachedRangesObserverObject, FrameAllocatorObject, GeckoImagePlane}; +use bindings::{GeckoMediaByteRange, GeckoPlanarYCbCrImage, GeckoPlanarYCbCrImageData}; use bindings::{GeckoMediaMetadata, GeckoMediaTimeInterval}; use bindings::{GeckoMedia_Player_CreateBlobPlayer, GeckoMedia_Player_CreateNetworkPlayer}; use bindings::{GeckoMedia_Player_Pause, GeckoMedia_Player_Play}; use bindings::{GeckoMedia_Player_Seek, GeckoMedia_Player_SetVolume}; -use bindings::{NetworkResourceObject, PlaneType_Cb, PlaneType_Cr, PlaneType_Y}; +use bindings::NetworkResourceObject; use bindings::{PlayerCallbackObject, RustVecU8Object}; use std::ffi::CStr; use std::ffi::CString; @@ -16,7 +16,7 @@ use std::mem; use std::ops::Range; use std::os::raw::c_char; use std::slice; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use timestamp::TimeStamp; /// Holds useful metadata extracted from a media resource during loading. @@ -30,28 +30,35 @@ pub struct Metadata { } /// Holds pixel data and geometry of a non-interleaved plane of data. -pub struct Plane<'a> { +pub struct Plane { /// Pixel sample data. One byte per sample. Will contain stride*height byte. - pub pixels: &'a [u8], + pub pixels: Vec, /// The width of a line of pixels in bytes. - pub width: i32, + pub width: usize, /// The stride of a line of pixels in bytes. - pub stride: i32, + pub stride: usize, /// The height of the plane in lines. - pub height: i32, + pub height: usize, +} + +struct PlanarYCbCrImageData { + pub y_plane: Plane, + pub cb_plane: Plane, + pub cr_plane: Plane, + pub picture: Region, } /// A subregion of an image buffer. #[derive(Clone)] pub struct Region { - // X coordinate of theExternalImage origin of the region. - pub x: u32, - // Y coordinate of the origin of the region. - pub y: u32, - // Width of region. - pub width: i32, - // Height of the region. - pub height: i32, + /// X coordinate of the origin of the region. + pub x: usize, + /// Y coordinate of the origin of the region. + pub y: usize, + /// Width of region. + pub width: usize, + /// Height of the region. + pub height: usize, } /// Stores a planar YCbCr image. @@ -68,14 +75,9 @@ pub struct Region { /// The color format is detected based on the height/width ratios /// defined above. pub struct PlanarYCbCrImage { - /// The sub-region of the buffer which contains the image to be rendered. - pub picture: Region, - /// The time at which this image should be rendered. - pub time_stamp: TimeStamp, - /// A stream-unique identifier. - pub frame_id: u32, - gecko_image: GeckoPlanarYCbCrImage, - pixel_buffer: Arc>, + timestamp: TimeStamp, + frame_id: u32, + image_data: Arc, } // When cloning, we need to ensure we increment the reference count on @@ -83,34 +85,32 @@ pub struct PlanarYCbCrImage { // the reference count appropriately. impl Clone for PlanarYCbCrImage { fn clone(&self) -> Self { - unsafe { - (self.gecko_image.mAddRefPixelData.unwrap())(self.frame_id); - } PlanarYCbCrImage { - picture: self.picture.clone(), - time_stamp: self.time_stamp.clone(), + timestamp: self.timestamp.clone(), frame_id: self.frame_id, - gecko_image: self.gecko_image, - pixel_buffer: self.pixel_buffer.clone(), + image_data: self.image_data.clone(), } } } impl PlanarYCbCrImage { - fn pixel_slice_for_channel<'a>(&'a self, channel_index: u8) -> &'a [u8] { - let img = &self.gecko_image; - let y_plane_size = (img.mYStride * img.mYHeight) as usize; - let cbcr_plane_size = (img.mCbCrStride * img.mCbCrHeight) as usize; - let (start, size) = match channel_index { - 0 => (0, y_plane_size), - 1 => (y_plane_size, cbcr_plane_size), - 2 => (y_plane_size + cbcr_plane_size, cbcr_plane_size), - _ => panic!("Invalid channel_index"), - }; - &self.pixel_buffer[start..(start + size)] + /// Returns the time at which this image should be rendered. + pub fn timestamp(&self) -> TimeStamp { + self.timestamp.clone() } - - pub fn plane_for_channel(&self, channel_index: u8) -> Plane { + /// Returns the image identifier, which is a stream-unique identifier + /// for this frame. This can be used to distinguish images. + pub fn frame_id(&self) -> u32 { + self.frame_id + } + /// Returns the sub-region of the buffer which contains the image to + /// be rendered. + pub fn picture_region(&self) -> &Region { + &self.image_data.picture + } + /// Returns the image plane for the image channels. + /// Pass 0 for Y, 1 for Cb, 2 for Cr. Other values panic. + pub fn plane_for_channel(&self, channel_index: u8) -> &Plane { match channel_index { 0 => self.y_plane(), 1 => self.cb_plane(), @@ -118,44 +118,17 @@ impl PlanarYCbCrImage { _ => panic!("Invalid planar index"), } } - - pub fn y_plane(&self) -> Plane { - let img = &self.gecko_image; - Plane { - pixels: self.pixel_slice_for_channel(0), - width: img.mYWidth, - stride: img.mYStride, - height: img.mYHeight, - } + /// Returns the Y plane data. + pub fn y_plane(&self) -> &Plane { + &self.image_data.y_plane } - - pub fn cb_plane(&self) -> Plane { - let img = &self.gecko_image; - Plane { - pixels: self.pixel_slice_for_channel(1), - width: img.mCbCrWidth, - stride: img.mCbCrStride, - height: img.mCbCrHeight, - } + /// Returns the Cb plane data. + pub fn cb_plane(&self) -> &Plane { + &self.image_data.cb_plane } - - pub fn cr_plane(&self) -> Plane { - let img = &self.gecko_image; - Plane { - pixels: self.pixel_slice_for_channel(2), - width: img.mCbCrWidth, - stride: img.mCbCrStride, - height: img.mCbCrHeight, - } - } -} - -impl Drop for PlanarYCbCrImage { - fn drop(&mut self) { - let frame_id = self.gecko_image.mFrameID; - unsafe { - (self.gecko_image.mFreePixelData.unwrap())(frame_id); - }; + /// Returns the Cr plane data. + pub fn cr_plane(&self) -> &Plane { + &self.image_data.cr_plane } } @@ -215,6 +188,84 @@ pub trait PlayerEventSink: Send + Sync { fn seekable(&self, ranges: Vec>); } +struct SharedVideoFrame { + id: usize, + frame: Arc, +} + +// Allocates video frames in Rust code that can be referenced via ID +// in C++ code. +struct VideoFrameAllocator { + allocated_frames: Vec, + next_frame_id: usize, +} + +impl VideoFrameAllocator { + pub fn new() -> VideoFrameAllocator { + VideoFrameAllocator { + allocated_frames: vec![], + next_frame_id: 1, + } + } + + unsafe fn to_plane(plane: &GeckoImagePlane) -> Plane { + assert!((*plane).mWidth > 0); + assert!((*plane).mHeight > 0); + assert!((*plane).mStride > 0); + let size = (*plane).mStride as usize * (*plane).mHeight as usize; + let src = slice::from_raw_parts((*plane).mPixelData, size); + Plane { + pixels: src.to_vec(), + width: (*plane).mWidth as usize, + stride: (*plane).mStride as usize, + height: (*plane).mHeight as usize, + } + } + + pub fn allocate_and_copy_image(&mut self, image: *const GeckoPlanarYCbCrImageData) -> usize { + let id = self.next_frame_id; + self.next_frame_id += 1; + + let frame = unsafe { + assert!((*image).mPicWidth > 0); + assert!((*image).mPicHeight > 0); + SharedVideoFrame { + id, + frame: Arc::new(PlanarYCbCrImageData { + y_plane: Self::to_plane(&(*image).mYPlane), + cb_plane: Self::to_plane(&(*image).mCbPlane), + cr_plane: Self::to_plane(&(*image).mCrPlane), + picture: Region { + x: (*image).mPicX as usize, + y: (*image).mPicY as usize, + width: (*image).mPicWidth as usize, + height: (*image).mPicHeight as usize, + }, + }), + } + }; + self.allocated_frames.push(frame); + id + } + + fn frame_position(&self, id: usize) -> Option { + self.allocated_frames.iter().position(|p| p.id == id) + } + + pub fn drop_frame(&mut self, id: usize) { + if let Some(index) = self.frame_position(id) { + self.allocated_frames.remove(index); + } + } + + pub fn get_frame(&self, id: usize) -> Option> { + match self.frame_position(id) { + Some(index) => Some(self.allocated_frames[index].frame.clone()), + None => None, + } + } +} + /// Plays a media resource. def_gecko_media_struct!(Player); @@ -228,19 +279,28 @@ impl Player { mime_type: &str, sink: Box, ) -> Result { - let callback = to_ffi_callback(sink); + let video_frame_allocator = Arc::new(Mutex::new(VideoFrameAllocator::new())); + let callback = to_ffi_callback(sink, video_frame_allocator.clone()); let media_data = to_ffi_vec(media_data); let mime_type = match CString::new(mime_type.as_bytes()) { Ok(mime_type) => mime_type, _ => return Err(()), }; - gecko_media.queue_task(move || unsafe { - GeckoMedia_Player_CreateBlobPlayer(id, media_data, mime_type.as_ptr(), callback); - }); - Ok(Player { + let ffi_frame_allocator = to_ffi_frame_allocator(video_frame_allocator.clone()); + let player = Player { gecko_media, id, - }) + }; + player.gecko_media.queue_task(move || unsafe { + GeckoMedia_Player_CreateBlobPlayer( + id, + media_data, + mime_type.as_ptr(), + callback, + ffi_frame_allocator, + ); + }); + Ok(player) } pub fn from_network_resource( @@ -254,15 +314,24 @@ impl Player { Ok(mime_type) => mime_type, _ => return Err(()), }; - let callback = to_ffi_callback(sink); + let video_frame_allocator = Arc::new(Mutex::new(VideoFrameAllocator::new())); + let callback = to_ffi_callback(sink, video_frame_allocator.clone()); + let ffi_frame_allocator = to_ffi_frame_allocator(video_frame_allocator.clone()); let resource = to_ffi_resource(resource); - gecko_media.queue_task(move || unsafe { - GeckoMedia_Player_CreateNetworkPlayer(id, resource, mime_type.as_ptr(), callback); - }); - Ok(Player { + let player = Player { gecko_media, id, - }) + }; + player.gecko_media.queue_task(move || unsafe { + GeckoMedia_Player_CreateNetworkPlayer( + id, + resource, + mime_type.as_ptr(), + callback, + ffi_frame_allocator, + ); + }); + Ok(player) } /// Starts playback of the media resource. While playing, @@ -357,11 +426,20 @@ pub trait NetworkResource: Send + Sync { fn read_from_cache(&self, offset: u64, buffer: &mut [u8]) -> Result<(), ()>; } -fn to_ffi_callback(callbacks: Box) -> PlayerCallbackObject { +fn to_ffi_callback( + callbacks: Box, + video_frame_allocator: Arc>, +) -> PlayerCallbackObject { // Can't cast from *c_void to a Trait, so wrap in a concrete type // when we pass into C++ code. - - def_gecko_callbacks_ffi_wrapper!(Box); + use std::os::raw::c_void; + struct Wrapper { + callbacks: Box, + video_frame_allocator: Arc>, + } + unsafe extern "C" fn free(ptr: *mut c_void) { + drop(Box::from_raw(ptr as *mut Wrapper)); + } impl_simple_ffi_callback_wrapper!(decode_error, ()); impl_simple_ffi_callback_wrapper!(playback_ended, ()); @@ -395,7 +473,7 @@ fn to_ffi_callback(callbacks: Box) -> PlayerCallbackObject { } unsafe extern "C" fn update_current_images(ptr: *mut c_void, size: usize, elements: *mut GeckoPlanarYCbCrImage) { let wrapper = &*(ptr as *mut Wrapper); - let images = to_ffi_planar_ycbycr_images(size, elements); + let images = to_ffi_planar_ycbycr_images(size, elements, &wrapper.video_frame_allocator); wrapper.callbacks.update_current_images(images); } unsafe extern "C" fn notify_buffered(ptr: *mut c_void, size: usize, ranges: *mut GeckoMediaTimeInterval) { @@ -412,6 +490,7 @@ fn to_ffi_callback(callbacks: Box) -> PlayerCallbackObject { PlayerCallbackObject { mContext: Box::into_raw(Box::new(Wrapper { callbacks, + video_frame_allocator, })) as *mut c_void, mPlaybackEnded: Some(playback_ended), mDecodeError: Some(decode_error), @@ -429,6 +508,31 @@ fn to_ffi_callback(callbacks: Box) -> PlayerCallbackObject { } } +fn to_ffi_frame_allocator(video_frame_allocator: Arc>) -> FrameAllocatorObject { + def_gecko_callbacks_ffi_wrapper!(Arc>); + + unsafe extern "C" fn drop_frame(ptr: *mut c_void, id: usize) { + let wrapper = &*(ptr as *mut Wrapper); + let mut allocator = wrapper.callbacks.lock().unwrap(); + allocator.drop_frame(id) + } + + unsafe extern "C" fn allocate_and_copy_image(ptr: *mut c_void, image: *const GeckoPlanarYCbCrImageData) -> usize { + let wrapper = &*(ptr as *mut Wrapper); + let mut allocator = wrapper.callbacks.lock().unwrap(); + allocator.allocate_and_copy_image(image) + } + + FrameAllocatorObject { + mContext: Box::into_raw(Box::new(Wrapper { + callbacks: video_frame_allocator, + })) as *mut c_void, + mAllocateFrame: Some(allocate_and_copy_image), + mDropFrame: Some(drop_frame), + mFree: Some(free), + } +} + fn to_ffi_resource(callbacks: Box) -> NetworkResourceObject { // Can't cast from *c_void to a Trait, so wrap in a concrete type // when we pass into C++ code. @@ -513,30 +617,26 @@ fn to_ffi_vec(bytes: Vec) -> RustVecU8Object { } } -fn to_ffi_planar_ycbycr_images(size: usize, elements: *mut GeckoPlanarYCbCrImage) -> Vec { +fn to_ffi_planar_ycbycr_images( + size: usize, + elements: *mut GeckoPlanarYCbCrImage, + video_frame_allocator: &Arc>, +) -> Vec { let elements = unsafe { slice::from_raw_parts(elements, size) }; - elements - .iter() - .map(|&img| -> PlanarYCbCrImage { - let byte_slice = unsafe { - let f = img.mGetSliceData.unwrap(); - f(img.mFrameID) - }; - let pixels = unsafe { slice::from_raw_parts(byte_slice.mData, byte_slice.mLength) }; - PlanarYCbCrImage { - picture: Region { - x: img.mPicX, - y: img.mPicY, - width: img.mPicWidth, - height: img.mPicHeight, - }, - time_stamp: TimeStamp(img.mTimeStamp), - frame_id: img.mFrameID, - gecko_image: img, - pixel_buffer: Arc::new(pixels.to_vec()), - } - }) - .collect::>() + let mut frames = vec![]; + let video_frame_allocator = video_frame_allocator.lock().unwrap(); + for &img in elements { + let image_data = match video_frame_allocator.get_frame(img.mImageHandle) { + Some(image_data) => image_data, + None => continue, + }; + frames.push(PlanarYCbCrImage { + timestamp: TimeStamp(img.mTimeStamp), + frame_id: img.mFrameID, + image_data, + }); + } + frames } fn to_ffi_time_ranges(size: usize, elements: *mut GeckoMediaTimeInterval) -> Vec> { From 55d06362b84e968da88ece9533753e262af2300e Mon Sep 17 00:00:00 2001 From: Chris Pearce Date: Tue, 19 Dec 2017 16:18:39 +1300 Subject: [PATCH 3/4] Auto format some unrelated code. --- gecko-media/gecko/glue/GeckoMediaDecoderOwner.cpp | 8 +++++--- .../gecko/glue/include/GeckoMediaDecoderOwner.h | 5 +++-- gecko-media/src/mse/sourcebuffer.rs | 12 +++++++----- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/gecko-media/gecko/glue/GeckoMediaDecoderOwner.cpp b/gecko-media/gecko/glue/GeckoMediaDecoderOwner.cpp index 9ae8d9c..05aa9bf 100644 --- a/gecko-media/gecko/glue/GeckoMediaDecoderOwner.cpp +++ b/gecko-media/gecko/glue/GeckoMediaDecoderOwner.cpp @@ -247,10 +247,12 @@ GeckoMediaDecoderOwner::SetDecoder(GeckoMediaDecoder* aDecoder) } void -GeckoMediaDecoderOwner::UpdateCurrentImages(nsTArray aImages) +GeckoMediaDecoderOwner::UpdateCurrentImages( + nsTArray aImages) { if (mCallback.mContext && mCallback.mUpdateCurrentImages) { - (*mCallback.mUpdateCurrentImages)(mCallback.mContext, aImages.Length(), aImages.Elements()); + (*mCallback.mUpdateCurrentImages)( + mCallback.mContext, aImages.Length(), aImages.Elements()); } } @@ -268,7 +270,7 @@ GeckoMediaDecoderOwner::NotifyBuffered() const i++; } (*mCallback.mNotifyBuffered)(mCallback.mContext, size, ranges); - delete [] ranges; + delete[] ranges; } } diff --git a/gecko-media/gecko/glue/include/GeckoMediaDecoderOwner.h b/gecko-media/gecko/glue/include/GeckoMediaDecoderOwner.h index fd8e7bb..ff23872 100644 --- a/gecko-media/gecko/glue/include/GeckoMediaDecoderOwner.h +++ b/gecko-media/gecko/glue/include/GeckoMediaDecoderOwner.h @@ -191,8 +191,9 @@ class GeckoMediaDecoderOwner : public MediaDecoderOwner // Called after the MediaStream we're playing rendered a frame to aContainer // with a different principalHandle than the previous frame. - void PrincipalHandleChangedForVideoFrameContainer(VideoFrameContainer* aContainer, - const PrincipalHandle& aNewPrincipalHandle) override; + void PrincipalHandleChangedForVideoFrameContainer( + VideoFrameContainer* aContainer, + const PrincipalHandle& aNewPrincipalHandle) override; void SetDecoder(GeckoMediaDecoder* aDecoder); diff --git a/gecko-media/src/mse/sourcebuffer.rs b/gecko-media/src/mse/sourcebuffer.rs index dd8dada..0df698c 100644 --- a/gecko-media/src/mse/sourcebuffer.rs +++ b/gecko-media/src/mse/sourcebuffer.rs @@ -9,11 +9,13 @@ use std::rc::Rc; def_gecko_media_struct!(SourceBuffer); impl SourceBuffer { - pub fn new(gecko_media: GeckoMedia, - id: usize, - callbacks: Rc, - parent_id: usize, - mime: &str) -> Result { + pub fn new( + gecko_media: GeckoMedia, + id: usize, + callbacks: Rc, + parent_id: usize, + mime: &str, + ) -> Result { let callbacks = to_ffi_callbacks(callbacks); let mime = match CString::new(mime.as_bytes()) { Ok(mime) => mime, From 5ee2696ec1f36d94e6d778ef9f6a221e5575aaf2 Mon Sep 17 00:00:00 2001 From: Chris Pearce Date: Tue, 19 Dec 2017 20:22:00 +1300 Subject: [PATCH 4/4] Silence servo-tidy --- gecko-media/src/player.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gecko-media/src/player.rs b/gecko-media/src/player.rs index 679ea04..559ad1f 100644 --- a/gecko-media/src/player.rs +++ b/gecko-media/src/player.rs @@ -2,13 +2,13 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. +use bindings::NetworkResourceObject; use bindings::{CachedRangesObserverObject, FrameAllocatorObject, GeckoImagePlane}; use bindings::{GeckoMediaByteRange, GeckoPlanarYCbCrImage, GeckoPlanarYCbCrImageData}; use bindings::{GeckoMediaMetadata, GeckoMediaTimeInterval}; use bindings::{GeckoMedia_Player_CreateBlobPlayer, GeckoMedia_Player_CreateNetworkPlayer}; use bindings::{GeckoMedia_Player_Pause, GeckoMedia_Player_Play}; use bindings::{GeckoMedia_Player_Seek, GeckoMedia_Player_SetVolume}; -use bindings::NetworkResourceObject; use bindings::{PlayerCallbackObject, RustVecU8Object}; use std::ffi::CStr; use std::ffi::CString;