diff --git a/dom/media/platforms/apple/AppleDecoderModule.cpp b/dom/media/platforms/apple/AppleDecoderModule.cpp index 39a23e8a4e90..dd2cf9a28775 100644 --- a/dom/media/platforms/apple/AppleDecoderModule.cpp +++ b/dom/media/platforms/apple/AppleDecoderModule.cpp @@ -10,6 +10,8 @@ #include "AppleATDecoder.h" #include "AppleVTDecoder.h" +#include "AppleVDADecoder.h" + #include "MP4Decoder.h" #include "VideoUtils.h" #include "VPXDecoder.h" @@ -42,7 +44,6 @@ void AppleDecoderModule::Init() { if (sInitialized) { return; } - sInitialized = true; if (RegisterSupplementalVP9Decoder()) { sCanUseVP9Decoder = CanCreateHWDecoder(MediaCodec::VP9); @@ -63,11 +64,19 @@ already_AddRefed AppleDecoderModule::CreateVideoDecoder( .isEmpty()) { return nullptr; } + RefPtr decoder; - if (IsVideoSupported(aParams.VideoConfig(), aParams.mOptions)) { - decoder = new AppleVTDecoder(aParams.VideoConfig(), aParams.mImageContainer, - aParams.mOptions, aParams.mKnowsCompositor, - aParams.mTrackingId); + + if(__builtin_available(macOS 10.8, *)) { + if (IsVideoSupported(aParams.VideoConfig(), aParams.mOptions)) { + decoder = new AppleVTDecoder(aParams.VideoConfig(), aParams.mImageContainer, + aParams.mOptions, aParams.mKnowsCompositor, + aParams.mTrackingId); + } + } else { + decoder = new AppleVDADecoder(aParams.VideoConfig(), aParams.mImageContainer, + aParams.mOptions, aParams.mKnowsCompositor, + aParams.mTrackingId); } return decoder.forget(); } @@ -250,10 +259,10 @@ bool AppleDecoderModule::CanCreateHWDecoder(MediaCodec aCodec) { vtReportsSupport = false; break; } - } else if (__builtin_available(macOS 10.7, *)) { //VTDecoder is only 10.7 and up. + } else if (__builtin_available(macOS 10.6.3, *)) { //HW decoder is 10.6.3 and up if (aCodec == media::MediaCodec::H264) { // VTIsHardwareDecodeSupported function is only available on 10.13+. - // For earlier versions, we must check H264 support by always + // For earlier versions (10.6.3+), we must check H264 support by always // attempting to create a decoder. info.mMimeType = "video/avc"; vtReportsSupport = true; @@ -261,13 +270,25 @@ bool AppleDecoderModule::CanCreateHWDecoder(MediaCodec aCodec) { } // VT reports HW decode is supported -- verify by creating an actual decoder if (vtReportsSupport) { - RefPtr decoder = - new AppleVTDecoder(info, nullptr, {}, nullptr, Nothing()); - MediaResult rv = decoder->InitializeSession(); + RefPtr decoder; + MediaResult rv; + char *type; + if(__builtin_available(macOS 10.8, *)) { + RefPtr decoder = + new AppleVTDecoder(info, nullptr, {}, nullptr, Nothing()); + rv = decoder->InitializeSession(); + type = strdup("VT"); + } else { + RefPtr decoder = + new AppleVDADecoder(info, nullptr, {}, nullptr, Nothing()); + rv = decoder->InitializeSession(); + type = strdup("VDA"); + } if (!NS_SUCCEEDED(rv)) { MOZ_LOG( sPDMLog, LogLevel::Debug, - ("Apple HW decode failure while initializing VT decoder session")); + ("Apple HW decode failure while initializing %s decoder session", type)); + free(type); return false; } nsAutoCString failureReason; @@ -278,8 +299,9 @@ bool AppleDecoderModule::CanCreateHWDecoder(MediaCodec aCodec) { aCodec == MediaCodec::H264; if (!hwSupport) { MOZ_LOG(sPDMLog, LogLevel::Debug, - ("Apple HW decode failure: '%s'", failureReason.BeginReading())); + ("Apple %s HW decode failure: '%s'", type, failureReason.BeginReading())); } + free(type); decoder->Shutdown(); return hwSupport; } diff --git a/dom/media/platforms/apple/AppleDecoderModule.h b/dom/media/platforms/apple/AppleDecoderModule.h index 46b0223d7532..a4f8c7101007 100644 --- a/dom/media/platforms/apple/AppleDecoderModule.h +++ b/dom/media/platforms/apple/AppleDecoderModule.h @@ -43,7 +43,6 @@ class AppleDecoderModule : public PlatformDecoderModule { static constexpr int kCMVideoCodecType_H264{'avc1'}; static constexpr int kCMVideoCodecType_VP9{'vp09'}; - private: AppleDecoderModule() = default; virtual ~AppleDecoderModule() = default; diff --git a/dom/media/platforms/apple/AppleVDADecoder.cpp b/dom/media/platforms/apple/AppleVDADecoder.cpp new file mode 100644 index 000000000000..3f0583e32882 --- /dev/null +++ b/dom/media/platforms/apple/AppleVDADecoder.cpp @@ -0,0 +1,656 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "AppleVDADecoder.h" + +#include "AOMDecoder.h" +#include "AppleDecoderModule.h" +#include "AppleUtils.h" +#include "CallbackThreadRegistry.h" + +#include "MediaInfo.h" +#include "MP4Decoder.h" +#include "MediaData.h" +#include "mozilla/ArrayUtils.h" +#include "mozilla/SyncRunnable.h" +#include "nsThreadUtils.h" +#include "mozilla/Logging.h" +#include "VideoUtils.h" +#include "gfxMacUtils.h" +#include +#include "gfxPlatform.h" + +#ifndef MOZ_WIDGET_UIKIT +#include "MacIOSurfaceImage.h" +#endif + +#define LOG(...) MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, (__VA_ARGS__)) +#define LOGEX(_this, ...) \ + DDMOZ_LOGEX(_this, sPDMLog, mozilla::LogLevel::Debug, __VA_ARGS__) + +//#define LOG_MEDIA_SHA1 + +namespace mozilla { + +AppleVDADecoder::AppleVDADecoder(const VideoInfo& aConfig, + layers::ImageContainer* aImageContainer, + const CreateDecoderParams::OptionSet& aOptions, + layers::KnowsCompositor* aKnowsCompositor, + Maybe aTrackingId) + : mExtraData(aConfig.mExtraData), + mPictureWidth(aConfig.mImage.width), + mPictureHeight(aConfig.mImage.height), + mDisplayWidth(aConfig.mDisplay.width), + mDisplayHeight(aConfig.mDisplay.height), + mColorSpace(aConfig.mColorSpace + ? *aConfig.mColorSpace + : DefaultColorSpace({mPictureWidth, mPictureHeight})), + mColorPrimaries(aConfig.mColorPrimaries ? *aConfig.mColorPrimaries + : gfx::ColorSpace2::BT709), + mTransferFunction(aConfig.mTransferFunction + ? *aConfig.mTransferFunction + : gfx::TransferFunction::BT709), + mColorRange(aConfig.mColorRange), + mColorDepth(aConfig.mColorDepth), + mStreamType(MP4Decoder::IsH264(aConfig.mMimeType) ? StreamType::H264 + : StreamType::Unknown), + mTaskQueue(TaskQueue::Create( + GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER), + "AppleVDADecoder")), + mDecoder(nullptr), + mMaxRefFrames( + mStreamType != StreamType::H264 || + aOptions.contains(CreateDecoderParams::Option::LowLatency) + ? 0 + : H264::ComputeMaxRefFrames(aConfig.mExtraData)), + mImageContainer(aImageContainer), + mKnowsCompositor(aKnowsCompositor) +#ifdef MOZ_WIDGET_UIKIT + , + mUseSoftwareImages(true) +#else + , + mUseSoftwareImages(aKnowsCompositor && + aKnowsCompositor->GetWebRenderCompositorType() == + layers::WebRenderCompositor::SOFTWARE) +#endif + , + mTrackingId(aTrackingId), + mIsFlushing(false), + mCallbackThreadId(), + mMonitor("AppleVDADecoder"), + mPromise(&mMonitor), // To ensure our PromiseHolder is only ever accessed + // with the monitor held. + mFormat(nullptr), + mSession(nullptr), + mIsHardwareAccelerated(false) { + MOZ_COUNT_CTOR(AppleVDADecoder); + MOZ_ASSERT(mStreamType != StreamType::Unknown); + // TODO: Verify aConfig.mime_type. + LOG("Creating AppleVDADecoder for %dx%d %s video", mDisplayWidth, + mDisplayHeight, EnumValueToString(mStreamType)); +} + +AppleVDADecoder::~AppleVDADecoder() +{ + MOZ_COUNT_DTOR(AppleVDADecoder); +} + +RefPtr +AppleVDADecoder::Init() +{ + LOG("tryna call initsession\n"); + MediaResult rv = InitializeSession(); + + if (NS_SUCCEEDED(rv)) { + return InitPromise::CreateAndResolve(TrackType::kVideoTrack, __func__); + } + return InitPromise::CreateAndReject(rv, __func__); +} + +RefPtr AppleVDADecoder::Flush() { + mIsFlushing = true; + return InvokeAsync(mTaskQueue, this, __func__, &AppleVDADecoder::ProcessFlush); +} + +RefPtr AppleVDADecoder::Drain() { + return InvokeAsync(mTaskQueue, this, __func__, &AppleVDADecoder::ProcessDrain); +} + +RefPtr +AppleVDADecoder::Shutdown() +{ + RefPtr self = this; + return InvokeAsync(mTaskQueue, __func__, [self]() { + self->ProcessShutdown(); + return self->mTaskQueue->BeginShutdown(); + }); +} + +void +AppleVDADecoder::ProcessShutdown() +{ + if (mDecoder) { + LOG("%s: cleaning up decoder %p", __func__, mDecoder); + VDADecoderDestroy(mDecoder); + mDecoder = nullptr; + } +} + +RefPtr AppleVDADecoder::ProcessFlush() { + AssertOnTaskQueue(); + OSStatus rv = VDADecoderFlush(mDecoder, 0 /*dont emit*/); + if (rv != noErr) { + LOG("AppleVDADecoder::Flush failed waiting for platform decoder " + "with error:%d.", rv); + } + MonitorAutoLock mon(mMonitor); + mPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); + while (!mReorderQueue.IsEmpty()) { + mReorderQueue.Pop(); + } + mIsFlushing = false; + return FlushPromise::CreateAndResolve(true, __func__); +} + +RefPtr AppleVDADecoder::ProcessDrain() { + AssertOnTaskQueue(); + MonitorAutoLock mon(mMonitor); + OSStatus rv = VDADecoderFlush(mDecoder, kVDADecoderFlush_EmitFrames); + if (rv != noErr) { + LOG("AppleVDADecoder::Drain failed waiting for platform decoder " + "with error:%d.", rv); + } + + DecodedData samples; + while (!mReorderQueue.IsEmpty()) { + samples.AppendElement(mReorderQueue.Pop()); + } + return DecodePromise::CreateAndResolve(std::move(samples), __func__); +} + +void AppleVDADecoder::SetSeekThreshold(const media::TimeUnit& aTime) { + if (aTime.IsValid()) { + mSeekTargetThreshold = Some(aTime); + } else { + mSeekTargetThreshold.reset(); + } +} + +// +// Implementation details. +// + +// Callback passed to the VideoToolbox decoder for returning data. +// This needs to be static because the API takes a C-style pair of +// function and userdata pointers. This validates parameters and +// forwards the decoded image back to an object method. +static void +PlatformCallback(void* decompressionOutputRefCon, + CFDictionaryRef frameInfo, + OSStatus status, + VDADecodeInfoFlags infoFlags, + CVImageBufferRef image) +{ + LOG("AppleVDADecoder[%s] status %d flags %d retainCount %ld", + __func__, status, infoFlags, CFGetRetainCount(frameInfo)); + + // Validate our arguments. + // According to Apple's TN2267 + // The output callback is still called for all flushed frames, + // but no image buffers will be returned. + // FIXME: Distinguish between errors and empty flushed frames. + if (status != noErr || !image) { + NS_WARNING("AppleVDADecoder decoder returned no data"); + image = nullptr; + } else if (infoFlags & kVDADecodeInfo_FrameDropped) { + NS_WARNING(" ...frame dropped..."); + image = nullptr; + } else { + MOZ_ASSERT(image || CFGetTypeID(image) == CVPixelBufferGetTypeID(), + "AppleVDADecoder returned an unexpected image type"); + } + + AppleVDADecoder* decoder = + static_cast(decompressionOutputRefCon); + + AutoCFRelease ptsref = + (CFNumberRef)CFDictionaryGetValue(frameInfo, CFSTR("FRAME_PTS")); + AutoCFRelease dtsref = + (CFNumberRef)CFDictionaryGetValue(frameInfo, CFSTR("FRAME_DTS")); + AutoCFRelease durref = + (CFNumberRef)CFDictionaryGetValue(frameInfo, CFSTR("FRAME_DURATION")); + AutoCFRelease boref = + (CFNumberRef)CFDictionaryGetValue(frameInfo, CFSTR("FRAME_OFFSET")); + AutoCFRelease kfref = + (CFNumberRef)CFDictionaryGetValue(frameInfo, CFSTR("FRAME_KEYFRAME")); + + int64_t dts; + int64_t pts; + int64_t duration; + int64_t byte_offset; + char is_sync_point; + + CFNumberGetValue(ptsref, kCFNumberSInt64Type, &pts); + CFNumberGetValue(dtsref, kCFNumberSInt64Type, &dts); + CFNumberGetValue(durref, kCFNumberSInt64Type, &duration); + CFNumberGetValue(boref, kCFNumberSInt64Type, &byte_offset); + CFNumberGetValue(kfref, kCFNumberSInt8Type, &is_sync_point); + + AppleVDADecoder::AppleFrameRef frameRef( + media::TimeUnit::FromMicroseconds(dts), + media::TimeUnit::FromMicroseconds(pts), + media::TimeUnit::FromMicroseconds(duration), + byte_offset, + is_sync_point == 1); + + decoder->OutputFrame(image, frameRef); +} + +void AppleVDADecoder::MaybeResolveBufferedFrames() { + mMonitor.AssertCurrentThreadOwns(); + + if (mPromise.IsEmpty()) { + return; + } + + DecodedData results; + while (mReorderQueue.Length() > mMaxRefFrames) { + results.AppendElement(mReorderQueue.Pop()); + } + mPromise.Resolve(std::move(results), __func__); +} + +void AppleVDADecoder::MaybeRegisterCallbackThread() { + ProfilerThreadId id = profiler_current_thread_id(); + if (MOZ_LIKELY(id == mCallbackThreadId)) { + return; + } + mCallbackThreadId = id; + CallbackThreadRegistry::Get()->Register(mCallbackThreadId, + "AppleVDADecoderCallback"); +} + +nsCString AppleVDADecoder::GetCodecName() const { + return nsCString(EnumValueToString(mStreamType)); +} + +// Copy and return a decoded frame. +void +AppleVDADecoder::OutputFrame(CVPixelBufferRef aImage, + AppleVDADecoder::AppleFrameRef aFrameRef) +{ + MaybeRegisterCallbackThread(); + + if (mIsFlushing) { + // We are in the process of flushing or shutting down; ignore frame. + return; + } + + LOG("mp4 output frame %lld dts %lld pts %lld duration %lld us%s", + aFrameRef.byte_offset, aFrameRef.decode_timestamp.ToMicroseconds(), + aFrameRef.composition_timestamp.ToMicroseconds(), + aFrameRef.duration.ToMicroseconds(), + aFrameRef.is_sync_point ? " keyframe" : ""); + + if (!aImage) { + // Image was dropped by decoder or none return yet. + // We need more input to continue. + MonitorAutoLock mon(mMonitor); + MaybeResolveBufferedFrames(); + return; + } + + bool useNullSample = false; + if (mSeekTargetThreshold.isSome()) { + if ((aFrameRef.composition_timestamp + aFrameRef.duration) < + mSeekTargetThreshold.ref()) { + useNullSample = true; + } else { + mSeekTargetThreshold.reset(); + } + } + + // Where our resulting image will end up. + RefPtr data; + // Bounds. + VideoInfo info; + info.mDisplay = gfx::IntSize(mDisplayWidth, mDisplayHeight); + + if (useNullSample) { + data = new NullData(aFrameRef.byte_offset, aFrameRef.composition_timestamp, + aFrameRef.duration); + } else if (mUseSoftwareImages) { + size_t width = CVPixelBufferGetWidth(aImage); + size_t height = CVPixelBufferGetHeight(aImage); + DebugOnly planes = CVPixelBufferGetPlaneCount(aImage); + MOZ_ASSERT(planes == 2, "Likely not NV12 format and it must be."); + + VideoData::YCbCrBuffer buffer; + + // Lock the returned image data. + CVReturn rv = + CVPixelBufferLockBaseAddress(aImage, kCVPixelBufferLock_ReadOnly); + if (rv != kCVReturnSuccess) { + NS_ERROR("error locking pixel data"); + MonitorAutoLock mon(mMonitor); + mPromise.Reject( + MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, + RESULT_DETAIL("CVPixelBufferLockBaseAddress:%x", rv)), + __func__); + return; + } + // Y plane. + buffer.mPlanes[0].mData = + static_cast(CVPixelBufferGetBaseAddressOfPlane(aImage, 0)); + buffer.mPlanes[0].mStride = CVPixelBufferGetBytesPerRowOfPlane(aImage, 0); + buffer.mPlanes[0].mWidth = width; + buffer.mPlanes[0].mHeight = height; + buffer.mPlanes[0].mSkip = 0; + // Cb plane. + buffer.mPlanes[1].mData = + static_cast(CVPixelBufferGetBaseAddressOfPlane(aImage, 1)); + buffer.mPlanes[1].mStride = CVPixelBufferGetBytesPerRowOfPlane(aImage, 1); + buffer.mPlanes[1].mWidth = (width+1) / 2; + buffer.mPlanes[1].mHeight = (height+1) / 2; + buffer.mPlanes[1].mSkip = 1; + // Cr plane. + buffer.mPlanes[2].mData = + static_cast(CVPixelBufferGetBaseAddressOfPlane(aImage, 1)); + buffer.mPlanes[2].mStride = CVPixelBufferGetBytesPerRowOfPlane(aImage, 1); + buffer.mPlanes[2].mWidth = (width+1) / 2; + buffer.mPlanes[2].mHeight = (height+1) / 2; + buffer.mPlanes[2].mSkip = 1; + + gfx::IntRect visible = gfx::IntRect(0, 0, mPictureWidth, mPictureHeight); + + // Copy the image data into our own format. + Result, MediaResult> result = + VideoData::CreateAndCopyData( + info, mImageContainer, aFrameRef.byte_offset, + aFrameRef.composition_timestamp, aFrameRef.duration, buffer, + aFrameRef.is_sync_point, aFrameRef.decode_timestamp, visible, + mKnowsCompositor); + // Unlock the returned image data. + CVPixelBufferUnlockBaseAddress(aImage, kCVPixelBufferLock_ReadOnly); + + } else { +#ifndef MOZ_WIDGET_UIKIT + CFTypeRefPtr surface = + CFTypeRefPtr::WrapUnderGetRule( + CVPixelBufferGetIOSurface(aImage)); + MOZ_ASSERT(surface, "Decoder didn't return an IOSurface backed buffer"); + + RefPtr macSurface = new MacIOSurface(std::move(surface)); + macSurface->SetYUVColorSpace(mColorSpace); + macSurface->mColorPrimaries = mColorPrimaries; + + RefPtr image = new layers::MacIOSurfaceImage(macSurface); + + data = VideoData::CreateFromImage( + info.mDisplay, aFrameRef.byte_offset, aFrameRef.composition_timestamp, + aFrameRef.duration, image.forget(), aFrameRef.is_sync_point, + aFrameRef.decode_timestamp); + +#else + MOZ_ASSERT_UNREACHABLE("No MacIOSurface on iOS"); +#endif + } + + if (!data) { + NS_ERROR("Couldn't create VideoData for frame"); + MonitorAutoLock mon(mMonitor); + mPromise.Reject(MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__), __func__); + return; + } + + // Frames come out in DTS order but we need to output them + // in composition order. + MonitorAutoLock mon(mMonitor); + mReorderQueue.Push(std::move(data)); + MaybeResolveBufferedFrames(); + + LOG("%llu decoded frames queued", + static_cast(mReorderQueue.Length())); + +} + + +void AppleVDADecoder::OnDecodeError(OSStatus aError) { + MonitorAutoLock mon(mMonitor); + mPromise.RejectIfExists( + MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, + RESULT_DETAIL("OnDecodeError:%x", aError)), + __func__); +} + +RefPtr AppleVDADecoder::Decode( + MediaRawData* aSample) { + LOG("mp4 input sample %p pts %lld duration %lld us%s %zu bytes", aSample, + aSample->mTime.ToMicroseconds(), aSample->mDuration.ToMicroseconds(), + aSample->mKeyframe ? " keyframe" : "", aSample->Size()); + + RefPtr self = this; + RefPtr sample = aSample; + return InvokeAsync(mTaskQueue, __func__, [self, this, sample] { + RefPtr p; + { + MonitorAutoLock mon(mMonitor); + p = mPromise.Ensure(__func__); + } + ProcessDecode(sample); + return p; + }); +} + +void +AppleVDADecoder::ProcessDecode(MediaRawData* aSample) +{ + + AutoCFRelease block = + CFDataCreate(kCFAllocatorDefault, aSample->Data(), aSample->Size()); + if (!block) { + NS_ERROR("Couldn't create CFData"); + return; + } + + AutoCFRelease pts = + CFNumberCreate(kCFAllocatorDefault, + kCFNumberSInt64Type, + &aSample->mTime); + AutoCFRelease dts = + CFNumberCreate(kCFAllocatorDefault, + kCFNumberSInt64Type, + &aSample->mTimecode); + AutoCFRelease duration = + CFNumberCreate(kCFAllocatorDefault, + kCFNumberSInt64Type, + &aSample->mDuration); + AutoCFRelease byte_offset = + CFNumberCreate(kCFAllocatorDefault, + kCFNumberSInt64Type, + &aSample->mOffset); + char keyframe = aSample->mKeyframe ? 1 : 0; + AutoCFRelease cfkeyframe = + CFNumberCreate(kCFAllocatorDefault, + kCFNumberSInt8Type, + &keyframe); + + const void* keys[] = { CFSTR("FRAME_PTS"), + CFSTR("FRAME_DTS"), + CFSTR("FRAME_DURATION"), + CFSTR("FRAME_OFFSET"), + CFSTR("FRAME_KEYFRAME") }; + const void* values[] = { pts, + dts, + duration, + byte_offset, + cfkeyframe }; + static_assert(std::size(keys) == std::size(values), + "Non matching keys/values array size"); + + AutoCFRelease frameInfo = + CFDictionaryCreate(kCFAllocatorDefault, + keys, + values, + std::size(keys), + &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks); + + OSStatus rv = VDADecoderDecode(mDecoder, + 0, + block, + frameInfo); + + if (rv != noErr) { + NS_WARNING("AppleVDADecoder: Couldn't pass frame to decoder"); + return; + } + + return; +} + +MediaResult +AppleVDADecoder::InitializeSession() +{ + OSStatus rv; + + AutoCFRelease decoderConfig = + CreateDecoderSpecification(); + + AutoCFRelease outputConfiguration = + CreateOutputConfiguration(); + + rv = + VDADecoderCreate(decoderConfig, + outputConfiguration, + (VDADecoderOutputCallback*)PlatformCallback, + this, + &mDecoder); + + rv == 0 ? mIsHardwareAccelerated = 1 : mIsHardwareAccelerated = 0; //kVDADecoderNoErr = 0 + if (rv != noErr) { + LOG("AppleVDADecoder: Couldn't create hardware VDA decoder, error %d", rv); + return NS_ERROR_FAILURE; + } + + LOG("AppleVDADecoder: %s hardware accelerated decoding", + mIsHardwareAccelerated ? "using" : "not using"); + + return NS_OK; +} + +CFDictionaryRef +AppleVDADecoder::CreateDecoderSpecification() +{ + + OSType format = 'avc1'; + AutoCFRelease avc_width = + CFNumberCreate(kCFAllocatorDefault, + kCFNumberSInt32Type, + &mPictureWidth); + AutoCFRelease avc_height = + CFNumberCreate(kCFAllocatorDefault, + kCFNumberSInt32Type, + &mPictureHeight); + AutoCFRelease avc_format = + CFNumberCreate(kCFAllocatorDefault, + kCFNumberSInt32Type, + &format); + + AutoCFRelease avc_data = + CFDataCreate(kCFAllocatorDefault, + mExtraData->Elements(), + AssertedCast(mExtraData->Length())); + + const void* decoderKeys[] = { kVDADecoderConfiguration_Width, + kVDADecoderConfiguration_Height, + kVDADecoderConfiguration_SourceFormat, + kVDADecoderConfiguration_avcCData }; + const void* decoderValue[] = { avc_width, + avc_height, + avc_format, + avc_data }; + static_assert(std::size(decoderKeys) == std::size(decoderValue), + "Non matching keys/values array size"); + + return CFDictionaryCreate(kCFAllocatorDefault, + decoderKeys, + decoderValue, + std::size(decoderKeys), + &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks); +} + +CFDictionaryRef +AppleVDADecoder::CreateOutputConfiguration() +{ + if (mUseSoftwareImages) { + // Output format type: + SInt32 PixelFormatTypeValue = + kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange; + AutoCFRelease PixelFormatTypeNumber = + CFNumberCreate(kCFAllocatorDefault, + kCFNumberSInt32Type, + &PixelFormatTypeValue); + const void* outputKeys[] = { kCVPixelBufferPixelFormatTypeKey }; + const void* outputValues[] = { PixelFormatTypeNumber }; + static_assert(std::size(outputKeys) == std::size(outputValues), + "Non matching keys/values array size"); + + return CFDictionaryCreate(kCFAllocatorDefault, + outputKeys, + outputValues, + std::size(outputKeys), + &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks); + } + +#ifndef MOZ_WIDGET_UIKIT + // Output format type: + OSType PixelFormatTypeValue = kCVPixelFormatType_422YpCbCr8; + AutoCFRelease PixelFormatTypeNumber = + CFNumberCreate(kCFAllocatorDefault, + kCFNumberSInt32Type, + &PixelFormatTypeValue); + // Construct IOSurface Properties + const void* IOSurfaceKeys[] = {kIOSurfaceIsGlobal}; + const void* IOSurfaceValues[] = {kCFBooleanTrue}; + + static_assert(std::size(IOSurfaceKeys) == std::size(IOSurfaceValues), + "Non matching keys/values array size"); + + // Contruct output configuration. + AutoCFRelease IOSurfaceProperties = + CFDictionaryCreate(kCFAllocatorDefault, + IOSurfaceKeys, + IOSurfaceValues, + std::size(IOSurfaceKeys), + &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks); + + const void* outputKeys[] = { kCVPixelBufferIOSurfacePropertiesKey, + kCVPixelBufferPixelFormatTypeKey, + kCVPixelBufferOpenGLCompatibilityKey }; + const void* outputValues[] = { IOSurfaceProperties, + PixelFormatTypeNumber, + kCFBooleanTrue }; + static_assert(std::size(outputKeys) == std::size(outputValues), + "Non matching keys/values array size"); + + return CFDictionaryCreate(kCFAllocatorDefault, + outputKeys, + outputValues, + std::size(outputKeys), + &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks); +#else + MOZ_ASSERT_UNREACHABLE("No MacIOSurface on iOS"); +#endif +} +} // namespace mozilla +#undef LOG +#undef LOGEX diff --git a/dom/media/platforms/apple/AppleVDADecoder.h b/dom/media/platforms/apple/AppleVDADecoder.h new file mode 100644 index 000000000000..0bb0f359f2c0 --- /dev/null +++ b/dom/media/platforms/apple/AppleVDADecoder.h @@ -0,0 +1,180 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_AppleVDADecoder_h +#define mozilla_AppleVDADecoder_h + +#include // For CFDictionaryRef +#include // For CMVideoFormatDescriptionRef +#include + +#include "AppleDecoderModule.h" +#include "AppleVTDecoder.h" +#include "PerformanceRecorder.h" +#include "PlatformDecoderModule.h" +#include "ReorderQueue.h" +#include "TimeUnits.h" +#include "mozilla/Atomics.h" +#include "mozilla/DefineEnum.h" +#include "mozilla/ProfilerUtils.h" +#include "mozilla/gfx/Types.h" + + +namespace mozilla { + +DDLoggedTypeDeclNameAndBase(AppleVDADecoder, MediaDataDecoder); + +typedef uint32_t VDADecodeFrameFlags; +typedef uint32_t VDADecodeInfoFlags; +enum { + kVDADecodeInfo_Asynchronous = 1UL << 0, + kVDADecodeInfo_FrameDropped = 1UL << 1 +}; + +enum { + kVDADecoderFlush_EmitFrames = 1 << 0 +}; + +class AppleVDADecoder final : public MediaDataDecoder, + public DecoderDoctorLifeLogger{ +public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(AppleVDADecoder, final); + AppleVDADecoder(const VideoInfo& aConfig, + layers::ImageContainer* aImageContainer, + const CreateDecoderParams::OptionSet& aOptions, + layers::KnowsCompositor* aKnowsCompositor, + Maybe aTrackingId); + + class AppleFrameRef { + public: + media::TimeUnit decode_timestamp; + media::TimeUnit composition_timestamp; + media::TimeUnit duration; + int64_t byte_offset; + bool is_sync_point; + + + explicit AppleFrameRef(const MediaRawData& aSample) + : decode_timestamp(aSample.mTimecode), + composition_timestamp(aSample.mTime), + duration(aSample.mDuration), + byte_offset(aSample.mOffset), + is_sync_point(aSample.mKeyframe) {} + + AppleFrameRef(const media::TimeUnit& aDts, + const media::TimeUnit& aPts, + const media::TimeUnit& aDuration, + int64_t aByte_offset, + bool aIs_sync_point) + : decode_timestamp(aDts) + , composition_timestamp(aPts) + , duration(aDuration) + , byte_offset(aByte_offset) + , is_sync_point(aIs_sync_point) + { + } + }; + + // Access from the taskqueue and the decoder's thread. + // OutputFrame is thread-safe. + void OutputFrame(CVPixelBufferRef aImage, + AppleFrameRef aFrameRef); + void OnDecodeError(OSStatus aError); + + RefPtr Init() override; + RefPtr Decode(MediaRawData* aSample) override; + RefPtr Drain() override; + RefPtr Flush() override; + RefPtr Shutdown() override; + bool IsHardwareAccelerated(nsACString& aFailureReason) const override + { + return true; + } + + nsCString GetDescriptionName() const override { + return mIsHardwareAccelerated ? "apple hardware VDA decoder"_ns + : "apple software VDA decoder"_ns; + } + + void SetSeekThreshold(const media::TimeUnit& aTime) override; + + + const RefPtr mExtraData; + const uint32_t mPictureWidth; + const uint32_t mPictureHeight; + const uint32_t mDisplayWidth; + const uint32_t mDisplayHeight; + const gfx::YUVColorSpace mColorSpace; + const gfx::ColorSpace2 mColorPrimaries; + const gfx::TransferFunction mTransferFunction; + const gfx::ColorRange mColorRange; + const gfx::ColorDepth mColorDepth; + + +private: + friend class AppleDecoderModule; // To access InitializeSession. + virtual ~AppleVDADecoder(); + + // Flush and Drain operation, always run + RefPtr ProcessFlush(); + RefPtr ProcessDrain(); + void ProcessShutdown(); + void ProcessDecode(MediaRawData* aSample); + void MaybeResolveBufferedFrames(); + + void MaybeRegisterCallbackThread(); + + void AssertOnTaskQueue() { MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); } + + AppleFrameRef* CreateAppleFrameRef(const MediaRawData* aSample); + CFDictionaryRef CreateOutputConfiguration(); + // Method to set up the decompression session. + MediaResult InitializeSession(); + nsresult WaitForAsynchronousFrames(); + CFDictionaryRef CreateDecoderSpecification(); + + MOZ_DEFINE_ENUM_CLASS_WITH_TOSTRING_AT_CLASS_SCOPE(StreamType, + (Unknown, H264)); + + const StreamType mStreamType; + const RefPtr mTaskQueue; + VDADecoder mDecoder; + const uint32_t mMaxRefFrames; + const RefPtr mImageContainer; + // Increased when Input is called, and decreased when ProcessFrame runs. + // Reaching 0 indicates that there's no pending Input. + const RefPtr mKnowsCompositor; + Atomic mInputIncoming; + Atomic mIsShutDown; + const bool mUseSoftwareImages; + const Maybe mTrackingId; + + // Set on reader/decode thread calling Flush() to indicate that output is + // not required and so input samples on mTaskQueue need not be processed. + Atomic mIsFlushing; + std::atomic mCallbackThreadId; + // Protects mReorderQueue and mPromise. + Monitor mMonitor MOZ_UNANNOTATED; + ReorderQueue mReorderQueue; + MozMonitoredPromiseHolder mPromise; + + nsCString GetCodecName() const override; + + // Decoded frame will be dropped if its pts is smaller than this + // value. It shold be initialized before Input() or after Flush(). So it is + // safe to access it in OutputFrame without protecting. + Maybe mSeekTargetThreshold; + + CMVideoFormatDescriptionRef mFormat; + VTDecompressionSessionRef mSession; + Atomic mIsHardwareAccelerated; + PerformanceRecorderMulti mPerformanceRecorder; + +}; + +} // namespace mozilla + +#endif // mozilla_AppleVDADecoder_h diff --git a/dom/media/platforms/moz.build b/dom/media/platforms/moz.build index 61536cc6e225..a6e2b4287908 100644 --- a/dom/media/platforms/moz.build +++ b/dom/media/platforms/moz.build @@ -89,6 +89,7 @@ if CONFIG["MOZ_APPLEMEDIA"]: "apple/AppleATDecoder.cpp", "apple/AppleDecoderModule.cpp", "apple/AppleEncoderModule.cpp", + 'apple/AppleVDADecoder.cpp', "apple/AppleVTDecoder.cpp", "apple/AppleVTEncoder.cpp", ] @@ -99,6 +100,7 @@ if CONFIG["MOZ_APPLEMEDIA"]: "-framework AudioToolbox", "-framework CoreMedia", "-framework VideoToolbox", + "-framework VideoDecodeAcceleration", # For some unknown reason, the documented method of using weak_import # attribute doesn't work with VideoToolbox's functions. # We want to lazily load _VTRegisterSupplementalVideoDecoderIfAvailable