diff --git a/Source/WebCore/Modules/webcodecs/WebCodecsAudioDecoder.cpp b/Source/WebCore/Modules/webcodecs/WebCodecsAudioDecoder.cpp index aeb16b70a032f..c13ec18fb813a 100644 --- a/Source/WebCore/Modules/webcodecs/WebCodecsAudioDecoder.cpp +++ b/Source/WebCore/Modules/webcodecs/WebCodecsAudioDecoder.cpp @@ -161,9 +161,12 @@ ExceptionOr WebCodecsAudioDecoder::decode(Ref& --m_decodeQueueSize; scheduleDequeueEvent(); - m_internalDecoder->decode({ chunk->span(), chunk->type() == WebCodecsEncodedAudioChunkType::Key, chunk->timestamp(), chunk->duration() }, [this, pendingActivity = takePendingWebCodecActivity()](String&& result) { - if (!result.isNull()) - closeDecoder(Exception { ExceptionCode::EncodingError, WTFMove(result) }); + protectedScriptExecutionContext()->enqueueTaskWhenSettled(m_internalDecoder->decode({ chunk->span(), chunk->type() == WebCodecsEncodedAudioChunkType::Key, chunk->timestamp(), chunk->duration() }), TaskSource::MediaElement, [weakThis = ThreadSafeWeakPtr { *this }, pendingActivity = takePendingWebCodecActivity()] (auto&& result) { + RefPtr protectedThis = weakThis.get(); + if (!protectedThis || !!result) + return; + + protectedThis->closeDecoder(Exception { ExceptionCode::EncodingError, WTFMove(result.error()) }); }); } }); return { }; @@ -177,11 +180,15 @@ ExceptionOr WebCodecsAudioDecoder::flush(Ref&& promise) m_isKeyChunkRequired = true; m_pendingFlushPromises.append(WTFMove(promise)); queueControlMessageAndProcess({ *this, [this, clearFlushPromiseCount = m_clearFlushPromiseCount] { - m_internalDecoder->flush([this, clearFlushPromiseCount, pendingActivity = takePendingWebCodecActivity()] { - if (clearFlushPromiseCount != m_clearFlushPromiseCount) + protectedScriptExecutionContext()->enqueueTaskWhenSettled(m_internalDecoder->flush(), TaskSource::MediaElement, [weakThis = ThreadSafeWeakPtr { *this }, clearFlushPromiseCount, pendingActivity = takePendingWebCodecActivity()] (auto&&) { + RefPtr protectedThis = weakThis.get(); + if (!protectedThis) return; - m_pendingFlushPromises.takeFirst()->resolve(); + if (clearFlushPromiseCount != protectedThis->m_clearFlushPromiseCount) + return; + + protectedThis->m_pendingFlushPromises.takeFirst()->resolve(); }); } }); return { }; @@ -302,13 +309,14 @@ void WebCodecsAudioDecoder::processControlMessageQueue() void WebCore::WebCodecsAudioDecoder::suspend(ReasonForSuspension) { - // FIXME: Implement. } void WebCodecsAudioDecoder::stop() { m_state = WebCodecsCodecState::Closed; m_internalDecoder = nullptr; + m_controlMessageQueue.clear(); + m_pendingFlushPromises.clear(); } bool WebCodecsAudioDecoder::virtualHasPendingActivity() const diff --git a/Source/WebCore/Modules/webcodecs/WebCodecsAudioEncoder.cpp b/Source/WebCore/Modules/webcodecs/WebCodecsAudioEncoder.cpp index 3f7af2c95f4b2..cb036837362d4 100644 --- a/Source/WebCore/Modules/webcodecs/WebCodecsAudioEncoder.cpp +++ b/Source/WebCore/Modules/webcodecs/WebCodecsAudioEncoder.cpp @@ -137,7 +137,8 @@ ExceptionOr WebCodecsAudioEncoder::configure(ScriptExecutionContext&, WebC if (m_internalEncoder) { queueControlMessageAndProcess({ *this, [this, config]() mutable { m_isMessageQueueBlocked = true; - m_internalEncoder->flush([weakThis = ThreadSafeWeakPtr { *this }, config = WTFMove(config).isolatedCopy()]() mutable { + + protectedScriptExecutionContext()->enqueueTaskWhenSettled(m_internalEncoder->flush(), TaskSource::MediaElement, [weakThis = ThreadSafeWeakPtr { *this }, config = WTFMove(config)] (auto&&) mutable { RefPtr protectedThis = weakThis.get(); if (!protectedThis) return; @@ -270,13 +271,15 @@ ExceptionOr WebCodecsAudioEncoder::encode(Ref&& frame) queueControlMessageAndProcess({ *this, [this, audioData = WTFMove(audioData), timestamp = frame->timestamp(), duration = frame->duration()]() mutable { --m_encodeQueueSize; scheduleDequeueEvent(); - m_internalEncoder->encode({ WTFMove(audioData), timestamp, duration }, [this, pendingActivity = takePendingWebCodecActivity()](auto&& result) { - if (!result.isNull()) { - if (auto* context = scriptExecutionContext()) - context->addConsoleMessage(MessageSource::JS, MessageLevel::Error, makeString("AudioEncoder encode failed: "_s, result)); - closeEncoder(Exception { ExceptionCode::EncodingError, WTFMove(result) }); + + protectedScriptExecutionContext()->enqueueTaskWhenSettled(m_internalEncoder->encode({ WTFMove(audioData), timestamp, duration }), TaskSource::MediaElement, [weakThis = ThreadSafeWeakPtr { *this }, pendingActivity = takePendingWebCodecActivity()] (auto&& result) { + RefPtr protectedThis = weakThis.get(); + if (!protectedThis || !!result) return; - } + + if (auto context = protectedThis->protectedScriptExecutionContext()) + context->addConsoleMessage(MessageSource::JS, MessageLevel::Error, makeString("AudioEncoder encode failed: "_s, result.error())); + protectedThis->closeEncoder(Exception { ExceptionCode::EncodingError, WTFMove(result.error()) }); }); } }); return { }; @@ -291,12 +294,12 @@ void WebCodecsAudioEncoder::flush(Ref&& promise) m_pendingFlushPromises.append(WTFMove(promise)); queueControlMessageAndProcess({ *this, [this, clearFlushPromiseCount = m_clearFlushPromiseCount]() mutable { - m_internalEncoder->flush([this, weakThis = ThreadSafeWeakPtr { *this }, clearFlushPromiseCount, pendingActivity = takePendingWebCodecActivity()] { + protectedScriptExecutionContext()->enqueueTaskWhenSettled(m_internalEncoder->flush(), TaskSource::MediaElement, [weakThis = ThreadSafeWeakPtr { *this }, clearFlushPromiseCount, pendingActivity = takePendingWebCodecActivity()] (auto&&) { RefPtr protectedThis = weakThis.get(); - if (!protectedThis || clearFlushPromiseCount != m_clearFlushPromiseCount) + if (!protectedThis || clearFlushPromiseCount != protectedThis->m_clearFlushPromiseCount) return; - m_pendingFlushPromises.takeFirst()->resolve(); + protectedThis->m_pendingFlushPromises.takeFirst()->resolve(); }); } }); } @@ -417,12 +420,14 @@ void WebCodecsAudioEncoder::processControlMessageQueue() void WebCore::WebCodecsAudioEncoder::suspend(ReasonForSuspension) { - // FIXME: Implement. } void WebCodecsAudioEncoder::stop() { - // FIXME: Implement. + m_state = WebCodecsCodecState::Closed; + m_internalEncoder = nullptr; + m_controlMessageQueue.clear(); + m_pendingFlushPromises.clear(); } bool WebCodecsAudioEncoder::virtualHasPendingActivity() const diff --git a/Source/WebCore/Modules/webcodecs/WebCodecsVideoDecoder.cpp b/Source/WebCore/Modules/webcodecs/WebCodecsVideoDecoder.cpp index a452b5e8b8fb3..27936769aacb1 100644 --- a/Source/WebCore/Modules/webcodecs/WebCodecsVideoDecoder.cpp +++ b/Source/WebCore/Modules/webcodecs/WebCodecsVideoDecoder.cpp @@ -197,9 +197,12 @@ ExceptionOr WebCodecsVideoDecoder::decode(Ref& --m_decodeQueueSize; scheduleDequeueEvent(); - m_internalDecoder->decode({ chunk->span(), chunk->type() == WebCodecsEncodedVideoChunkType::Key, chunk->timestamp(), chunk->duration() }, [this, pendingActivity = takePendingWebCodecActivity()](auto&& result) { - if (!result.isNull()) - closeDecoder(Exception { ExceptionCode::EncodingError, WTFMove(result) }); + protectedScriptExecutionContext()->enqueueTaskWhenSettled(m_internalDecoder->decode({ chunk->span(), chunk->type() == WebCodecsEncodedVideoChunkType::Key, chunk->timestamp(), chunk->duration() }), TaskSource::MediaElement, [weakThis = ThreadSafeWeakPtr { * this }, pendingActivity = takePendingWebCodecActivity()] (auto&& result) { + RefPtr protectedThis = weakThis.get(); + if (!protectedThis || !!result) + return; + + protectedThis->closeDecoder(Exception { ExceptionCode::EncodingError, WTFMove(result.error()) }); }); } }); return { }; @@ -213,11 +216,15 @@ ExceptionOr WebCodecsVideoDecoder::flush(Ref&& promise) m_isKeyChunkRequired = true; m_pendingFlushPromises.append(WTFMove(promise)); queueControlMessageAndProcess({ *this, [this, clearFlushPromiseCount = m_clearFlushPromiseCount] { - m_internalDecoder->flush([this, clearFlushPromiseCount, pendingActivity = takePendingWebCodecActivity()] { - if (clearFlushPromiseCount != m_clearFlushPromiseCount) + protectedScriptExecutionContext()->enqueueTaskWhenSettled(m_internalDecoder->flush(), TaskSource::MediaElement, [weakThis = ThreadSafeWeakPtr { *this }, clearFlushPromiseCount, pendingActivity = takePendingWebCodecActivity()] (auto&&) { + RefPtr protectedThis = weakThis.get(); + if (!protectedThis) return; - m_pendingFlushPromises.takeFirst()->resolve(); + if (clearFlushPromiseCount != protectedThis->m_clearFlushPromiseCount) + return; + + protectedThis->m_pendingFlushPromises.takeFirst()->resolve(); }); } }); return { }; @@ -338,13 +345,14 @@ void WebCodecsVideoDecoder::processControlMessageQueue() void WebCore::WebCodecsVideoDecoder::suspend(ReasonForSuspension) { - // FIXME: Implement. } void WebCodecsVideoDecoder::stop() { m_state = WebCodecsCodecState::Closed; m_internalDecoder = nullptr; + m_controlMessageQueue.clear(); + m_pendingFlushPromises.clear(); } bool WebCodecsVideoDecoder::virtualHasPendingActivity() const diff --git a/Source/WebCore/Modules/webcodecs/WebCodecsVideoEncoder.cpp b/Source/WebCore/Modules/webcodecs/WebCodecsVideoEncoder.cpp index 65fc0f1174133..2c38bb748d595 100644 --- a/Source/WebCore/Modules/webcodecs/WebCodecsVideoEncoder.cpp +++ b/Source/WebCore/Modules/webcodecs/WebCodecsVideoEncoder.cpp @@ -113,30 +113,27 @@ static ExceptionOr createVideoEncoderConfig(const WebCodec return VideoEncoder::Config { config.width, config.height, useAnnexB, config.bitrate.value_or(0), config.framerate.value_or(0), config.latencyMode == LatencyMode::Realtime, scalabilityMode }; } -bool WebCodecsVideoEncoder::updateRates(const WebCodecsVideoEncoderConfig& config) +void WebCodecsVideoEncoder::updateRates(const WebCodecsVideoEncoderConfig& config) { auto bitrate = config.bitrate.value_or(0); auto framerate = config.framerate.value_or(0); m_isMessageQueueBlocked = true; - bool isChangingRatesSupported = m_internalEncoder->setRates(bitrate, framerate, [this, weakThis = WeakPtr { *this }, bitrate, framerate]() mutable { + protectedScriptExecutionContext()->enqueueTaskWhenSettled(m_internalEncoder->setRates(bitrate, framerate), TaskSource::MediaElement, [weakThis = ThreadSafeWeakPtr { *this }, bitrate, framerate] (auto&&) mutable { auto protectedThis = weakThis.get(); if (!protectedThis) return; - if (m_state == WebCodecsCodecState::Closed || !scriptExecutionContext()) + if (protectedThis->m_state == WebCodecsCodecState::Closed || !protectedThis->scriptExecutionContext()) return; if (bitrate) - m_baseConfiguration.bitrate = bitrate; + protectedThis->m_baseConfiguration.bitrate = bitrate; if (framerate) - m_baseConfiguration.framerate = framerate; - m_isMessageQueueBlocked = false; - processControlMessageQueue(); + protectedThis->m_baseConfiguration.framerate = framerate; + protectedThis->m_isMessageQueueBlocked = false; + protectedThis->processControlMessageQueue(); }); - if (!isChangingRatesSupported) - m_isMessageQueueBlocked = false; - return isChangingRatesSupported; } ExceptionOr WebCodecsVideoEncoder::configure(ScriptExecutionContext& context, WebCodecsVideoEncoderConfig&& config) @@ -152,11 +149,13 @@ ExceptionOr WebCodecsVideoEncoder::configure(ScriptExecutionContext& conte if (m_internalEncoder) { queueControlMessageAndProcess({ *this, [this, config]() mutable { - if (isSameConfigurationExceptBitrateAndFramerate(m_baseConfiguration, config) && updateRates(config)) + if (isSameConfigurationExceptBitrateAndFramerate(m_baseConfiguration, config)) { + updateRates(config); return; + } m_isMessageQueueBlocked = true; - m_internalEncoder->flush([weakThis = ThreadSafeWeakPtr { *this }, config = WTFMove(config).isolatedCopy(), pendingActivity = takePendingWebCodecActivity()]() mutable { + protectedScriptExecutionContext()->enqueueTaskWhenSettled(m_internalEncoder->flush(), TaskSource::MediaElement, [weakThis = ThreadSafeWeakPtr { *this }, config = WTFMove(config), pendingActivity = takePendingWebCodecActivity()] (auto&&) mutable { RefPtr protectedThis = weakThis.get(); if (!protectedThis) return; @@ -172,8 +171,10 @@ ExceptionOr WebCodecsVideoEncoder::configure(ScriptExecutionContext& conte bool isSupportedCodec = isSupportedEncoderCodec(config.codec, context.settingsValues()); queueControlMessageAndProcess({ *this, [this, config = WTFMove(config), isSupportedCodec, identifier = scriptExecutionContext()->identifier()]() mutable { - if (isSupportedCodec && isSameConfigurationExceptBitrateAndFramerate(m_baseConfiguration, config) && updateRates(config)) + if (isSupportedCodec && isSameConfigurationExceptBitrateAndFramerate(m_baseConfiguration, config)) { + updateRates(config); return; + } m_isMessageQueueBlocked = true; VideoEncoder::PostTaskCallback postTaskCallback = [weakThis = ThreadSafeWeakPtr { *this }, identifier](auto&& task) { @@ -283,18 +284,15 @@ ExceptionOr WebCodecsVideoEncoder::encode(Ref&& frame queueControlMessageAndProcess({ *this, [this, internalFrame = internalFrame.releaseNonNull(), timestamp = frame->timestamp(), duration = frame->duration(), options = WTFMove(options)]() mutable { --m_encodeQueueSize; scheduleDequeueEvent(); - m_internalEncoder->encode({ WTFMove(internalFrame), timestamp, duration }, options.keyFrame, [weakThis = ThreadSafeWeakPtr { *this }, pendingActivity = takePendingWebCodecActivity()](String&& result) { + + protectedScriptExecutionContext()->enqueueTaskWhenSettled(m_internalEncoder->encode({ WTFMove(internalFrame), timestamp, duration }, options.keyFrame), TaskSource::MediaElement, [weakThis = ThreadSafeWeakPtr { *this }, pendingActivity = takePendingWebCodecActivity()] (auto&& result) { RefPtr protectedThis = weakThis.get(); - ASSERT(protectedThis); - if (!protectedThis) + if (!protectedThis || !!result) return; - if (!result.isNull()) { - if (RefPtr context = protectedThis->scriptExecutionContext()) - context->addConsoleMessage(MessageSource::JS, MessageLevel::Error, makeString("VideoEncoder encode failed: "_s, result)); - protectedThis->closeEncoder(Exception { ExceptionCode::EncodingError, WTFMove(result) }); - return; - } + if (RefPtr context = protectedThis->scriptExecutionContext()) + context->addConsoleMessage(MessageSource::JS, MessageLevel::Error, makeString("VideoEncoder encode failed: "_s, result.error())); + protectedThis->closeEncoder(Exception { ExceptionCode::EncodingError, WTFMove(result.error()) }); }); } }); return { }; @@ -309,7 +307,7 @@ void WebCodecsVideoEncoder::flush(Ref&& promise) m_pendingFlushPromises.append(WTFMove(promise)); queueControlMessageAndProcess({ *this, [this, clearFlushPromiseCount = m_clearFlushPromiseCount]() mutable { - m_internalEncoder->flush([weakThis = ThreadSafeWeakPtr { *this }, clearFlushPromiseCount, pendingActivity = takePendingWebCodecActivity()] { + protectedScriptExecutionContext()->enqueueTaskWhenSettled(m_internalEncoder->flush(), TaskSource::MediaElement, [weakThis = ThreadSafeWeakPtr { *this }, clearFlushPromiseCount, pendingActivity = takePendingWebCodecActivity()] (auto&&) { RefPtr protectedThis = weakThis.get(); if (!protectedThis || clearFlushPromiseCount != protectedThis->m_clearFlushPromiseCount) return; @@ -435,12 +433,14 @@ void WebCodecsVideoEncoder::processControlMessageQueue() void WebCore::WebCodecsVideoEncoder::suspend(ReasonForSuspension) { - // FIXME: Implement. } void WebCodecsVideoEncoder::stop() { - // FIXME: Implement. + m_state = WebCodecsCodecState::Closed; + m_internalEncoder = nullptr; + m_controlMessageQueue.clear(); + m_pendingFlushPromises.clear(); } bool WebCodecsVideoEncoder::virtualHasPendingActivity() const diff --git a/Source/WebCore/Modules/webcodecs/WebCodecsVideoEncoder.h b/Source/WebCore/Modules/webcodecs/WebCodecsVideoEncoder.h index 80081b570a8cc..ff3f28bf0d757 100644 --- a/Source/WebCore/Modules/webcodecs/WebCodecsVideoEncoder.h +++ b/Source/WebCore/Modules/webcodecs/WebCodecsVideoEncoder.h @@ -102,7 +102,7 @@ class WebCodecsVideoEncoder void queueControlMessageAndProcess(WebCodecsControlMessage&&); void processControlMessageQueue(); WebCodecsEncodedVideoChunkMetadata createEncodedChunkMetadata(std::optional); - bool updateRates(const WebCodecsVideoEncoderConfig&); + void updateRates(const WebCodecsVideoEncoderConfig&); WebCodecsCodecState m_state { WebCodecsCodecState::Unconfigured }; size_t m_encodeQueueSize { 0 }; diff --git a/Source/WebCore/platform/AudioDecoder.h b/Source/WebCore/platform/AudioDecoder.h index f95e7866fc23d..ad176b8be1380 100644 --- a/Source/WebCore/platform/AudioDecoder.h +++ b/Source/WebCore/platform/AudioDecoder.h @@ -30,8 +30,7 @@ #include #include -#include -#include +#include namespace WebCore { @@ -70,10 +69,10 @@ class AudioDecoder { static void create(const String&, const Config&, CreateCallback&&, OutputCallback&&, PostTaskCallback&&); - using DecodeCallback = Function; - virtual void decode(EncodedData&&, DecodeCallback&&) = 0; + using DecodePromise = NativePromise; + virtual Ref decode(EncodedData&&) = 0; - virtual void flush(Function&&) = 0; + virtual Ref flush() = 0; virtual void reset() = 0; virtual void close() = 0; }; diff --git a/Source/WebCore/platform/AudioEncoder.h b/Source/WebCore/platform/AudioEncoder.h index deae00ee4cad8..079ee903373d1 100644 --- a/Source/WebCore/platform/AudioEncoder.h +++ b/Source/WebCore/platform/AudioEncoder.h @@ -34,7 +34,7 @@ #include "WebCodecsAudioInternalData.h" #include #include -#include +#include #include namespace WebCore { @@ -86,10 +86,10 @@ class AudioEncoder { static void create(const String&, const Config&, CreateCallback&&, DescriptionCallback&&, OutputCallback&&, PostTaskCallback&&); - using EncodeCallback = Function; - virtual void encode(RawFrame&&, EncodeCallback&&) = 0; + using EncodePromise = NativePromise; + virtual Ref encode(RawFrame&&) = 0; - virtual void flush(Function&&) = 0; + virtual Ref flush() = 0; virtual void reset() = 0; virtual void close() = 0; }; diff --git a/Source/WebCore/platform/VideoDecoder.h b/Source/WebCore/platform/VideoDecoder.h index 0fe12a7188f65..8ae12a9c776b3 100644 --- a/Source/WebCore/platform/VideoDecoder.h +++ b/Source/WebCore/platform/VideoDecoder.h @@ -28,8 +28,7 @@ #include "ProcessIdentity.h" #include #include -#include -#include +#include namespace WebCore { @@ -74,10 +73,10 @@ class VideoDecoder { static void create(const String&, const Config&, CreateCallback&&, OutputCallback&&, PostTaskCallback&&); WEBCORE_EXPORT static void createLocalDecoder(const String&, const Config&, CreateCallback&&, OutputCallback&&, PostTaskCallback&&); - using DecodeCallback = Function; - virtual void decode(EncodedFrame&&, DecodeCallback&&) = 0; + using DecodePromise = NativePromise; + virtual Ref decode(EncodedFrame&&) = 0; - virtual void flush(Function&&) = 0; + virtual Ref flush() = 0; virtual void reset() = 0; virtual void close() = 0; diff --git a/Source/WebCore/platform/VideoEncoder.h b/Source/WebCore/platform/VideoEncoder.h index 91c86e856cbb3..8f01f97602def 100644 --- a/Source/WebCore/platform/VideoEncoder.h +++ b/Source/WebCore/platform/VideoEncoder.h @@ -32,7 +32,7 @@ #include "VideoFrame.h" #include #include -#include +#include namespace WebCore { @@ -77,12 +77,11 @@ class VideoEncoder { static void create(const String&, const Config&, CreateCallback&&, DescriptionCallback&&, OutputCallback&&, PostTaskCallback&&); WEBCORE_EXPORT static void createLocalEncoder(const String&, const Config&, CreateCallback&&, DescriptionCallback&&, OutputCallback&&, PostTaskCallback&&); - using EncodeCallback = Function; - virtual void encode(RawFrame&&, bool shouldGenerateKeyFrame, EncodeCallback&&) = 0; + using EncodePromise = NativePromise; + virtual Ref encode(RawFrame&&, bool shouldGenerateKeyFrame) = 0; - // FIXME: Evaluate whether we can make it virtual pure and not return a boolean. - virtual bool setRates(uint64_t /* bitRate */, double /* frameRate */, Function&&) { return false; } - virtual void flush(Function&&) = 0; + virtual Ref setRates(uint64_t /* bitRate */, double /* frameRate */) = 0; + virtual Ref flush() = 0; virtual void reset() = 0; virtual void close() = 0; diff --git a/Source/WebCore/platform/audio/gstreamer/AudioDecoderGStreamer.cpp b/Source/WebCore/platform/audio/gstreamer/AudioDecoderGStreamer.cpp index 7ae46fd434d28..bdc9b11453eef 100644 --- a/Source/WebCore/platform/audio/gstreamer/AudioDecoderGStreamer.cpp +++ b/Source/WebCore/platform/audio/gstreamer/AudioDecoderGStreamer.cpp @@ -57,8 +57,8 @@ class GStreamerInternalAudioDecoder : public ThreadSafeRefCounted&& task) { m_postTaskCallback(WTFMove(task)); } - void decode(std::span, bool isKeyFrame, int64_t timestamp, std::optional duration, AudioDecoder::DecodeCallback&&); - void flush(Function&&); + Ref decode(std::span, bool isKeyFrame, int64_t timestamp, std::optional duration); + void flush(); void close() { m_isClosed = true; } bool isConfigured() const { return !!m_inputCaps; } @@ -136,17 +136,18 @@ GStreamerAudioDecoder::~GStreamerAudioDecoder() close(); } -void GStreamerAudioDecoder::decode(EncodedData&& data, DecodeCallback&& callback) +Ref GStreamerAudioDecoder::decode(EncodedData&& data) { - gstDecoderWorkQueue().dispatch([value = Vector { data.data }, isKeyFrame = data.isKeyFrame, timestamp = data.timestamp, duration = data.duration, decoder = m_internalDecoder, callback = WTFMove(callback)]() mutable { - decoder->decode({ value.data(), value.size() }, isKeyFrame, timestamp, duration, WTFMove(callback)); + return invokeAsync(gstDecoderWorkQueue(), [value = Vector { data.data }, isKeyFrame = data.isKeyFrame, timestamp = data.timestamp, duration = data.duration, decoder = m_internalDecoder] { + return decoder->decode({ value.data(), value.size() }, isKeyFrame, timestamp, duration); }); } -void GStreamerAudioDecoder::flush(Function&& callback) +Ref GStreamerAudioDecoder::flush() { - gstDecoderWorkQueue().dispatch([decoder = m_internalDecoder, callback = WTFMove(callback)]() mutable { - decoder->flush(WTFMove(callback)); + return invokeAsync(gstDecoderWorkQueue(), [decoder = m_internalDecoder] { + decoder->flush(); + return GenericPromise::createAndResolve(); }); } @@ -261,7 +262,8 @@ GStreamerInternalAudioDecoder::GStreamerInternalAudioDecoder(const String& codec harnessedElement = WTFMove(element); m_harness = GStreamerElementHarness::create(WTFMove(harnessedElement), [weakThis = WeakPtr { *this }, this](auto&, GRefPtr&& outputSample) { - if (!weakThis) + RefPtr protectedThis = weakThis.get(); + if (!protectedThis) return; if (m_isClosed) return; @@ -290,23 +292,13 @@ GStreamerInternalAudioDecoder::GStreamerInternalAudioDecoder(const String& codec }); } -void GStreamerInternalAudioDecoder::decode(std::span frameData, bool isKeyFrame, int64_t timestamp, std::optional duration, AudioDecoder::DecodeCallback&& callback) +Ref GStreamerInternalAudioDecoder::decode(std::span frameData, bool isKeyFrame, int64_t timestamp, std::optional duration) { GST_DEBUG_OBJECT(m_harness->element(), "Decoding%s frame", isKeyFrame ? " key" : ""); auto encodedData = wrapSpanData(frameData); - if (!encodedData) { - m_postTaskCallback([weakThis = WeakPtr { *this }, this, callback = WTFMove(callback)]() mutable { - if (!weakThis) - return; - if (m_isClosed) - return; - - m_outputCallback(makeUnexpected("Empty frame"_s)); - callback({ }); - }); - return; - } + if (!encodedData) + return AudioDecoder::DecodePromise::createAndReject("Empty frame"_s); GstSegment segment; gst_segment_init(&segment, GST_FORMAT_TIME); @@ -324,26 +316,17 @@ void GStreamerInternalAudioDecoder::decode(std::span frameData, b GST_BUFFER_DURATION(encodedData.get()) = *duration; auto result = m_harness->pushSample(adoptGRef(gst_sample_new(encodedData.get(), m_inputCaps.get(), &segment, nullptr))); - m_postTaskCallback([weakThis = WeakPtr { *this }, this, callback = WTFMove(callback), result]() mutable { - if (!weakThis) - return; - if (m_isClosed) - return; + if (!result) + return AudioDecoder::DecodePromise::createAndReject("Decode error"_s); - if (result) - m_harness->processOutputSamples(); - else - m_outputCallback(makeUnexpected("Decode error"_s)); - - callback({ }); - }); + m_harness->processOutputSamples(); + return AudioDecoder::DecodePromise::createAndResolve(); } -void GStreamerInternalAudioDecoder::flush(Function&& callback) +void GStreamerInternalAudioDecoder::flush() { if (m_isClosed) { GST_DEBUG_OBJECT(m_harness->element(), "Decoder closed, nothing to flush"); - m_postTaskCallback(WTFMove(callback)); return; } @@ -352,7 +335,6 @@ void GStreamerInternalAudioDecoder::flush(Function&& callback) m_harness->pushBuffer(WTFMove(buffer)); m_harness->flushBuffers(); - m_postTaskCallback(WTFMove(callback)); } #undef GST_CAT_DEFAULT diff --git a/Source/WebCore/platform/audio/gstreamer/AudioDecoderGStreamer.h b/Source/WebCore/platform/audio/gstreamer/AudioDecoderGStreamer.h index ce2fea57629e5..32590eea393ba 100644 --- a/Source/WebCore/platform/audio/gstreamer/AudioDecoderGStreamer.h +++ b/Source/WebCore/platform/audio/gstreamer/AudioDecoderGStreamer.h @@ -40,8 +40,8 @@ class GStreamerAudioDecoder : public ThreadSafeRefCounted ~GStreamerAudioDecoder(); private: - void decode(EncodedData&&, DecodeCallback&&) final; - void flush(Function&&) final; + Ref decode(EncodedData&&) final; + Ref flush() final; void reset() final; void close() final; diff --git a/Source/WebCore/platform/audio/gstreamer/AudioEncoderGStreamer.cpp b/Source/WebCore/platform/audio/gstreamer/AudioEncoderGStreamer.cpp index 139456f6862fd..4cc8142ada1e1 100644 --- a/Source/WebCore/platform/audio/gstreamer/AudioEncoderGStreamer.cpp +++ b/Source/WebCore/platform/audio/gstreamer/AudioEncoderGStreamer.cpp @@ -56,7 +56,7 @@ class GStreamerInternalAudioEncoder : public ThreadSafeRefCountedAndCanMakeThrea String initialize(const String& codecName, const AudioEncoder::Config&); void postTask(Function&& task) { m_postTaskCallback(WTFMove(task)); } bool encode(AudioEncoder::RawFrame&&); - void flush(Function&&); + void flush(); void close() { m_isClosed = true; } const RefPtr harness() const { return m_harness; } @@ -139,33 +139,26 @@ GStreamerAudioEncoder::~GStreamerAudioEncoder() close(); } -void GStreamerAudioEncoder::encode(RawFrame&& frame, EncodeCallback&& callback) +Ref GStreamerAudioEncoder::encode(RawFrame&& frame) { - gstEncoderWorkQueue().dispatch([frame = WTFMove(frame), encoder = m_internalEncoder, callback = WTFMove(callback)]() mutable { + return invokeAsync(gstEncoderWorkQueue(), [frame = WTFMove(frame), encoder = m_internalEncoder]() mutable { auto result = encoder->encode(WTFMove(frame)); if (encoder->isClosed()) - return; + return EncodePromise::createAndReject("Empty frame"_s); - String resultString; - if (result) - encoder->harness()->processOutputSamples(); - else - resultString = "Encoding failed"_s; + if (!result) + return EncodePromise::createAndReject("Encoding failed"_s); - encoder->postTask([weakEncoder = ThreadSafeWeakPtr { encoder.get() }, result = WTFMove(resultString), callback = WTFMove(callback)]() mutable { - auto encoder = weakEncoder.get(); - if (!encoder || encoder->isClosed()) - return; - - callback(WTFMove(result)); - }); + encoder->harness()->processOutputSamples(); + return EncodePromise::createAndResolve(); }); } -void GStreamerAudioEncoder::flush(Function&& callback) +Ref GStreamerAudioEncoder::flush() { - gstEncoderWorkQueue().dispatch([encoder = m_internalEncoder, callback = WTFMove(callback)]() mutable { - encoder->flush(WTFMove(callback)); + return invokeAsync(gstEncoderWorkQueue(), [encoder = m_internalEncoder] { + encoder->flush(); + return GenericPromise::createAndResolve(); }); } @@ -245,7 +238,8 @@ GStreamerInternalAudioEncoder::GStreamerInternalAudioEncoder(AudioEncoder::Descr }, static_cast(0)); m_harness = GStreamerElementHarness::create(WTFMove(harnessedElement), [weakThis = ThreadSafeWeakPtr { *this }, this](auto&, GRefPtr&& outputSample) { - if (!weakThis.get()) + RefPtr protectedThis = weakThis.get(); + if (!protectedThis) return; if (m_isClosed) return; @@ -375,10 +369,9 @@ bool GStreamerInternalAudioEncoder::encode(AudioEncoder::RawFrame&& rawFrame) return m_harness->pushSample(gstAudioFrame->sample()); } -void GStreamerInternalAudioEncoder::flush(Function&& callback) +void GStreamerInternalAudioEncoder::flush() { m_harness->flush(); - m_postTaskCallback(WTFMove(callback)); } #undef GST_CAT_DEFAULT diff --git a/Source/WebCore/platform/audio/gstreamer/AudioEncoderGStreamer.h b/Source/WebCore/platform/audio/gstreamer/AudioEncoderGStreamer.h index c09c3e7dc3196..78fa98c0f76f0 100644 --- a/Source/WebCore/platform/audio/gstreamer/AudioEncoderGStreamer.h +++ b/Source/WebCore/platform/audio/gstreamer/AudioEncoderGStreamer.h @@ -40,8 +40,8 @@ class GStreamerAudioEncoder : public AudioEncoder { ~GStreamerAudioEncoder(); private: - void encode(RawFrame&&, EncodeCallback&&) final; - void flush(Function&&) final; + Ref encode(RawFrame&&) final; + Ref flush() final; void reset() final; void close() final; diff --git a/Source/WebCore/platform/graphics/cocoa/WebCoreDecompressionSession.mm b/Source/WebCore/platform/graphics/cocoa/WebCoreDecompressionSession.mm index c41a6ac709b93..b47c39e7a5d9c 100644 --- a/Source/WebCore/platform/graphics/cocoa/WebCoreDecompressionSession.mm +++ b/Source/WebCore/platform/graphics/cocoa/WebCoreDecompressionSession.mm @@ -379,7 +379,7 @@ auto durationInUs = duration.toTimeScale(1000000); DecodingPromise::Producer producer; auto promise = producer.promise(); - m_videoDecoder->decode({ { byteCast(data), size }, true, presentationTimeInUs.timeValue(), durationInUs.timeValue() }, [weakThis = ThreadSafeWeakPtr { *this }, this, duration = PAL::toCMTime(duration), producer = WTFMove(producer)](String&&) { + m_videoDecoder->decode({ { byteCast(data), size }, true, presentationTimeInUs.timeValue(), durationInUs.timeValue() })->whenSettled(m_decompressionQueue.get(), [weakThis = ThreadSafeWeakPtr { *this }, this, duration = PAL::toCMTime(duration), producer = WTFMove(producer)] (auto&&) { RefPtr protectedThis = weakThis.get(); if (!protectedThis || isInvalidated()) { producer.reject(0); diff --git a/Source/WebCore/platform/graphics/gstreamer/VideoDecoderGStreamer.cpp b/Source/WebCore/platform/graphics/gstreamer/VideoDecoderGStreamer.cpp index a8598df5aaf21..03ed8a9a83fd9 100644 --- a/Source/WebCore/platform/graphics/gstreamer/VideoDecoderGStreamer.cpp +++ b/Source/WebCore/platform/graphics/gstreamer/VideoDecoderGStreamer.cpp @@ -62,8 +62,8 @@ class GStreamerInternalVideoDecoder : public ThreadSafeRefCounted&& task) { m_postTaskCallback(WTFMove(task)); } - void decode(std::span, bool isKeyFrame, int64_t timestamp, std::optional duration, VideoDecoder::DecodeCallback&&); - void flush(Function&&); + Ref decode(std::span, bool isKeyFrame, int64_t timestamp, std::optional duration); + void flush(); void close() { m_isClosed = true; } bool isConfigured() const { return !!m_inputCaps; } @@ -131,17 +131,18 @@ GStreamerVideoDecoder::~GStreamerVideoDecoder() close(); } -void GStreamerVideoDecoder::decode(EncodedFrame&& frame, DecodeCallback&& callback) +Ref GStreamerVideoDecoder::decode(EncodedFrame&& frame) { - gstDecoderWorkQueue().dispatch([value = Vector { frame.data }, isKeyFrame = frame.isKeyFrame, timestamp = frame.timestamp, duration = frame.duration, decoder = m_internalDecoder, callback = WTFMove(callback)]() mutable { - decoder->decode({ value.data(), value.size() }, isKeyFrame, timestamp, duration, WTFMove(callback)); + return invokeAsync(gstDecoderWorkQueue(), [value = Vector { frame.data }, isKeyFrame = frame.isKeyFrame, timestamp = frame.timestamp, duration = frame.duration, decoder = m_internalDecoder] { + return decoder->decode({ value.data(), value.size() }, isKeyFrame, timestamp, duration); }); } -void GStreamerVideoDecoder::flush(Function&& callback) +Ref GStreamerVideoDecoder::flush() { - gstDecoderWorkQueue().dispatch([decoder = m_internalDecoder, callback = WTFMove(callback)]() mutable { - decoder->flush(WTFMove(callback)); + return invokeAsync(gstDecoderWorkQueue(), [decoder = m_internalDecoder] { + decoder->flush(); + return GenericPromise::createAndResolve(); }); } @@ -251,23 +252,12 @@ GStreamerInternalVideoDecoder::GStreamerInternalVideoDecoder(const String& codec }); } -void GStreamerInternalVideoDecoder::decode(std::span frameData, bool isKeyFrame, int64_t timestamp, std::optional duration, VideoDecoder::DecodeCallback&& callback) +Ref GStreamerInternalVideoDecoder::decode(std::span frameData, bool isKeyFrame, int64_t timestamp, std::optional duration) { GST_DEBUG_OBJECT(m_harness->element(), "Decoding%s frame", isKeyFrame ? " key" : ""); auto buffer = wrapSpanData(frameData); - if (!buffer) { - m_postTaskCallback([weakThis = WeakPtr { *this }, this, callback = WTFMove(callback)]() mutable { - if (!weakThis) - return; - - if (m_isClosed) - return; - - m_outputCallback(makeUnexpected("Empty frame"_s)); - callback({ }); - }); - return; - } + if (!buffer) + return VideoDecoder::DecodePromise::createAndReject("Empty frame"_s); m_timestamp = timestamp; m_duration = duration; @@ -281,32 +271,21 @@ void GStreamerInternalVideoDecoder::decode(std::span frameData, b // FIXME: Maybe configure segment here, could be useful for reverse playback. auto result = m_harness->pushSample(adoptGRef(gst_sample_new(buffer.get(), m_inputCaps.get(), nullptr, nullptr))); - if (result) - m_harness->processOutputSamples(); - m_postTaskCallback([weakThis = WeakPtr { *this }, this, callback = WTFMove(callback), result]() mutable { - if (!weakThis) - return; + if (!result) + return VideoDecoder::DecodePromise::createAndReject("Decode error"_s); - if (weakThis->m_isClosed) - return; - - if (!result) - m_outputCallback(makeUnexpected("Decode error"_s)); - - callback({ }); - }); + m_harness->processOutputSamples(); + return VideoDecoder::DecodePromise::createAndResolve(); } -void GStreamerInternalVideoDecoder::flush(Function&& callback) +void GStreamerInternalVideoDecoder::flush() { if (m_isClosed) { GST_DEBUG_OBJECT(m_harness->element(), "Decoder closed, nothing to flush"); - m_postTaskCallback(WTFMove(callback)); return; } m_harness->reset(); - m_postTaskCallback(WTFMove(callback)); } #undef GST_CAT_DEFAULT diff --git a/Source/WebCore/platform/graphics/gstreamer/VideoDecoderGStreamer.h b/Source/WebCore/platform/graphics/gstreamer/VideoDecoderGStreamer.h index fdcd2be4b58d0..28ae3656a0c03 100644 --- a/Source/WebCore/platform/graphics/gstreamer/VideoDecoderGStreamer.h +++ b/Source/WebCore/platform/graphics/gstreamer/VideoDecoderGStreamer.h @@ -41,8 +41,8 @@ class GStreamerVideoDecoder : public ThreadSafeRefCounted ~GStreamerVideoDecoder(); private: - void decode(EncodedFrame&&, DecodeCallback&&) final; - void flush(Function&&) final; + Ref decode(EncodedFrame&&) final; + Ref flush() final; void reset() final; void close() final; diff --git a/Source/WebCore/platform/graphics/gstreamer/VideoEncoderGStreamer.cpp b/Source/WebCore/platform/graphics/gstreamer/VideoEncoderGStreamer.cpp index 1f8f1ded22120..7c5a9dafb417f 100644 --- a/Source/WebCore/platform/graphics/gstreamer/VideoEncoderGStreamer.cpp +++ b/Source/WebCore/platform/graphics/gstreamer/VideoEncoderGStreamer.cpp @@ -52,21 +52,24 @@ class GStreamerInternalVideoEncoder : public ThreadSafeRefCountedAndCanMakeThrea WTF_MAKE_NONCOPYABLE(GStreamerInternalVideoEncoder); public: - static Ref create(VideoEncoder::DescriptionCallback&& descriptionCallback, VideoEncoder::OutputCallback&& outputCallback, VideoEncoder::PostTaskCallback&& postTaskCallback) { return adoptRef(*new GStreamerInternalVideoEncoder(WTFMove(descriptionCallback), WTFMove(outputCallback), WTFMove(postTaskCallback))); } + static Ref create(const VideoEncoder::Config& config, VideoEncoder::DescriptionCallback&& descriptionCallback, VideoEncoder::OutputCallback&& outputCallback, VideoEncoder::PostTaskCallback&& postTaskCallback) { return adoptRef(*new GStreamerInternalVideoEncoder(config, WTFMove(descriptionCallback), WTFMove(outputCallback), WTFMove(postTaskCallback))); } ~GStreamerInternalVideoEncoder(); - String initialize(const String& codecName, const VideoEncoder::Config&); + String initialize(const String& codecName); void postTask(Function&& task) { m_postTaskCallback(WTFMove(task)); } bool encode(VideoEncoder::RawFrame&&, bool shouldGenerateKeyFrame); - void flush(Function&&); + void setRates(uint64_t bitRate, double frameRate); + void applyRates(); + void flush(); void close() { m_isClosed = true; } const RefPtr harness() const { return m_harness; } bool isClosed() const { return m_isClosed; } private: - GStreamerInternalVideoEncoder(VideoEncoder::DescriptionCallback&&, VideoEncoder::OutputCallback&&, VideoEncoder::PostTaskCallback&&); + GStreamerInternalVideoEncoder(const VideoEncoder::Config&, VideoEncoder::DescriptionCallback&&, VideoEncoder::OutputCallback&&, VideoEncoder::PostTaskCallback&&); + VideoEncoder::Config m_config; VideoEncoder::DescriptionCallback m_descriptionCallback; VideoEncoder::OutputCallback m_outputCallback; VideoEncoder::PostTaskCallback m_postTaskCallback; @@ -97,9 +100,9 @@ void GStreamerVideoEncoder::create(const String& codecName, const VideoEncoder:: return; } - auto encoder = makeUniqueRef(WTFMove(descriptionCallback), WTFMove(outputCallback), WTFMove(postTaskCallback)); + auto encoder = makeUniqueRef(config, WTFMove(descriptionCallback), WTFMove(outputCallback), WTFMove(postTaskCallback)); auto internalEncoder = encoder->m_internalEncoder; - auto error = internalEncoder->initialize(codecName, config); + auto error = internalEncoder->initialize(codecName); if (!error.isEmpty()) { encoder->m_internalEncoder->postTask([callback = WTFMove(callback), error = WTFMove(error)]() mutable { GST_WARNING("Error creating encoder: %s", error.ascii().data()); @@ -116,8 +119,8 @@ void GStreamerVideoEncoder::create(const String& codecName, const VideoEncoder:: }); } -GStreamerVideoEncoder::GStreamerVideoEncoder(DescriptionCallback&& descriptionCallback, OutputCallback&& outputCallback, PostTaskCallback&& postTaskCallback) - : m_internalEncoder(GStreamerInternalVideoEncoder::create(WTFMove(descriptionCallback), WTFMove(outputCallback), WTFMove(postTaskCallback))) +GStreamerVideoEncoder::GStreamerVideoEncoder(const VideoEncoder::Config& config, DescriptionCallback&& descriptionCallback, OutputCallback&& outputCallback, PostTaskCallback&& postTaskCallback) + : m_internalEncoder(GStreamerInternalVideoEncoder::create(config, WTFMove(descriptionCallback), WTFMove(outputCallback), WTFMove(postTaskCallback))) { } @@ -127,33 +130,31 @@ GStreamerVideoEncoder::~GStreamerVideoEncoder() close(); } -void GStreamerVideoEncoder::encode(RawFrame&& frame, bool shouldGenerateKeyFrame, EncodeCallback&& callback) +Ref GStreamerVideoEncoder::encode(RawFrame&& frame, bool shouldGenerateKeyFrame) { - gstEncoderWorkQueue().dispatch([frame = WTFMove(frame), shouldGenerateKeyFrame, encoder = m_internalEncoder, callback = WTFMove(callback)]() mutable { + return invokeAsync(gstEncoderWorkQueue(), [frame = WTFMove(frame), shouldGenerateKeyFrame, encoder = m_internalEncoder]() mutable { auto result = encoder->encode(WTFMove(frame), shouldGenerateKeyFrame); - if (encoder->isClosed()) - return; - - String resultString; - if (result) - encoder->harness()->processOutputSamples(); - else - resultString = "Encoding failed"_s; + if (!result) + return EncodePromise::createAndReject("Encoding failed"_s); - encoder->postTask([weakEncoder = ThreadSafeWeakPtr { encoder.get() }, result = WTFMove(resultString), callback = WTFMove(callback)]() mutable { - auto encoder = weakEncoder.get(); - if (!encoder || encoder->isClosed()) - return; + encoder->harness()->processOutputSamples(); + return EncodePromise::createAndResolve(); + }); +} - callback(WTFMove(result)); - }); +Ref GStreamerVideoEncoder::flush() +{ + return invokeAsync(gstEncoderWorkQueue(), [encoder = m_internalEncoder] { + encoder->flush(); + return GenericPromise::createAndResolve(); }); } -void GStreamerVideoEncoder::flush(Function&& callback) +Ref GStreamerVideoEncoder::setRates(uint64_t bitRate, double frameRate) { - gstEncoderWorkQueue().dispatch([encoder = m_internalEncoder, callback = WTFMove(callback)]() mutable { - encoder->flush(WTFMove(callback)); + return invokeAsync(gstEncoderWorkQueue(), [encoder = m_internalEncoder, bitRate, frameRate] { + encoder->setRates(bitRate, frameRate); + return GenericPromise::createAndResolve(); }); } @@ -193,8 +194,9 @@ static std::optional retrieveTemporalIndex(const GRefPtr& s return { }; } -GStreamerInternalVideoEncoder::GStreamerInternalVideoEncoder(VideoEncoder::DescriptionCallback&& descriptionCallback, VideoEncoder::OutputCallback&& outputCallback, VideoEncoder::PostTaskCallback&& postTaskCallback) - : m_descriptionCallback(WTFMove(descriptionCallback)) +GStreamerInternalVideoEncoder::GStreamerInternalVideoEncoder(const VideoEncoder::Config& config, VideoEncoder::DescriptionCallback&& descriptionCallback, VideoEncoder::OutputCallback&& outputCallback, VideoEncoder::PostTaskCallback&& postTaskCallback) + : m_config(config) + , m_descriptionCallback(WTFMove(descriptionCallback)) , m_outputCallback(WTFMove(outputCallback)) , m_postTaskCallback(WTFMove(postTaskCallback)) { @@ -243,7 +245,8 @@ GStreamerInternalVideoEncoder::GStreamerInternalVideoEncoder(VideoEncoder::Descr }, static_cast(0)); m_harness = GStreamerElementHarness::create(WTFMove(element), [weakThis = ThreadSafeWeakPtr { *this }, this](auto&, GRefPtr&& outputSample) { - if (!weakThis.get()) + RefPtr protectedThis = weakThis.get(); + if (!protectedThis) return; if (m_isClosed) return; @@ -281,36 +284,14 @@ GStreamerInternalVideoEncoder::~GStreamerInternalVideoEncoder() g_signal_handlers_disconnect_by_data(pad.get(), this); } -String GStreamerInternalVideoEncoder::initialize(const String& codecName, const VideoEncoder::Config& config) +String GStreamerInternalVideoEncoder::initialize(const String& codecName) { GST_DEBUG_OBJECT(m_harness->element(), "Initializing encoder for codec %s", codecName.ascii().data()); - IntSize size { static_cast(config.width), static_cast(config.height) }; - // FIXME: Propagate config.frameRate to caps? + IntSize size { static_cast(m_config.width), static_cast(m_config.height) }; if (!videoEncoderSetCodec(WEBKIT_VIDEO_ENCODER(m_harness->element()), codecName, { size })) return "Unable to set encoder format"_s; - if (config.bitRate > 1000) - g_object_set(m_harness->element(), "bitrate", static_cast(config.bitRate / 1000), nullptr); - - auto bitRateAllocation = WebKitVideoEncoderBitRateAllocation::create(config.scalabilityMode); - auto totalBitRate = config.bitRate ? config.bitRate : 3 * config.width * config.height; - switch (config.scalabilityMode) { - case VideoEncoder::ScalabilityMode::L1T1: - bitRateAllocation->setBitRate(0, 0, totalBitRate); - break; - case VideoEncoder::ScalabilityMode::L1T2: - m_hasMultipleTemporalLayers = true; - bitRateAllocation->setBitRate(0, 0, totalBitRate * 0.6); - bitRateAllocation->setBitRate(0, 1, totalBitRate * 0.4); - break; - case VideoEncoder::ScalabilityMode::L1T3: - m_hasMultipleTemporalLayers = true; - bitRateAllocation->setBitRate(0, 0, totalBitRate * 0.5); - bitRateAllocation->setBitRate(0, 1, totalBitRate * 0.3); - bitRateAllocation->setBitRate(0, 2, totalBitRate * 0.2); - break; - } - videoEncoderSetBitRateAllocation(WEBKIT_VIDEO_ENCODER(m_harness->element()), WTFMove(bitRateAllocation)); + applyRates(); m_isInitialized = true; return emptyString(); @@ -337,10 +318,43 @@ bool GStreamerInternalVideoEncoder::encode(VideoEncoder::RawFrame&& rawFrame, bo return m_harness->pushSample(gstVideoFrame.downloadSample()); } -void GStreamerInternalVideoEncoder::flush(Function && callback) +void GStreamerInternalVideoEncoder::setRates(uint64_t bitRate, double frameRate) +{ + m_config.bitRate = bitRate; + m_config.frameRate = frameRate; + applyRates(); +} + +void GStreamerInternalVideoEncoder::applyRates() +{ + // FIXME: Propagate m_config.frameRate to caps? + if (m_config.bitRate > 1000) + g_object_set(m_harness->element(), "bitrate", static_cast(m_config.bitRate / 1000), nullptr); + + auto bitRateAllocation = WebKitVideoEncoderBitRateAllocation::create(m_config.scalabilityMode); + auto totalBitRate = m_config.bitRate ? m_config.bitRate : 3 * m_config.width * m_config.height; + switch (m_config.scalabilityMode) { + case VideoEncoder::ScalabilityMode::L1T1: + bitRateAllocation->setBitRate(0, 0, totalBitRate); + break; + case VideoEncoder::ScalabilityMode::L1T2: + m_hasMultipleTemporalLayers = true; + bitRateAllocation->setBitRate(0, 0, totalBitRate * 0.6); + bitRateAllocation->setBitRate(0, 1, totalBitRate * 0.4); + break; + case VideoEncoder::ScalabilityMode::L1T3: + m_hasMultipleTemporalLayers = true; + bitRateAllocation->setBitRate(0, 0, totalBitRate * 0.5); + bitRateAllocation->setBitRate(0, 1, totalBitRate * 0.3); + bitRateAllocation->setBitRate(0, 2, totalBitRate * 0.2); + break; + } + videoEncoderSetBitRateAllocation(WEBKIT_VIDEO_ENCODER(m_harness->element()), WTFMove(bitRateAllocation)); +} + +void GStreamerInternalVideoEncoder::flush() { m_harness->flush(); - m_postTaskCallback(WTFMove(callback)); } #undef GST_CAT_DEFAULT diff --git a/Source/WebCore/platform/graphics/gstreamer/VideoEncoderGStreamer.h b/Source/WebCore/platform/graphics/gstreamer/VideoEncoderGStreamer.h index 879d306839da3..5f334caa1526f 100644 --- a/Source/WebCore/platform/graphics/gstreamer/VideoEncoderGStreamer.h +++ b/Source/WebCore/platform/graphics/gstreamer/VideoEncoderGStreamer.h @@ -35,14 +35,15 @@ class GStreamerVideoEncoder : public VideoEncoder { public: static void create(const String& codecName, const Config&, CreateCallback&&, DescriptionCallback&&, OutputCallback&&, PostTaskCallback&&); - GStreamerVideoEncoder(DescriptionCallback&&, OutputCallback&&, PostTaskCallback&&); + GStreamerVideoEncoder(const Config&, DescriptionCallback&&, OutputCallback&&, PostTaskCallback&&); ~GStreamerVideoEncoder(); private: - void encode(RawFrame&&, bool shouldGenerateKeyFrame, EncodeCallback&&) final; - void flush(Function&&) final; + Ref encode(RawFrame&&, bool shouldGenerateKeyFrame) final; + Ref flush() final; void reset() final; void close() final; + Ref setRates(uint64_t bitRate, double frameRate) final; Ref m_internalEncoder; }; diff --git a/Source/WebCore/platform/libwebrtc/LibWebRTCVPXVideoDecoder.cpp b/Source/WebCore/platform/libwebrtc/LibWebRTCVPXVideoDecoder.cpp index 98ad7f5cd755e..482a2e6a9969d 100644 --- a/Source/WebCore/platform/libwebrtc/LibWebRTCVPXVideoDecoder.cpp +++ b/Source/WebCore/platform/libwebrtc/LibWebRTCVPXVideoDecoder.cpp @@ -69,7 +69,7 @@ class LibWebRTCVPXInternalVideoDecoder : public ThreadSafeRefCounted&& task) { m_postTaskCallback(WTFMove(task)); } - void decode(std::span, bool isKeyFrame, int64_t timestamp, std::optional duration, VideoDecoder::DecodeCallback&&); + Ref decode(std::span, bool isKeyFrame, int64_t timestamp, std::optional duration); void close() { m_isClosed = true; } private: LibWebRTCVPXInternalVideoDecoder(LibWebRTCVPXVideoDecoder::Type, const VideoDecoder::Config&, VideoDecoder::OutputCallback&&, VideoDecoder::PostTaskCallback&&); @@ -112,17 +112,17 @@ LibWebRTCVPXVideoDecoder::~LibWebRTCVPXVideoDecoder() { } -void LibWebRTCVPXVideoDecoder::decode(EncodedFrame&& frame, DecodeCallback&& callback) +Ref LibWebRTCVPXVideoDecoder::decode(EncodedFrame&& frame) { - vpxDecoderQueue().dispatch([value = Vector { frame.data }, isKeyFrame = frame.isKeyFrame, timestamp = frame.timestamp, duration = frame.duration, decoder = m_internalDecoder, callback = WTFMove(callback)]() mutable { - decoder->decode({ value.data(), value.size() }, isKeyFrame, timestamp, duration, WTFMove(callback)); + return invokeAsync(vpxDecoderQueue(), [value = Vector { frame.data }, isKeyFrame = frame.isKeyFrame, timestamp = frame.timestamp, duration = frame.duration, decoder = m_internalDecoder] { + return decoder->decode({ value.data(), value.size() }, isKeyFrame, timestamp, duration); }); } -void LibWebRTCVPXVideoDecoder::flush(Function&& callback) +Ref LibWebRTCVPXVideoDecoder::flush() { - vpxDecoderQueue().dispatch([decoder = m_internalDecoder, callback = WTFMove(callback)]() mutable { - decoder->postTask(WTFMove(callback)); + return invokeAsync(vpxDecoderQueue(), [] { + return GenericPromise::createAndResolve(); }); } @@ -137,7 +137,7 @@ void LibWebRTCVPXVideoDecoder::close() } -void LibWebRTCVPXInternalVideoDecoder::decode(std::span data, bool isKeyFrame, int64_t timestamp, std::optional duration, VideoDecoder::DecodeCallback&& callback) +Ref LibWebRTCVPXInternalVideoDecoder::decode(std::span data, bool isKeyFrame, int64_t timestamp, std::optional duration) { m_timestamp = timestamp; m_duration = duration; @@ -148,15 +148,10 @@ void LibWebRTCVPXInternalVideoDecoder::decode(std::span data, boo auto error = m_internalDecoder->Decode(image, false, 0); - m_postTaskCallback([protectedThis = Ref { *this }, error, callback = WTFMove(callback)]() mutable { - if (protectedThis->m_isClosed) - return; + if (error) + return VideoDecoder::DecodePromise::createAndReject(makeString("VPx decoding failed with error "_s, error)); - if (error) - protectedThis->m_outputCallback(makeUnexpected(makeString("VPx decoding failed with error "_s, error))); - - callback({ }); - }); + return VideoDecoder::DecodePromise::createAndResolve(); } static UniqueRef createInternalDecoder(LibWebRTCVPXVideoDecoder::Type type) diff --git a/Source/WebCore/platform/libwebrtc/LibWebRTCVPXVideoDecoder.h b/Source/WebCore/platform/libwebrtc/LibWebRTCVPXVideoDecoder.h index 376fb7079016d..907b87ceabca8 100644 --- a/Source/WebCore/platform/libwebrtc/LibWebRTCVPXVideoDecoder.h +++ b/Source/WebCore/platform/libwebrtc/LibWebRTCVPXVideoDecoder.h @@ -52,8 +52,8 @@ class LibWebRTCVPXVideoDecoder : public VideoDecoder { ~LibWebRTCVPXVideoDecoder(); private: - void decode(EncodedFrame&&, DecodeCallback&&) final; - void flush(Function&&) final; + Ref decode(EncodedFrame&&) final; + Ref flush() final; void reset() final; void close() final; diff --git a/Source/WebCore/platform/libwebrtc/LibWebRTCVPXVideoEncoder.cpp b/Source/WebCore/platform/libwebrtc/LibWebRTCVPXVideoEncoder.cpp index 5448bbf83c405..8fe220d03a908 100644 --- a/Source/WebCore/platform/libwebrtc/LibWebRTCVPXVideoEncoder.cpp +++ b/Source/WebCore/platform/libwebrtc/LibWebRTCVPXVideoEncoder.cpp @@ -68,7 +68,7 @@ class LibWebRTCVPXInternalVideoEncoder : public ThreadSafeRefCounted&& task) { m_postTaskCallback(WTFMove(task)); } - void encode(VideoEncoder::RawFrame&&, bool shouldGenerateKeyFrame, VideoEncoder::EncodeCallback&&); + Ref encode(VideoEncoder::RawFrame&&, bool shouldGenerateKeyFrame); void close() { m_isClosed = true; } void setRates(uint64_t bitRate, double frameRate); @@ -124,17 +124,17 @@ int LibWebRTCVPXVideoEncoder::initialize(LibWebRTCVPXVideoEncoder::Type type, co return m_internalEncoder->initialize(type, config); } -void LibWebRTCVPXVideoEncoder::encode(RawFrame&& frame, bool shouldGenerateKeyFrame, EncodeCallback&& callback) +Ref LibWebRTCVPXVideoEncoder::encode(RawFrame&& frame, bool shouldGenerateKeyFrame) { - vpxEncoderQueue().dispatch([frame = WTFMove(frame), shouldGenerateKeyFrame, encoder = m_internalEncoder, callback = WTFMove(callback)]() mutable { - encoder->encode(WTFMove(frame), shouldGenerateKeyFrame, WTFMove(callback)); + return invokeAsync(vpxEncoderQueue(), [frame = WTFMove(frame), shouldGenerateKeyFrame, encoder = m_internalEncoder]() mutable { + return encoder->encode(WTFMove(frame), shouldGenerateKeyFrame); }); } -void LibWebRTCVPXVideoEncoder::flush(Function&& callback) +Ref LibWebRTCVPXVideoEncoder::flush() { - vpxEncoderQueue().dispatch([encoder = m_internalEncoder, callback = WTFMove(callback)]() mutable { - encoder->postTask(WTFMove(callback)); + return invokeAsync(vpxEncoderQueue(), [] { + return GenericPromise::createAndResolve(); }); } @@ -148,13 +148,12 @@ void LibWebRTCVPXVideoEncoder::close() m_internalEncoder->close(); } -bool LibWebRTCVPXVideoEncoder::setRates(uint64_t bitRate, double frameRate, Function&& callback) +Ref LibWebRTCVPXVideoEncoder::setRates(uint64_t bitRate, double frameRate) { - vpxEncoderQueue().dispatch([encoder = m_internalEncoder, bitRate, frameRate, callback = WTFMove(callback)]() mutable { + return invokeAsync(vpxEncoderQueue(), [encoder = m_internalEncoder, bitRate, frameRate] { encoder->setRates(bitRate, frameRate); - encoder->postTask(WTFMove(callback)); + return GenericPromise::createAndResolve(); }); - return true; } static UniqueRef createInternalEncoder(LibWebRTCVPXVideoEncoder::Type type) @@ -264,10 +263,10 @@ int LibWebRTCVPXInternalVideoEncoder::initialize(LibWebRTCVPXVideoEncoder::Type return 0; } -void LibWebRTCVPXInternalVideoEncoder::encode(VideoEncoder::RawFrame&& rawFrame, bool shouldGenerateKeyFrame, VideoEncoder::EncodeCallback&& callback) +Ref LibWebRTCVPXInternalVideoEncoder::encode(VideoEncoder::RawFrame&& rawFrame, bool shouldGenerateKeyFrame) { if (!m_isInitialized) - return; + return VideoEncoder::EncodePromise::createAndReject("Encoder is not initialized"_s); if (rawFrame.timestamp + m_timestampOffset <= 0) m_timestampOffset = 1 - rawFrame.timestamp; @@ -288,15 +287,10 @@ void LibWebRTCVPXInternalVideoEncoder::encode(VideoEncoder::RawFrame&& rawFrame, if (!m_hasEncoded) m_hasEncoded = !error; - m_postTaskCallback([protectedThis = Ref { *this }, error, callback = WTFMove(callback)]() mutable { - if (protectedThis->m_isClosed) - return; + if (error) + return VideoEncoder::EncodePromise::createAndReject("Encoder is not initialized"_s); - String result; - if (error) - result = makeString("VPx encoding failed with error "_s, error); - callback(WTFMove(result)); - }); + return VideoEncoder::EncodePromise::createAndResolve(); } void LibWebRTCVPXInternalVideoEncoder::setRates(uint64_t bitRate, double frameRate) diff --git a/Source/WebCore/platform/libwebrtc/LibWebRTCVPXVideoEncoder.h b/Source/WebCore/platform/libwebrtc/LibWebRTCVPXVideoEncoder.h index 9bbe24a7d2de6..db0760d47d3e9 100644 --- a/Source/WebCore/platform/libwebrtc/LibWebRTCVPXVideoEncoder.h +++ b/Source/WebCore/platform/libwebrtc/LibWebRTCVPXVideoEncoder.h @@ -53,11 +53,11 @@ class LibWebRTCVPXVideoEncoder : public VideoEncoder { private: int initialize(LibWebRTCVPXVideoEncoder::Type, const VideoEncoder::Config&); - void encode(RawFrame&&, bool shouldGenerateKeyFrame, EncodeCallback&&) final; - void flush(Function&&) final; + Ref encode(RawFrame&&, bool shouldGenerateKeyFrame) final; + Ref flush() final; void reset() final; void close() final; - bool setRates(uint64_t bitRate, double frameRate, Function&&) final; + Ref setRates(uint64_t bitRate, double frameRate) final; Ref m_internalEncoder; }; diff --git a/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.h b/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.h index 82c994d1b1cf2..b70e72e850dd4 100644 --- a/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.h +++ b/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.h @@ -85,7 +85,7 @@ class LibWebRTCCodecsProxy final : public IPC::WorkQueueMessageReceiver { void createDecoder(VideoDecoderIdentifier, WebCore::VideoCodecType, const String& codecString, bool useRemoteFrames, bool enableAdditionalLogging, CompletionHandler&&); void releaseDecoder(VideoDecoderIdentifier); - void flushDecoder(VideoDecoderIdentifier); + void flushDecoder(VideoDecoderIdentifier, CompletionHandler&&); void setDecoderFormatDescription(VideoDecoderIdentifier, std::span, uint16_t width, uint16_t height); void decodeFrame(VideoDecoderIdentifier, int64_t timeStamp, std::span, CompletionHandler&&); void setFrameSize(VideoDecoderIdentifier, uint16_t width, uint16_t height); @@ -95,7 +95,7 @@ class LibWebRTCCodecsProxy final : public IPC::WorkQueueMessageReceiver { void initializeEncoder(VideoEncoderIdentifier, uint16_t width, uint16_t height, unsigned startBitrate, unsigned maxBitrate, unsigned minBitrate, uint32_t maxFramerate); void encodeFrame(VideoEncoderIdentifier, SharedVideoFrame&&, int64_t timeStamp, std::optional duration, bool shouldEncodeAsKeyFrame, CompletionHandler&&); void flushEncoder(VideoEncoderIdentifier, CompletionHandler&&); - void setEncodeRates(VideoEncoderIdentifier, uint32_t bitRate, uint32_t frameRate); + void setEncodeRates(VideoEncoderIdentifier, uint32_t bitRate, uint32_t frameRate, CompletionHandler&&); void setSharedVideoFrameSemaphore(VideoEncoderIdentifier, IPC::Semaphore&&); void setSharedVideoFrameMemory(VideoEncoderIdentifier, WebCore::SharedMemory::Handle&&); void setRTCLoggingLevel(WTFLogLevel); diff --git a/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.messages.in b/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.messages.in index 7ad14a3e1117d..0ffad063ba0bf 100644 --- a/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.messages.in +++ b/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.messages.in @@ -26,7 +26,7 @@ messages -> LibWebRTCCodecsProxy NotRefCounted { CreateDecoder(WebKit::VideoDecoderIdentifier id, enum:uint8_t WebCore::VideoCodecType codecType, String codecString, bool useRemoteFrames, bool enableAdditionalLogging) -> (bool success); ReleaseDecoder(WebKit::VideoDecoderIdentifier id) - FlushDecoder(WebKit::VideoDecoderIdentifier id) + FlushDecoder(WebKit::VideoDecoderIdentifier id) -> () SetDecoderFormatDescription(WebKit::VideoDecoderIdentifier id, std::span description, uint16_t width, uint16_t height) DecodeFrame(WebKit::VideoDecoderIdentifier id, int64_t timeStamp, std::span data) -> (bool success) SetFrameSize(WebKit::VideoDecoderIdentifier id, uint16_t width, uint16_t height) @@ -36,7 +36,7 @@ messages -> LibWebRTCCodecsProxy NotRefCounted { InitializeEncoder(WebKit::VideoEncoderIdentifier id, uint16_t width, uint16_t height, unsigned startBitrate, unsigned maxBitrate, unsigned minBitrate, uint32_t maxFramerate) EncodeFrame(WebKit::VideoEncoderIdentifier id, struct WebKit::SharedVideoFrame buffer, int64_t timeStamp, std::optional duration, bool shouldEncodeAsKeyFrame) -> (bool success) FlushEncoder(WebKit::VideoEncoderIdentifier id) -> () - SetEncodeRates(WebKit::VideoEncoderIdentifier id, uint32_t bitRate, uint32_t frameRate) + SetEncodeRates(WebKit::VideoEncoderIdentifier id, uint32_t bitRate, uint32_t frameRate) -> () SetSharedVideoFrameSemaphore(WebKit::VideoEncoderIdentifier id, IPC::Semaphore semaphore) SetSharedVideoFrameMemory(WebKit::VideoEncoderIdentifier id, WebCore::SharedMemory::Handle storageHandle) SetRTCLoggingLevel(WTFLogLevel level) diff --git a/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.mm b/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.mm index 5ee6aa2eb7064..3c3294f030b1d 100644 --- a/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.mm +++ b/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.mm @@ -46,6 +46,7 @@ #import #import #import +#import #import ALLOW_COMMA_BEGIN @@ -226,11 +227,12 @@ static bool validateCodecString(WebCore::VideoCodecType codecType, const String& m_hasEncodersOrDecoders = !m_encoders.isEmpty() || !m_decoders.isEmpty(); } -void LibWebRTCCodecsProxy::flushDecoder(VideoDecoderIdentifier identifier) +void LibWebRTCCodecsProxy::flushDecoder(VideoDecoderIdentifier identifier, CompletionHandler&& completionHandler) { doDecoderTask(identifier, [&](auto& decoder) { decoder.webrtcDecoder->flush(); - m_connection->send(Messages::LibWebRTCCodecs::FlushDecoderCompleted { identifier }, 0); + // FIXME: It would be nice to ASSERT that when executing callback, the decoding task deque is empty. + workQueue().dispatch(WTFMove(completionHandler)); }); } @@ -492,8 +494,12 @@ static bool validateEncoderConfiguration(WebCore::VideoCodecType codecType, cons workQueue().dispatch(WTFMove(callback)); } -void LibWebRTCCodecsProxy::setEncodeRates(VideoEncoderIdentifier identifier, uint32_t bitRate, uint32_t frameRate) +void LibWebRTCCodecsProxy::setEncodeRates(VideoEncoderIdentifier identifier, uint32_t bitRate, uint32_t frameRate, CompletionHandler&& callback) { + auto scope = makeScopeExit([&] { + callback(); + }); + assertIsCurrent(workQueue()); auto* encoder = findEncoder(identifier); if (!encoder) diff --git a/Source/WebKit/WebProcess/GPU/media/RemoteVideoCodecFactory.cpp b/Source/WebKit/WebProcess/GPU/media/RemoteVideoCodecFactory.cpp index 54888f068df70..f925b714a7395 100644 --- a/Source/WebKit/WebProcess/GPU/media/RemoteVideoCodecFactory.cpp +++ b/Source/WebKit/WebProcess/GPU/media/RemoteVideoCodecFactory.cpp @@ -63,8 +63,8 @@ class RemoteVideoDecoder : public WebCore::VideoDecoder { ~RemoteVideoDecoder(); private: - void decode(EncodedFrame&&, DecodeCallback&&) final; - void flush(Function&&) final; + Ref decode(EncodedFrame&&) final; + Ref flush() final; void reset() final; void close() final; @@ -103,11 +103,11 @@ class RemoteVideoEncoder : public WebCore::VideoEncoder { ~RemoteVideoEncoder(); private: - void encode(RawFrame&&, bool shouldGenerateKeyFrame, EncodeCallback&&) final; - void flush(Function&&) final; + Ref encode(RawFrame&&, bool shouldGenerateKeyFrame) final; + Ref flush() final; void reset() final; void close() final; - bool setRates(uint64_t bitRate, double frameRate, Function&&) final; + Ref setRates(uint64_t bitRate, double frameRate) final; LibWebRTCCodecs::Encoder& m_internalEncoder; Ref m_callbacks; @@ -219,24 +219,18 @@ RemoteVideoDecoder::~RemoteVideoDecoder() WebProcess::singleton().libWebRTCCodecs().releaseDecoder(m_internalDecoder); } -void RemoteVideoDecoder::decode(EncodedFrame&& frame, DecodeCallback&& callback) +Ref RemoteVideoDecoder::decode(EncodedFrame&& frame) { if (frame.duration) m_callbacks->addDuration(frame.timestamp, *frame.duration); - WebProcess::singleton().libWebRTCCodecs().decodeFrame(m_internalDecoder, frame.timestamp, frame.data, m_width, m_height, [callback = WTFMove(callback), callbacks = m_callbacks] (bool result) mutable { - callbacks->postTask([callback = WTFMove(callback), result]() mutable { - callback(result ? String { } : "Decoding task failed"_s); - }); - }); + + auto& codecs = WebProcess::singleton().libWebRTCCodecs(); + return codecs.decodeFrame(m_internalDecoder, frame.timestamp, frame.data, m_width, m_height); } -void RemoteVideoDecoder::flush(Function&& callback) +Ref RemoteVideoDecoder::flush() { - WebProcess::singleton().libWebRTCCodecs().flushDecoder(m_internalDecoder, [callback = WTFMove(callback), callbacks = m_callbacks]() mutable { - callbacks->postTask([callback = WTFMove(callback)]() mutable { - callback(); - }); - }); + return WebProcess::singleton().libWebRTCCodecs().flushDecoder(m_internalDecoder); } void RemoteVideoDecoder::reset() @@ -294,28 +288,21 @@ RemoteVideoEncoder::~RemoteVideoEncoder() WebProcess::singleton().libWebRTCCodecs().releaseEncoder(m_internalEncoder); } -void RemoteVideoEncoder::encode(RawFrame&& rawFrame, bool shouldGenerateKeyFrame, EncodeCallback&& callback) +Ref RemoteVideoEncoder::encode(RawFrame&& rawFrame, bool shouldGenerateKeyFrame) { - WebProcess::singleton().libWebRTCCodecs().encodeFrame(m_internalEncoder, rawFrame.frame.get(), rawFrame.timestamp, rawFrame.duration, shouldGenerateKeyFrame, [callback = WTFMove(callback), callbacks = m_callbacks] (bool result) mutable { - callbacks->postTask([callback = WTFMove(callback), result]() mutable { - callback(result ? String { } : "Encoding task failed"_s); - }); - }); + auto& codecs = WebProcess::singleton().libWebRTCCodecs(); + return codecs.encodeFrame(m_internalEncoder, rawFrame.frame.get(), rawFrame.timestamp, rawFrame.duration, shouldGenerateKeyFrame); } -bool RemoteVideoEncoder::setRates(uint64_t bitRate, double frameRate, Function&& callback) +Ref RemoteVideoEncoder::setRates(uint64_t bitRate, double frameRate) { auto bitRateInKbps = bitRate / 1000; - WebProcess::singleton().libWebRTCCodecs().setEncodeRates(m_internalEncoder, bitRateInKbps, frameRate); - m_callbacks->postTask(WTFMove(callback)); - return true; + return WebProcess::singleton().libWebRTCCodecs().setEncodeRates(m_internalEncoder, bitRateInKbps, frameRate); } -void RemoteVideoEncoder::flush(Function&& callback) +Ref RemoteVideoEncoder::flush() { - WebProcess::singleton().libWebRTCCodecs().flushEncoder(m_internalEncoder, [callback = WTFMove(callback), callbacks = m_callbacks]() mutable { - callbacks->postTask(WTFMove(callback)); - }); + return WebProcess::singleton().libWebRTCCodecs().flushEncoder(m_internalEncoder); } void RemoteVideoEncoder::reset() diff --git a/Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.cpp b/Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.cpp index ae869720899d8..06670c8069eb1 100644 --- a/Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.cpp +++ b/Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.cpp @@ -152,7 +152,7 @@ static int32_t releaseVideoDecoder(webrtc::WebKitVideoDecoder::Value decoder) static int32_t decodeVideoFrame(webrtc::WebKitVideoDecoder::Value decoder, uint32_t timeStamp, const uint8_t* data, size_t size, uint16_t width, uint16_t height) { - return WebProcess::singleton().libWebRTCCodecs().decodeFrame(*static_cast(decoder), timeStamp, { data, size }, width, height, [] (bool) { }); + return WebProcess::singleton().libWebRTCCodecs().decodeWebRTCFrame(*static_cast(decoder), timeStamp, { data, size }, width, height); } static int32_t registerDecodeCompleteCallback(webrtc::WebKitVideoDecoder::Value decoder, void* decodedImageCallback) @@ -398,17 +398,15 @@ int32_t LibWebRTCCodecs::releaseDecoder(Decoder& decoder) } // May be called on any thread. -void LibWebRTCCodecs::flushDecoder(Decoder& decoder, Function&& callback) +Ref LibWebRTCCodecs::flushDecoder(Decoder& decoder) { Locker locker { m_connectionLock }; - if (!decoder.connection || decoder.hasError) { - callback(); - return; - } + if (!decoder.connection || decoder.hasError) + return GenericPromise::createAndResolve(); - decoder.connection->send(Messages::LibWebRTCCodecsProxy::FlushDecoder { decoder.identifier }, 0); - Locker flushLocker { decoder.flushCallbacksLock }; - decoder.flushCallbacks.append(WTFMove(callback)); + return decoder.connection->sendWithPromisedReply(Messages::LibWebRTCCodecsProxy::FlushDecoder { decoder.identifier }, 0)->whenSettled(workQueue(), [] (auto&&) { + return GenericPromise::createAndResolve(); + }); } void LibWebRTCCodecs::setDecoderFormatDescription(Decoder& decoder, std::span data, uint16_t width, uint16_t height) @@ -421,31 +419,51 @@ void LibWebRTCCodecs::setDecoderFormatDescription(Decoder& decoder, std::spansend(Messages::LibWebRTCCodecsProxy::SetDecoderFormatDescription { decoder.identifier, data, width, height }, 0); } -void LibWebRTCCodecs::sendFrameToDecode(Decoder& decoder, int64_t timeStamp, std::span data, uint16_t width, uint16_t height, Function&& callback) +Ref LibWebRTCCodecs::sendFrameToDecode(Decoder& decoder, int64_t timeStamp, std::span data, uint16_t width, uint16_t height) { if (decoder.type == WebCore::VideoCodecType::VP9 && (width || height)) decoder.connection->send(Messages::LibWebRTCCodecsProxy::SetFrameSize { decoder.identifier, width, height }, 0); - decoder.connection->sendWithPromisedReply(Messages::LibWebRTCCodecsProxy::DecodeFrame { decoder.identifier, timeStamp, data }, 0)->whenSettled(workQueue(), [callback = WTFMove(callback)] (auto&& result) mutable { - callback(result ? result.value() : false); + return decoder.connection->sendWithPromisedReply(Messages::LibWebRTCCodecsProxy::DecodeFrame { decoder.identifier, timeStamp, data }, 0)->whenSettled(workQueue(), [] (auto&& result) mutable { + if (!result) + return FramePromise::createAndReject("Decoding task did not complete"_s); + + if (!*result) + return FramePromise::createAndReject("Decoding task failed"_s); + + return FramePromise::createAndResolve(); }); } -int32_t LibWebRTCCodecs::decodeFrame(Decoder& decoder, int64_t timeStamp, std::span data, uint16_t width, uint16_t height, Function&& callback) +int32_t LibWebRTCCodecs::decodeWebRTCFrame(Decoder& decoder, int64_t timeStamp, std::span data, uint16_t width, uint16_t height) +{ + auto promise = decodeFrameInternal(decoder, timeStamp, data, width, height); + return promise ? WEBRTC_VIDEO_CODEC_OK : WEBRTC_VIDEO_CODEC_ERROR; +} + +Ref LibWebRTCCodecs::decodeFrame(Decoder& decoder, int64_t timeStamp, std::span data, uint16_t width, uint16_t height) +{ + auto promise = decodeFrameInternal(decoder, timeStamp, data, width, height); + return promise ? promise.releaseNonNull() : FramePromise::createAndReject("Decoding task did not complete"_s); +} + +RefPtr LibWebRTCCodecs::decodeFrameInternal(Decoder& decoder, int64_t timeStamp, std::span data, uint16_t width, uint16_t height) { Locker locker { m_connectionLock }; if (decoder.hasError) { decoder.hasError = false; - return WEBRTC_VIDEO_CODEC_ERROR; + return nullptr; } if (!decoder.connection) { - decoder.pendingFrames.append({ timeStamp, data, width, height }); - return WEBRTC_VIDEO_CODEC_OK; + FramePromise::AutoRejectProducer producer; + auto promise = producer.promise(); + + decoder.pendingFrames.append({ timeStamp, data, width, height, WTFMove(producer) }); + return promise; } - sendFrameToDecode(decoder, timeStamp, data, width, height, WTFMove(callback)); - return WEBRTC_VIDEO_CODEC_OK; + return sendFrameToDecode(decoder, timeStamp, data, width, height); } void LibWebRTCCodecs::registerDecodeFrameCallback(Decoder& decoder, void* decodedImageCallback) @@ -477,19 +495,6 @@ void LibWebRTCCodecs::failedDecoding(VideoDecoderIdentifier decoderIdentifier) } } -void LibWebRTCCodecs::flushDecoderCompleted(VideoDecoderIdentifier decoderIdentifier) -{ - assertIsCurrent(workQueue()); - - auto* decoder = m_decoders.get(decoderIdentifier); - if (!decoder) - return; - - Locker locker { decoder->flushCallbacksLock }; - if (!decoder->flushCallbacks.isEmpty()) - decoder->flushCallbacks.takeFirst()(); -} - void LibWebRTCCodecs::completedDecoding(VideoDecoderIdentifier decoderIdentifier, int64_t timeStamp, int64_t timeStampNs, RemoteVideoFrameProxy::Properties&& properties) { assertIsCurrent(workQueue()); @@ -677,46 +682,54 @@ void LibWebRTCCodecs::initializeEncoderInternal(Encoder& encoder, uint16_t width encoderConnection(encoder)->send(Messages::LibWebRTCCodecsProxy::InitializeEncoder { encoder.identifier, width, height, startBitRate, maxBitRate, minBitRate, maxFrameRate }, 0); } -template int32_t LibWebRTCCodecs::encodeFrameInternal(Encoder& encoder, const Frame& frame, bool shouldEncodeAsKeyFrame, WebCore::VideoFrame::Rotation rotation, MediaTime mediaTime, int64_t timestamp, std::optional duration, Function&& callback) +template RefPtr LibWebRTCCodecs::encodeFrameInternal(Encoder& encoder, const Frame& frame, bool shouldEncodeAsKeyFrame, WebCore::VideoFrame::Rotation rotation, MediaTime mediaTime, int64_t timestamp, std::optional duration) { Locker locker { m_encodersConnectionLock }; auto* connection = encoderConnection(encoder); if (!connection) - return WEBRTC_VIDEO_CODEC_ERROR; + return nullptr; auto buffer = encoder.sharedVideoFrameWriter.writeBuffer(frame, [&](auto& semaphore) { encoder.connection->send(Messages::LibWebRTCCodecsProxy::SetSharedVideoFrameSemaphore { encoder.identifier, semaphore }, 0); }, [&](SharedMemory::Handle&& handle) { encoder.connection->send(Messages::LibWebRTCCodecsProxy::SetSharedVideoFrameMemory { encoder.identifier, WTFMove(handle) }, 0); }); if (!buffer) - return WEBRTC_VIDEO_CODEC_ERROR; + return nullptr; SharedVideoFrame sharedVideoFrame { mediaTime, false, rotation, WTFMove(*buffer) }; - encoder.connection->sendWithPromisedReply(Messages::LibWebRTCCodecsProxy::EncodeFrame { encoder.identifier, WTFMove(sharedVideoFrame), timestamp, duration, shouldEncodeAsKeyFrame })->whenSettled(workQueue(), [callback = WTFMove(callback)] (auto&& result) mutable { - callback(result ? result.value() : false); + auto promise = encoder.connection->sendWithPromisedReply(Messages::LibWebRTCCodecsProxy::EncodeFrame { encoder.identifier, WTFMove(sharedVideoFrame), timestamp, duration, shouldEncodeAsKeyFrame }); + return promise->whenSettled(workQueue(), [] (auto&& result) mutable { + if (!result) + return FramePromise::createAndReject("Encoding task did not complete"_s); + + if (!*result) + return FramePromise::createAndReject("Encoding task failed"_s); + + return FramePromise::createAndResolve(); }); - return WEBRTC_VIDEO_CODEC_OK; } int32_t LibWebRTCCodecs::encodeFrame(Encoder& encoder, const webrtc::VideoFrame& frame, bool shouldEncodeAsKeyFrame) { - return encodeFrameInternal(encoder, frame, shouldEncodeAsKeyFrame, toVideoRotation(frame.rotation()), MediaTime::createWithDouble(Seconds::fromMicroseconds(frame.timestamp_us()).value()), frame.timestamp(), { }, [](auto) { }); + auto promise = encodeFrameInternal(encoder, frame, shouldEncodeAsKeyFrame, toVideoRotation(frame.rotation()), MediaTime::createWithDouble(Seconds::fromMicroseconds(frame.timestamp_us()).value()), frame.timestamp(), { }); + return promise ? WEBRTC_VIDEO_CODEC_OK : WEBRTC_VIDEO_CODEC_ERROR; } -int32_t LibWebRTCCodecs::encodeFrame(Encoder& encoder, const WebCore::VideoFrame& frame, int64_t timestamp, std::optional duration, bool shouldEncodeAsKeyFrame, Function&& callback) +Ref LibWebRTCCodecs::encodeFrame(Encoder& encoder, const WebCore::VideoFrame& frame, int64_t timestamp, std::optional duration, bool shouldEncodeAsKeyFrame) { - return encodeFrameInternal(encoder, frame, shouldEncodeAsKeyFrame, frame.rotation(), frame.presentationTime(), timestamp, duration, WTFMove(callback)); + auto promise = encodeFrameInternal(encoder, frame, shouldEncodeAsKeyFrame, frame.rotation(), frame.presentationTime(), timestamp, duration); + return promise ? promise.releaseNonNull() : FramePromise::createAndReject("Encoding task did not complete"_s); } -void LibWebRTCCodecs::flushEncoder(Encoder& encoder, Function&& callback) +Ref LibWebRTCCodecs::flushEncoder(Encoder& encoder) { Locker locker { m_encodersConnectionLock }; RefPtr connection = encoderConnection(encoder); - if (!connection) { - callback(); - return; - } + if (!connection) + return GenericPromise::createAndResolve(); - connection->sendWithPromisedReply(Messages::LibWebRTCCodecsProxy::FlushEncoder { encoder.identifier })->whenSettled(workQueue(), WTFMove(callback)); + return connection->sendWithPromisedReply(Messages::LibWebRTCCodecsProxy::FlushEncoder { encoder.identifier })->whenSettled(workQueue(), [] (auto&&) { + return GenericPromise::createAndResolve(); + }); } void LibWebRTCCodecs::registerEncodeFrameCallback(Encoder& encoder, void* encodedImageCallback) @@ -747,13 +760,16 @@ void LibWebRTCCodecs::registerEncoderDescriptionCallback(Encoder& encoder, Descr } #endif -void LibWebRTCCodecs::setEncodeRates(Encoder& encoder, uint32_t bitRateInKbps, uint32_t frameRate) +Ref LibWebRTCCodecs::setEncodeRates(Encoder& encoder, uint32_t bitRateInKbps, uint32_t frameRate) { Locker locker { m_encodersConnectionLock }; auto* connection = encoderConnection(encoder); if (!connection) { - ensureGPUProcessConnectionAndDispatchToThread([this, hasSentInitialEncodeRates = &encoder.hasSentInitialEncodeRates, encoderIdentifier = encoder.identifier, bitRateInKbps, frameRate] { + GenericPromise::AutoRejectProducer producer; + auto promise = producer.promise(); + + ensureGPUProcessConnectionAndDispatchToThread([this, hasSentInitialEncodeRates = &encoder.hasSentInitialEncodeRates, encoderIdentifier = encoder.identifier, bitRateInKbps, frameRate, producer = WTFMove(producer)] () mutable { assertIsCurrent(workQueue()); ASSERT(m_encoders.get(encoderIdentifier)); @@ -762,12 +778,17 @@ void LibWebRTCCodecs::setEncodeRates(Encoder& encoder, uint32_t bitRateInKbps, u return; Locker locker { m_connectionLock }; - m_connection->send(Messages::LibWebRTCCodecsProxy::SetEncodeRates { encoderIdentifier, bitRateInKbps, frameRate }, 0); + m_connection->sendWithPromisedReply(Messages::LibWebRTCCodecsProxy::SetEncodeRates { encoderIdentifier, bitRateInKbps, frameRate })->whenSettled(workQueue(), [producer = WTFMove(producer)] (auto&&) { + producer.resolve(); + }); }); - return; + + return promise; } encoder.hasSentInitialEncodeRates = true; - connection->send(Messages::LibWebRTCCodecsProxy::SetEncodeRates { encoder.identifier, bitRateInKbps, frameRate }, 0); + return connection->sendWithPromisedReply(Messages::LibWebRTCCodecsProxy::SetEncodeRates { encoder.identifier, bitRateInKbps, frameRate })->whenSettled(workQueue(), [] (auto&&) { + return GenericPromise::createAndResolve(); + }); } void LibWebRTCCodecs::completedEncoding(VideoEncoderIdentifier identifier, std::span data, const webrtc::WebKitEncodedFrameInfo& info) @@ -850,11 +871,6 @@ void LibWebRTCCodecs::gpuProcessConnectionDidClose(GPUProcessConnection&) { Locker locker { m_connectionLock }; for (auto& decoder : m_decoders.values()) { - { - Locker locker { decoder->flushCallbacksLock }; - while (!decoder->flushCallbacks.isEmpty()) - decoder->flushCallbacks.takeFirst()(); - } createRemoteDecoder(*decoder, *connection, m_useRemoteFrames, m_enableAdditionalLogging, [](auto) { }); setDecoderConnection(*decoder, connection.get()); } @@ -904,7 +920,7 @@ void LibWebRTCCodecs::setDecoderConnection(Decoder& decoder, RefPtrchainTo(WTFMove(frame.producer)); } inline RefPtr LibWebRTCCodecs::protectedVideoFrameObjectHeapProxy() const diff --git a/Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.h b/Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.h index 13347eee908a7..c038915006eab 100644 --- a/Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.h +++ b/Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.h @@ -81,6 +81,7 @@ class LibWebRTCCodecs : public IPC::WorkQueueMessageReceiver, public GPUProcessC std::optional videoCodecTypeFromWebCodec(const String&); std::optional videoEncoderTypeFromWebCodec(const String&); + using FramePromise = NativePromise; using DecoderCallback = Function&&, int64_t timestamp)>; struct Decoder { WTF_MAKE_TZONE_ALLOCATED(Decoder); @@ -90,6 +91,7 @@ class LibWebRTCCodecs : public IPC::WorkQueueMessageReceiver, public GPUProcessC Vector data; uint16_t width { 0 }; uint16_t height { 0 }; + FramePromise::AutoRejectProducer producer; }; VideoDecoderIdentifier identifier; @@ -101,17 +103,16 @@ class LibWebRTCCodecs : public IPC::WorkQueueMessageReceiver, public GPUProcessC bool hasError { false }; RefPtr connection; Vector pendingFrames; - Deque> flushCallbacks WTF_GUARDED_BY_LOCK(flushCallbacksLock); - Lock flushCallbacksLock; }; Decoder* createDecoder(WebCore::VideoCodecType); void createDecoderAndWaitUntilReady(WebCore::VideoCodecType, const String& codec, Function&&); int32_t releaseDecoder(Decoder&); - void flushDecoder(Decoder&, Function&&); + Ref flushDecoder(Decoder&); void setDecoderFormatDescription(Decoder&, std::span, uint16_t width, uint16_t height); - int32_t decodeFrame(Decoder&, int64_t timeStamp, std::span, uint16_t width, uint16_t height, Function&&); + int32_t decodeWebRTCFrame(Decoder&, int64_t timeStamp, std::span, uint16_t width, uint16_t height); + Ref decodeFrame(Decoder&, int64_t timeStamp, std::span, uint16_t width, uint16_t height); void registerDecodeFrameCallback(Decoder&, void* decodedImageCallback); void registerDecodedVideoFrameCallback(Decoder&, DecoderCallback&&); @@ -155,15 +156,15 @@ class LibWebRTCCodecs : public IPC::WorkQueueMessageReceiver, public GPUProcessC #endif int32_t releaseEncoder(Encoder&); int32_t initializeEncoder(Encoder&, uint16_t width, uint16_t height, unsigned startBitrate, unsigned maxBitrate, unsigned minBitrate, uint32_t maxFramerate); - int32_t encodeFrame(Encoder&, const WebCore::VideoFrame&, int64_t timestamp, std::optional duration, bool shouldEncodeAsKeyFrame, Function&&); + Ref encodeFrame(Encoder&, const WebCore::VideoFrame&, int64_t timestamp, std::optional duration, bool shouldEncodeAsKeyFrame); int32_t encodeFrame(Encoder&, const webrtc::VideoFrame&, bool shouldEncodeAsKeyFrame); - void flushEncoder(Encoder&, Function&&); + Ref flushEncoder(Encoder&); void registerEncodeFrameCallback(Encoder&, void* encodedImageCallback); void registerEncodedVideoFrameCallback(Encoder&, EncoderCallback&&); #if ENABLE(WEB_CODECS) void registerEncoderDescriptionCallback(Encoder&, DescriptionCallback&&); #endif - void setEncodeRates(Encoder&, uint32_t bitRateInKbps, uint32_t frameRate); + Ref setEncodeRates(Encoder&, uint32_t bitRateInKbps, uint32_t frameRate); CVPixelBufferPoolRef pixelBufferPool(size_t width, size_t height, OSType); @@ -213,9 +214,11 @@ class LibWebRTCCodecs : public IPC::WorkQueueMessageReceiver, public GPUProcessC Decoder* createDecoderInternal(WebCore::VideoCodecType, const String& codec, Function&&); Encoder* createEncoderInternal(WebCore::VideoCodecType, const String& codec, const std::map&, bool isRealtime, bool useAnnexB, WebCore::VideoEncoderScalabilityMode, Function&&); - template int32_t encodeFrameInternal(Encoder&, const Frame&, bool shouldEncodeAsKeyFrame, WebCore::VideoFrameRotation, MediaTime, int64_t timestamp, std::optional duration, Function&&); + template RefPtr encodeFrameInternal(Encoder&, const Frame&, bool shouldEncodeAsKeyFrame, WebCore::VideoFrameRotation, MediaTime, int64_t timestamp, std::optional duration); + void initializeEncoderInternal(Encoder&, uint16_t width, uint16_t height, unsigned startBitrate, unsigned maxBitrate, unsigned minBitrate, uint32_t maxFramerate); - void sendFrameToDecode(Decoder&, int64_t timeStamp, std::span, uint16_t width, uint16_t height, Function&&); + RefPtr decodeFrameInternal(Decoder&, int64_t timeStamp, std::span, uint16_t width, uint16_t height); + Ref sendFrameToDecode(Decoder&, int64_t timeStamp, std::span, uint16_t width, uint16_t height); RefPtr protectedConnection() const WTF_REQUIRES_LOCK(m_connectionLock) { return m_connection; } RefPtr protectedVideoFrameObjectHeapProxy() const WTF_REQUIRES_LOCK(m_connectionLock); diff --git a/Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.messages.in b/Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.messages.in index 2dc4f153b2316..548f34dc207a1 100644 --- a/Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.messages.in +++ b/Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.messages.in @@ -24,7 +24,6 @@ messages -> LibWebRTCCodecs NotRefCounted { FailedDecoding(WebKit::VideoDecoderIdentifier identifier) - FlushDecoderCompleted(WebKit::VideoDecoderIdentifier identifier) CompletedDecoding(WebKit::VideoDecoderIdentifier identifier, int64_t timeStamp, int64_t timeStampNs, struct WebKit::RemoteVideoFrameProxyProperties frame) CompletedDecodingCV(WebKit::VideoDecoderIdentifier identifier, int64_t timeStamp, int64_t timeStampNs, RetainPtr pixelBuffer) CompletedEncoding(WebKit::VideoEncoderIdentifier identifier, std::span data, struct webrtc::WebKitEncodedFrameInfo info);