diff --git a/api/media_stream_interface.cc b/api/media_stream_interface.cc index e07907917b..5362522262 100644 --- a/api/media_stream_interface.cc +++ b/api/media_stream_interface.cc @@ -18,6 +18,10 @@ const char* const MediaStreamTrackInterface::kVideoKind = const char* const MediaStreamTrackInterface::kAudioKind = cricket::kMediaTypeAudio; +bool VideoTrackInterface::should_receive() const { + return true; +} + VideoTrackInterface::ContentHint VideoTrackInterface::content_hint() const { return ContentHint::kNone; } diff --git a/api/media_stream_interface.h b/api/media_stream_interface.h index 9d336739e4..6d9b4aa6ca 100644 --- a/api/media_stream_interface.h +++ b/api/media_stream_interface.h @@ -188,6 +188,8 @@ class RTC_EXPORT VideoTrackInterface virtual VideoTrackSourceInterface* GetSource() const = 0; + virtual void set_should_receive(bool should_receive) {} + virtual bool should_receive() const; virtual ContentHint content_hint() const; virtual void set_content_hint(ContentHint hint) {} diff --git a/media/base/media_channel.h b/media/base/media_channel.h index 378042f1b2..f66ee7769a 100644 --- a/media/base/media_channel.h +++ b/media/base/media_channel.h @@ -997,6 +997,8 @@ class VideoMediaReceiveChannelInterface : public MediaReceiveChannelInterface { bool nack_enabled, webrtc::RtcpMode rtcp_mode, absl::optional rtx_time) = 0; + virtual void StartReceive(uint32_t ssrc) {} + virtual void StopReceive(uint32_t ssrc) {} }; } // namespace cricket diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index 83581bf9fd..8a63ffcbb3 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc @@ -928,6 +928,24 @@ void WebRtcVideoChannel::RequestEncoderSwitch( } } +void WebRtcVideoChannel::StartReceive(uint32_t ssrc) { + RTC_DCHECK_RUN_ON(&thread_checker_); + WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc); + if(!stream) { + return; + } + stream->StartStream(); +} + +void WebRtcVideoChannel::StopReceive(uint32_t ssrc) { + RTC_DCHECK_RUN_ON(&thread_checker_); + WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc); + if(!stream) { + return; + } + stream->StopStream(); +} + bool WebRtcVideoChannel::ApplyChangedParams( const ChangedSendParameters& changed_params) { RTC_DCHECK_RUN_ON(&thread_checker_); @@ -3182,6 +3200,17 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetRecvParameters( } } +void WebRtcVideoChannel::WebRtcVideoReceiveStream::StartStream(){ + if (stream_) { + stream_->Start(); + } +} +void WebRtcVideoChannel::WebRtcVideoReceiveStream::StopStream(){ + if (stream_) { + stream_->Stop(); + } +} + void WebRtcVideoChannel::WebRtcVideoReceiveStream::RecreateReceiveStream() { RTC_DCHECK(stream_); absl::optional base_minimum_playout_delay_ms; diff --git a/media/engine/webrtc_video_engine.h b/media/engine/webrtc_video_engine.h index 2cdd05ffd6..a3e433f002 100644 --- a/media/engine/webrtc_video_engine.h +++ b/media/engine/webrtc_video_engine.h @@ -270,6 +270,8 @@ class WebRtcVideoChannel : public VideoMediaChannel, webrtc::RtcpMode rtcp_mode, absl::optional rtx_time) override; + void StartReceive(uint32_t ssrc) override; + void StopReceive(uint32_t ssrc) override; private: class WebRtcVideoReceiveStream; @@ -537,6 +539,9 @@ class WebRtcVideoChannel : public VideoMediaChannel, void SetDepacketizerToDecoderFrameTransformer( rtc::scoped_refptr frame_transformer); + + void StartStream(); + void StopStream(); void SetLocalSsrc(uint32_t local_ssrc); void UpdateRtxSsrc(uint32_t ssrc); diff --git a/pc/media_stream_track_proxy.h b/pc/media_stream_track_proxy.h index 2af3aedb22..fab23d17ec 100644 --- a/pc/media_stream_track_proxy.h +++ b/pc/media_stream_track_proxy.h @@ -55,6 +55,8 @@ PROXY_SECONDARY_METHOD2(void, PROXY_SECONDARY_METHOD1(void, RemoveSink, rtc::VideoSinkInterface*) PROXY_SECONDARY_METHOD0(void, RequestRefreshFrame) BYPASS_PROXY_CONSTMETHOD0(VideoTrackSourceInterface*, GetSource) +PROXY_CONSTMETHOD0(bool, should_receive) +PROXY_METHOD1(void, set_should_receive, bool) PROXY_METHOD1(void, RegisterObserver, ObserverInterface*) PROXY_METHOD1(void, UnregisterObserver, ObserverInterface*) diff --git a/pc/video_rtp_receiver.cc b/pc/video_rtp_receiver.cc index 8a2e65c162..63a2d72879 100644 --- a/pc/video_rtp_receiver.cc +++ b/pc/video_rtp_receiver.cc @@ -41,15 +41,20 @@ VideoRtpReceiver::VideoRtpReceiver( rtc::Thread::Current(), worker_thread, VideoTrack::Create(receiver_id, source_, worker_thread))), - attachment_id_(GenerateUniqueId()) { + cached_track_should_receive_(track_->should_receive()), + attachment_id_(GenerateUniqueId()), + worker_thread_safety_(PendingTaskSafetyFlag::CreateDetachedInactive()) { RTC_DCHECK(worker_thread_); SetStreams(streams); + track_->RegisterObserver(this); RTC_DCHECK_EQ(source_->state(), MediaSourceInterface::kInitializing); } VideoRtpReceiver::~VideoRtpReceiver() { RTC_DCHECK_RUN_ON(&signaling_thread_checker_); RTC_DCHECK(!media_channel_); + + track_->UnregisterObserver(this); } std::vector VideoRtpReceiver::stream_ids() const { @@ -114,6 +119,39 @@ void VideoRtpReceiver::Stop() { track_->internal()->set_ended(); } +void VideoRtpReceiver::OnChanged() { + RTC_DCHECK_RUN_ON(&signaling_thread_checker_); + if (cached_track_should_receive_ != track_->should_receive()) { + cached_track_should_receive_ = track_->should_receive(); + worker_thread_->PostTask( + [this, receive = cached_track_should_receive_]() { + RTC_DCHECK_RUN_ON(worker_thread_); + if(receive) { + StartMediaChannel(); + } else { + StopMediaChannel(); + } + }); + } +} + +void VideoRtpReceiver::StartMediaChannel() { + RTC_DCHECK_RUN_ON(worker_thread_); + if (!media_channel_) { + return; + } + media_channel_->StartReceive(signaled_ssrc_.value_or(0)); + OnGenerateKeyFrame(); +} + +void VideoRtpReceiver::StopMediaChannel() { + RTC_DCHECK_RUN_ON(worker_thread_); + if (!media_channel_) { + return; + } + media_channel_->StopReceive(signaled_ssrc_.value_or(0)); +} + void VideoRtpReceiver::RestartMediaChannel(absl::optional ssrc) { RTC_DCHECK_RUN_ON(&signaling_thread_checker_); MediaSourceInterface::SourceState state = source_->state(); @@ -209,6 +247,7 @@ void VideoRtpReceiver::set_transport( void VideoRtpReceiver::SetStreams( const std::vector>& streams) { RTC_DCHECK_RUN_ON(&signaling_thread_checker_); + // Remove remote track from any streams that are going away. for (const auto& existing_stream : streams_) { bool removed = true; diff --git a/pc/video_rtp_receiver.h b/pc/video_rtp_receiver.h index ef88016052..491efe2931 100644 --- a/pc/video_rtp_receiver.h +++ b/pc/video_rtp_receiver.h @@ -42,7 +42,8 @@ namespace webrtc { -class VideoRtpReceiver : public RtpReceiverInternal { +class VideoRtpReceiver : public RtpReceiverInternal, + public ObserverInterface { public: // An SSRC of 0 will create a receiver that will match the first SSRC it // sees. Must be called on signaling thread. @@ -60,6 +61,9 @@ class VideoRtpReceiver : public RtpReceiverInternal { rtc::scoped_refptr video_track() const { return track_; } + // ObserverInterface implementation + void OnChanged() override; + // RtpReceiverInterface implementation rtc::scoped_refptr track() const override { return track_; @@ -115,6 +119,8 @@ class VideoRtpReceiver : public RtpReceiverInternal { cricket::MediaReceiveChannelInterface* media_channel); private: + void StartMediaChannel(); + void StopMediaChannel(); void RestartMediaChannel(absl::optional ssrc) RTC_RUN_ON(&signaling_thread_checker_); void RestartMediaChannel_w(absl::optional ssrc, @@ -162,6 +168,8 @@ class VideoRtpReceiver : public RtpReceiverInternal { RTC_GUARDED_BY(&signaling_thread_checker_) = nullptr; bool received_first_packet_ RTC_GUARDED_BY(&signaling_thread_checker_) = false; + + bool cached_track_should_receive_ RTC_GUARDED_BY(&signaling_thread_checker_); const int attachment_id_; rtc::scoped_refptr frame_decryptor_ RTC_GUARDED_BY(worker_thread_); @@ -177,6 +185,7 @@ class VideoRtpReceiver : public RtpReceiverInternal { // or switched. bool saved_generate_keyframe_ RTC_GUARDED_BY(worker_thread_) = false; bool saved_encoded_sink_enabled_ RTC_GUARDED_BY(worker_thread_) = false; + const rtc::scoped_refptr worker_thread_safety_; }; } // namespace webrtc diff --git a/pc/video_track.cc b/pc/video_track.cc index 0bf8687af3..8922cdaf1f 100644 --- a/pc/video_track.cc +++ b/pc/video_track.cc @@ -76,6 +76,19 @@ VideoTrackSourceInterface* VideoTrack::GetSourceInternal() const { return video_source_->internal(); } +void VideoTrack::set_should_receive(bool receive) { + RTC_DCHECK_RUN_ON(&signaling_thread_); + if (should_receive_ == receive) + return; + should_receive_ = receive; + Notifier::FireOnChanged(); +} + +bool VideoTrack::should_receive() const { + RTC_DCHECK_RUN_ON(&signaling_thread_); + return should_receive_; +} + VideoTrackInterface::ContentHint VideoTrack::content_hint() const { RTC_DCHECK_RUN_ON(&signaling_thread_); return content_hint_; diff --git a/pc/video_track.h b/pc/video_track.h index 13a51c454b..b56c64ef20 100644 --- a/pc/video_track.h +++ b/pc/video_track.h @@ -48,6 +48,9 @@ class VideoTrack : public MediaStreamTrack, void RequestRefreshFrame() override; VideoTrackSourceInterface* GetSource() const override; + void set_should_receive(bool should_receive) override; + bool should_receive() const override; + ContentHint content_hint() const override; void set_content_hint(ContentHint hint) override; bool set_enabled(bool enable) override; @@ -81,6 +84,7 @@ class VideoTrack : public MediaStreamTrack, // be queried without blocking on the worker thread by callers that don't // use an api proxy to call the `enabled()` method. bool enabled_w_ RTC_GUARDED_BY(worker_thread_) = true; + bool should_receive_ RTC_GUARDED_BY(signaling_thread_) = true; }; } // namespace webrtc diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn index b88d228839..73cfe51730 100644 --- a/sdk/android/BUILD.gn +++ b/sdk/android/BUILD.gn @@ -287,6 +287,7 @@ if (is_android) { "api/org/webrtc/RTCStatsCollectorCallback.java", "api/org/webrtc/RTCStatsReport.java", "api/org/webrtc/RtcCertificatePem.java", + "api/org/webrtc/RtpCapabilities.java", "api/org/webrtc/RtpParameters.java", "api/org/webrtc/RtpReceiver.java", "api/org/webrtc/RtpSender.java", @@ -359,6 +360,7 @@ if (is_android) { sources = [ "api/org/webrtc/DefaultVideoDecoderFactory.java", "api/org/webrtc/DefaultVideoEncoderFactory.java", + "api/org/webrtc/WrappedVideoDecoderFactory.java", ] deps = [ @@ -736,6 +738,8 @@ if (current_os == "linux" || is_android) { "src/jni/pc/rtc_certificate.h", "src/jni/pc/rtc_stats_collector_callback_wrapper.cc", "src/jni/pc/rtc_stats_collector_callback_wrapper.h", + "src/jni/pc/rtp_capabilities.cc", + "src/jni/pc/rtp_capabilities.h", "src/jni/pc/rtp_parameters.cc", "src/jni/pc/rtp_parameters.h", "src/jni/pc/rtp_receiver.cc", @@ -1409,6 +1413,7 @@ if (current_os == "linux" || is_android) { "api/org/webrtc/RTCStatsCollectorCallback.java", "api/org/webrtc/RTCStatsReport.java", "api/org/webrtc/RtcCertificatePem.java", + "api/org/webrtc/RtpCapabilities.java", "api/org/webrtc/RtpParameters.java", "api/org/webrtc/RtpReceiver.java", "api/org/webrtc/RtpSender.java", diff --git a/sdk/android/api/org/webrtc/PeerConnectionFactory.java b/sdk/android/api/org/webrtc/PeerConnectionFactory.java index ca67b3afc1..e093056b55 100644 --- a/sdk/android/api/org/webrtc/PeerConnectionFactory.java +++ b/sdk/android/api/org/webrtc/PeerConnectionFactory.java @@ -18,6 +18,7 @@ import org.webrtc.PeerConnection; import org.webrtc.audio.AudioDeviceModule; import org.webrtc.audio.JavaAudioDeviceModule; +import org.webrtc.RtpCapabilities; /** * Java wrapper for a C++ PeerConnectionFactoryInterface. Main entry point to @@ -471,6 +472,16 @@ public AudioTrack createAudioTrack(String id, AudioSource source) { return new AudioTrack(nativeCreateAudioTrack(nativeFactory, id, source.getNativeAudioSource())); } + public RtpCapabilities getRtpReceiverCapabilities(MediaStreamTrack.MediaType mediaType) { + checkPeerConnectionFactoryExists(); + return nativeGetRtpReceiverCapabilities(nativeFactory, mediaType); + } + + public RtpCapabilities getRtpSenderCapabilities(MediaStreamTrack.MediaType mediaType) { + checkPeerConnectionFactoryExists(); + return nativeGetRtpSenderCapabilities(nativeFactory, mediaType); + } + // Starts recording an AEC dump. Ownership of the file is transfered to the // native code. If an AEC dump is already in progress, it will be stopped and // a new one will start using the provided file. @@ -615,4 +626,6 @@ private static native boolean nativeStartAecDump( private static native void nativeInjectLoggable(JNILogging jniLogging, int severity); private static native void nativeDeleteLoggable(); private static native void nativePrintStackTrace(int tid); + private static native RtpCapabilities nativeGetRtpSenderCapabilities(long factory, MediaStreamTrack.MediaType mediaType); + private static native RtpCapabilities nativeGetRtpReceiverCapabilities(long factory, MediaStreamTrack.MediaType mediaType); } diff --git a/sdk/android/api/org/webrtc/RtpCapabilities.java b/sdk/android/api/org/webrtc/RtpCapabilities.java new file mode 100644 index 0000000000..612acb8cf6 --- /dev/null +++ b/sdk/android/api/org/webrtc/RtpCapabilities.java @@ -0,0 +1,131 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.webrtc; + +import androidx.annotation.Nullable; +import java.util.List; +import java.util.Map; +import org.webrtc.MediaStreamTrack; + +public class RtpCapabilities { + public static class CodecCapability { + public int preferredPayloadType; + // Name used to identify the codec. Equivalent to MIME subtype. + public String name; + // The media type of this codec. Equivalent to MIME top-level type. + public MediaStreamTrack.MediaType kind; + // Clock rate in Hertz. + public Integer clockRate; + // The number of audio channels used. Set to null for video codecs. + public Integer numChannels; + // The "format specific parameters" field from the "a=fmtp" line in the SDP + public Map parameters; + // The MIME type of the codec. This is a convenience field. + public String mimeType; + + public CodecCapability() {} + + @CalledByNative("CodecCapability") + CodecCapability(int preferredPayloadType, String name, MediaStreamTrack.MediaType kind, + Integer clockRate, Integer numChannels, String mimeType, Map parameters) { + this.preferredPayloadType = preferredPayloadType; + this.name = name; + this.kind = kind; + this.clockRate = clockRate; + this.numChannels = numChannels; + this.parameters = parameters; + this.mimeType = mimeType; + } + + @CalledByNative("CodecCapability") + int getPreferredPayloadType() { + return preferredPayloadType; + } + + @CalledByNative("CodecCapability") + String getName() { + return name; + } + + @CalledByNative("CodecCapability") + MediaStreamTrack.MediaType getKind() { + return kind; + } + + @CalledByNative("CodecCapability") + Integer getClockRate() { + return clockRate; + } + + @CalledByNative("CodecCapability") + Integer getNumChannels() { + return numChannels; + } + + @CalledByNative("CodecCapability") + Map getParameters() { + return parameters; + } + } + + public static class HeaderExtensionCapability { + private final String uri; + private final int preferredId; + private final boolean preferredEncrypted; + + @CalledByNative("HeaderExtensionCapability") + HeaderExtensionCapability(String uri, int preferredId, boolean preferredEncrypted) { + this.uri = uri; + this.preferredId = preferredId; + this.preferredEncrypted = preferredEncrypted; + } + + @CalledByNative("HeaderExtensionCapability") + public String getUri() { + return uri; + } + + @CalledByNative("HeaderExtensionCapability") + public int getPreferredId() { + return preferredId; + } + + @CalledByNative("HeaderExtensionCapability") + public boolean getPreferredEncrypted() { + return preferredEncrypted; + } + } + + public List codecs; + public List headerExtensions; + + @CalledByNative + RtpCapabilities(List codecs, List headerExtensions) { + this.headerExtensions = headerExtensions; + this.codecs = codecs; + } + + @CalledByNative + public List getHeaderExtensions() { + return headerExtensions; + } + + @CalledByNative + List getCodecs() { + return codecs; + } +} \ No newline at end of file diff --git a/sdk/android/api/org/webrtc/RtpTransceiver.java b/sdk/android/api/org/webrtc/RtpTransceiver.java index 1102bd7eb1..b60ac6450a 100644 --- a/sdk/android/api/org/webrtc/RtpTransceiver.java +++ b/sdk/android/api/org/webrtc/RtpTransceiver.java @@ -215,6 +215,11 @@ public void stop() { nativeStopInternal(nativeRtpTransceiver); } + public void setCodecPreferences(List codecs) { + checkRtpTransceiverExists(); + nativeSetCodecPreferences(nativeRtpTransceiver, codecs); + } + /** * The StopInternal method stops the RtpTransceiver, like Stop, but goes * immediately to Stopped state. @@ -263,4 +268,5 @@ private void checkRtpTransceiverExists() { private static native void nativeStopStandard(long rtpTransceiver); private static native boolean nativeSetDirection( long rtpTransceiver, RtpTransceiverDirection rtpTransceiverDirection); + private static native void nativeSetCodecPreferences(long rtpTransceiver, List codecs); } diff --git a/sdk/android/api/org/webrtc/VideoTrack.java b/sdk/android/api/org/webrtc/VideoTrack.java index 5593d424f3..bd5022f39a 100644 --- a/sdk/android/api/org/webrtc/VideoTrack.java +++ b/sdk/android/api/org/webrtc/VideoTrack.java @@ -54,6 +54,24 @@ public void removeSink(VideoSink sink) { } } + /** + * For a remote video track, starts/stops receiving the video stream. + * + * If this is a local video track, this is a no-op. + */ + public void setShouldReceive(boolean shouldReceive){ + nativeSetShouldReceive(getNativeMediaStreamTrack(), shouldReceive); + } + + /** + * The current receive status for a remote video track. + * + * This has no meaning for a local video track. + */ + public boolean shouldReceive(){ + return nativeGetShouldReceive(getNativeMediaStreamTrack()); + } + @Override public void dispose() { for (long nativeSink : sinks.values()) { @@ -73,4 +91,6 @@ long getNativeVideoTrack() { private static native void nativeRemoveSink(long track, long nativeSink); private static native long nativeWrapSink(VideoSink sink); private static native void nativeFreeSink(long sink); + private static native void nativeSetShouldReceive(long track, boolean shouldReceive); + private static native boolean nativeGetShouldReceive(long track); } diff --git a/sdk/android/api/org/webrtc/WrappedVideoDecoderFactory.java b/sdk/android/api/org/webrtc/WrappedVideoDecoderFactory.java new file mode 100644 index 0000000000..a7acd37289 --- /dev/null +++ b/sdk/android/api/org/webrtc/WrappedVideoDecoderFactory.java @@ -0,0 +1,75 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.webrtc; + +import android.media.MediaCodecInfo; +import androidx.annotation.Nullable; + +import java.util.Arrays; +import java.util.LinkedHashSet; + +public class WrappedVideoDecoderFactory implements VideoDecoderFactory { + public WrappedVideoDecoderFactory(@Nullable EglBase.Context eglContext) { + this.hardwareVideoDecoderFactory = new HardwareVideoDecoderFactory(eglContext); + this.platformSoftwareVideoDecoderFactory = new PlatformSoftwareVideoDecoderFactory(eglContext); + } + + private final VideoDecoderFactory hardwareVideoDecoderFactory; + private final VideoDecoderFactory hardwareVideoDecoderFactoryWithoutEglContext = new HardwareVideoDecoderFactory(null) ; + private final VideoDecoderFactory softwareVideoDecoderFactory = new SoftwareVideoDecoderFactory(); + @Nullable + private final VideoDecoderFactory platformSoftwareVideoDecoderFactory; + + @Override + public VideoDecoder createDecoder(VideoCodecInfo codecType) { + VideoDecoder softwareDecoder = this.softwareVideoDecoderFactory.createDecoder(codecType); + VideoDecoder hardwareDecoder = this.hardwareVideoDecoderFactory.createDecoder(codecType); + if (softwareDecoder == null && this.platformSoftwareVideoDecoderFactory != null) { + softwareDecoder = this.platformSoftwareVideoDecoderFactory.createDecoder(codecType); + } + + if(hardwareDecoder != null && disableSurfaceTextureFrame(hardwareDecoder.getImplementationName())) { + hardwareDecoder.release(); + hardwareDecoder = this.hardwareVideoDecoderFactoryWithoutEglContext.createDecoder(codecType); + } + + if (hardwareDecoder != null && softwareDecoder != null) { + return new VideoDecoderFallback(softwareDecoder, hardwareDecoder); + } else { + return hardwareDecoder != null ? hardwareDecoder : softwareDecoder; + } + } + + private boolean disableSurfaceTextureFrame(String name) { + if (name.startsWith("OMX.qcom.") || name.startsWith("OMX.hisi.")) { + return true; + } + return false; + } + + @Override + public VideoCodecInfo[] getSupportedCodecs() { + LinkedHashSet supportedCodecInfos = new LinkedHashSet(); + supportedCodecInfos.addAll(Arrays.asList(this.softwareVideoDecoderFactory.getSupportedCodecs())); + supportedCodecInfos.addAll(Arrays.asList(this.hardwareVideoDecoderFactory.getSupportedCodecs())); + if (this.platformSoftwareVideoDecoderFactory != null) { + supportedCodecInfos.addAll(Arrays.asList(this.platformSoftwareVideoDecoderFactory.getSupportedCodecs())); + } + + return (VideoCodecInfo[])supportedCodecInfos.toArray(new VideoCodecInfo[supportedCodecInfos.size()]); + } +} diff --git a/sdk/android/src/jni/pc/peer_connection_factory.cc b/sdk/android/src/jni/pc/peer_connection_factory.cc index 4c682089db..f4a1b16e89 100644 --- a/sdk/android/src/jni/pc/peer_connection_factory.cc +++ b/sdk/android/src/jni/pc/peer_connection_factory.cc @@ -39,7 +39,9 @@ #include "sdk/android/src/jni/logging/log_sink.h" #include "sdk/android/src/jni/pc/android_network_monitor.h" #include "sdk/android/src/jni/pc/audio.h" +#include "sdk/android/src/jni/pc/rtp_capabilities.h" #include "sdk/android/src/jni/pc/ice_candidate.h" +#include "sdk/android/src/jni/pc/media_stream_track.h" #include "sdk/android/src/jni/pc/owned_factory_and_threads.h" #include "sdk/android/src/jni/pc/peer_connection.h" #include "sdk/android/src/jni/pc/ssl_certificate_verifier_wrapper.h" @@ -393,6 +395,22 @@ jlong JNI_PeerConnectionFactory_CreateAudioTrack( return jlongFromPointer(track.release()); } +ScopedJavaLocalRef JNI_PeerConnectionFactory_GetRtpSenderCapabilities( + JNIEnv* jni, + jlong native_factory, + const JavaParamRef& media_type) { + auto factory = PeerConnectionFactoryFromJava(native_factory); + return NativeToJavaRtpCapabilities(jni, factory->GetRtpSenderCapabilities(JavaToNativeMediaType(jni, media_type))); +} + +ScopedJavaLocalRef JNI_PeerConnectionFactory_GetRtpReceiverCapabilities( + JNIEnv* jni, + jlong native_factory, + const JavaParamRef& media_type) { + auto factory = PeerConnectionFactoryFromJava(native_factory); + return NativeToJavaRtpCapabilities(jni, factory->GetRtpReceiverCapabilities(JavaToNativeMediaType(jni, media_type))); +} + static jboolean JNI_PeerConnectionFactory_StartAecDump( JNIEnv* jni, jlong native_factory, diff --git a/sdk/android/src/jni/pc/rtp_capabilities.cc b/sdk/android/src/jni/pc/rtp_capabilities.cc new file mode 100644 index 0000000000..fb71491ee1 --- /dev/null +++ b/sdk/android/src/jni/pc/rtp_capabilities.cc @@ -0,0 +1,114 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "sdk/android/src/jni/pc/rtp_capabilities.h" + +#include "sdk/android/generated_peerconnection_jni/RtpCapabilities_jni.h" +#include "sdk/android/native_api/jni/java_types.h" +#include "sdk/android/src/jni/jni_helpers.h" +#include "sdk/android/src/jni/pc/media_stream_track.h" + +namespace webrtc { +namespace jni { + +namespace { + +ScopedJavaLocalRef NativeToJavaRtpCodecParameter( + JNIEnv* env, + const RtpCodecCapability& codec) { + return Java_CodecCapability_Constructor(env, codec.preferred_payload_type.value(), + NativeToJavaString(env, codec.name), + NativeToJavaMediaType(env, codec.kind), + NativeToJavaInteger(env, codec.clock_rate), + NativeToJavaInteger(env, codec.num_channels), + NativeToJavaString(env, codec.mime_type()), + NativeToJavaStringMap(env, codec.parameters)); +} + +ScopedJavaLocalRef NativeToJavaRtpHeaderExtensionParameter( + JNIEnv* env, + const RtpHeaderExtensionCapability& extension) { + return Java_HeaderExtensionCapability_Constructor( + env, NativeToJavaString(env, extension.uri), extension.preferred_id.value(), + extension.preferred_encrypt); +} +} // namespace + +RtpCapabilities JavaToNativeRtpCapabilities(JNIEnv* jni, + const JavaRef& j_capabilities) { + RtpCapabilities capabilities; + + ScopedJavaLocalRef j_header_extensions = + Java_RtpCapabilities_getHeaderExtensions(jni, j_capabilities); + for (const JavaRef& j_header_extension : + Iterable(jni, j_header_extensions)) { + RtpHeaderExtensionCapability header_extension; + header_extension.uri = JavaToStdString( + jni, Java_HeaderExtensionCapability_getUri(jni, j_header_extension)); + header_extension.preferred_id = Java_HeaderExtensionCapability_getPreferredId(jni, j_header_extension); + header_extension.preferred_encrypt = + Java_HeaderExtensionCapability_getPreferredEncrypted(jni, j_header_extension); + capabilities.header_extensions.push_back(header_extension); + } + + // Convert codecs. + ScopedJavaLocalRef j_codecs = + Java_RtpCapabilities_getCodecs(jni, j_capabilities); + for (const JavaRef& j_codec : Iterable(jni, j_codecs)) { + RtpCodecCapability codec; + codec.preferred_payload_type = Java_CodecCapability_getPreferredPayloadType(jni, j_codec); + codec.name = JavaToStdString(jni, Java_CodecCapability_getName(jni, j_codec)); + codec.kind = JavaToNativeMediaType(jni, Java_CodecCapability_getKind(jni, j_codec)); + codec.clock_rate = + JavaToNativeOptionalInt(jni, Java_CodecCapability_getClockRate(jni, j_codec)); + codec.num_channels = + JavaToNativeOptionalInt(jni, Java_CodecCapability_getNumChannels(jni, j_codec)); + auto parameters_map = + JavaToNativeStringMap(jni, Java_CodecCapability_getParameters(jni, j_codec)); + codec.parameters.insert(parameters_map.begin(), parameters_map.end()); + capabilities.codecs.push_back(codec); + } + return capabilities; +} + +ScopedJavaLocalRef NativeToJavaRtpCapabilities( + JNIEnv* env, + const RtpCapabilities& capabilities) { + return Java_RtpCapabilities_Constructor( + env, NativeToJavaList(env, capabilities.codecs, &NativeToJavaRtpCodecParameter), + NativeToJavaList(env, capabilities.header_extensions, + &NativeToJavaRtpHeaderExtensionParameter) + ); +} + +RtpCodecCapability JavaToNativeRtpCodecCapability(JNIEnv* jni, + const JavaRef& j_codec) { + RtpCodecCapability codec; + codec.preferred_payload_type = Java_CodecCapability_getPreferredPayloadType(jni, j_codec); + codec.name = JavaToStdString(jni, Java_CodecCapability_getName(jni, j_codec)); + codec.kind = JavaToNativeMediaType(jni, Java_CodecCapability_getKind(jni, j_codec)); + codec.clock_rate = + JavaToNativeOptionalInt(jni, Java_CodecCapability_getClockRate(jni, j_codec)); + codec.num_channels = + JavaToNativeOptionalInt(jni, Java_CodecCapability_getNumChannels(jni, j_codec)); + auto parameters_map = + JavaToNativeStringMap(jni, Java_CodecCapability_getParameters(jni, j_codec)); + codec.parameters.insert(parameters_map.begin(), parameters_map.end()); + return codec; +} + +} // namespace jni +} // namespace webrtc diff --git a/sdk/android/src/jni/pc/rtp_capabilities.h b/sdk/android/src/jni/pc/rtp_capabilities.h new file mode 100644 index 0000000000..4acf611c82 --- /dev/null +++ b/sdk/android/src/jni/pc/rtp_capabilities.h @@ -0,0 +1,41 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SDK_ANDROID_SRC_JNI_PC_RTP_CAPABLILITES_H_ +#define SDK_ANDROID_SRC_JNI_PC_RTP_CAPABLILITES_H_ + +#include + +#include "api/rtp_parameters.h" +#include "sdk/android/native_api/jni/scoped_java_ref.h" + +namespace webrtc { +namespace jni { + +RtpCapabilities JavaToNativeRtpCapabilities(JNIEnv* jni, + const JavaRef& j_capabilities); + +ScopedJavaLocalRef NativeToJavaRtpCapabilities( + JNIEnv* jni, + const RtpCapabilities& capabilities); + +RtpCodecCapability JavaToNativeRtpCodecCapability(JNIEnv* jni, + const JavaRef& j_codec_capability); + +} // namespace jni +} // namespace webrtc + +#endif // SDK_ANDROID_SRC_JNI_PC_RTP_CAPABLILITES_H_ diff --git a/sdk/android/src/jni/pc/rtp_transceiver.cc b/sdk/android/src/jni/pc/rtp_transceiver.cc index 1d468461f1..34cba6ab5f 100644 --- a/sdk/android/src/jni/pc/rtp_transceiver.cc +++ b/sdk/android/src/jni/pc/rtp_transceiver.cc @@ -16,6 +16,7 @@ #include "sdk/android/native_api/jni/java_types.h" #include "sdk/android/src/jni/jni_helpers.h" #include "sdk/android/src/jni/pc/media_stream_track.h" +#include "sdk/android/src/jni/pc/rtp_capabilities.h" #include "sdk/android/src/jni/pc/rtp_parameters.h" #include "sdk/android/src/jni/pc/rtp_receiver.h" #include "sdk/android/src/jni/pc/rtp_sender.h" @@ -139,6 +140,16 @@ ScopedJavaLocalRef JNI_RtpTransceiver_CurrentDirection( : nullptr; } + +void JNI_RtpTransceiver_SetCodecPreferences(JNIEnv* jni, + jlong j_rtp_transceiver_pointer, + const JavaParamRef& j_codecs) { + std::vector codecs = + JavaListToNativeVector( + jni, j_codecs, &JavaToNativeRtpCodecCapability); + reinterpret_cast(j_rtp_transceiver_pointer)->SetCodecPreferences(codecs); +} + void JNI_RtpTransceiver_StopInternal(JNIEnv* jni, jlong j_rtp_transceiver_pointer) { reinterpret_cast(j_rtp_transceiver_pointer) diff --git a/sdk/android/src/jni/video_track.cc b/sdk/android/src/jni/video_track.cc index eb343ebdb3..2078359cbc 100644 --- a/sdk/android/src/jni/video_track.cc +++ b/sdk/android/src/jni/video_track.cc @@ -44,5 +44,16 @@ static void JNI_VideoTrack_FreeSink(JNIEnv* jni, jlong j_native_sink) { delete reinterpret_cast*>(j_native_sink); } +static void JNI_VideoTrack_SetShouldReceive(JNIEnv* jni, + jlong j_native_track, + jboolean should_receive) { + reinterpret_cast(j_native_track)->set_should_receive(should_receive); +} + +static jboolean JNI_VideoTrack_GetShouldReceive(JNIEnv* jni, + jlong j_native_track) { + return reinterpret_cast(j_native_track)->should_receive(); +} + } // namespace jni } // namespace webrtc diff --git a/sdk/objc/api/peerconnection/RTCVideoTrack.h b/sdk/objc/api/peerconnection/RTCVideoTrack.h index 5382b7169f..56d25c1568 100644 --- a/sdk/objc/api/peerconnection/RTCVideoTrack.h +++ b/sdk/objc/api/peerconnection/RTCVideoTrack.h @@ -25,6 +25,9 @@ RTC_OBJC_EXPORT /** The video source for this video track. */ @property(nonatomic, readonly) RTC_OBJC_TYPE(RTCVideoSource) *source; +/** The receive state, if this is a remote video track. */ +@property(nonatomic, assign) BOOL shouldReceive; + - (instancetype)init NS_UNAVAILABLE; /** Register a renderer that will render all frames received on this track. */ diff --git a/sdk/objc/api/peerconnection/RTCVideoTrack.mm b/sdk/objc/api/peerconnection/RTCVideoTrack.mm index d3296f6279..df294d2f3e 100644 --- a/sdk/objc/api/peerconnection/RTCVideoTrack.mm +++ b/sdk/objc/api/peerconnection/RTCVideoTrack.mm @@ -70,6 +70,14 @@ - (void)dealloc { return _source; } +- (BOOL)shouldReceive { + return self.nativeVideoTrack->should_receive(); +} + +- (void)setShouldReceive:(BOOL)shouldReceive { + self.nativeVideoTrack->set_should_receive(shouldReceive); +} + - (void)addRenderer:(id)renderer { if (!_workerThread->IsCurrent()) { _workerThread->BlockingCall([renderer, self] { [self addRenderer:renderer]; });