From e9891d83fc1b5d7219d612812c40346f5db52d7f Mon Sep 17 00:00:00 2001 From: Kanat Kiialbaev Date: Wed, 20 Sep 2023 12:44:37 -0700 Subject: [PATCH] Android improvements. (#20) Start/Stop receiving stream method for VideoTrack (#25) Properly remove observer upon deconstruction (#26) feat: Expose setCodecPreferences/getCapabilities for android. (#61) fix: add WrappedVideoDecoderFactory.java. (#74) Exposing Adapter types in PeerConnectionFactory (#78) Co-authored-by: davidliu Co-authored-by: Mohamed Risaldar UT <73091075+MohamedRisaldarTA@users.noreply.github.com> (cherry picked from commit e91f003820c099422e4d26869fb1eeb1611d72d0) # Conflicts: # media/base/media_channel.h # media/engine/webrtc_video_engine.cc # media/engine/webrtc_video_engine.h --- api/media_stream_interface.cc | 4 ++ api/media_stream_interface.h | 2 + media/base/media_channel.h | 2 + media/engine/webrtc_video_engine.cc | 29 +++++++++++++++ media/engine/webrtc_video_engine.h | 6 ++- pc/media_stream_track_proxy.h | 2 + pc/video_rtp_receiver.cc | 39 +++++++++++++++++++- pc/video_rtp_receiver.h | 11 +++++- pc/video_track.cc | 13 +++++++ pc/video_track.h | 4 ++ sdk/android/api/org/webrtc/VideoTrack.java | 20 ++++++++++ sdk/android/src/jni/video_track.cc | 11 ++++++ sdk/objc/api/peerconnection/RTCVideoTrack.h | 3 ++ sdk/objc/api/peerconnection/RTCVideoTrack.mm | 8 ++++ 14 files changed, 151 insertions(+), 3 deletions(-) diff --git a/api/media_stream_interface.cc b/api/media_stream_interface.cc index 6b0a6a9297..d8a79896f3 100644 --- a/api/media_stream_interface.cc +++ b/api/media_stream_interface.cc @@ -19,6 +19,10 @@ const char* const MediaStreamTrackInterface::kVideoKind = const char* const MediaStreamTrackInterface::kAudioKind = cricket::kMediaTypeAudio; +bool VideoTrackInterface::should_receive() const { + return true; +} + VideoTrackInterface::ContentHint VideoTrackInterface::content_hint() const { return ContentHint::kNone; } diff --git a/api/media_stream_interface.h b/api/media_stream_interface.h index 9d336739e4..6d9b4aa6ca 100644 --- a/api/media_stream_interface.h +++ b/api/media_stream_interface.h @@ -188,6 +188,8 @@ class RTC_EXPORT VideoTrackInterface virtual VideoTrackSourceInterface* GetSource() const = 0; + virtual void set_should_receive(bool should_receive) {} + virtual bool should_receive() const; virtual ContentHint content_hint() const; virtual void set_content_hint(ContentHint hint) {} diff --git a/media/base/media_channel.h b/media/base/media_channel.h index 2282b57d7e..15da82e457 100644 --- a/media/base/media_channel.h +++ b/media/base/media_channel.h @@ -1001,6 +1001,8 @@ class VideoMediaReceiveChannelInterface : public MediaReceiveChannelInterface { webrtc::RtcpMode rtcp_mode, absl::optional rtx_time) = 0; virtual bool AddDefaultRecvStreamForTesting(const StreamParams& sp) = 0; + virtual void StartReceive(uint32_t ssrc) {} + virtual void StopReceive(uint32_t ssrc) {} }; } // namespace cricket diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index e8bffc2d05..3e62e7cdbe 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc @@ -2579,6 +2579,24 @@ WebRtcVideoReceiveChannel::~WebRtcVideoReceiveChannel() { delete kv.second; } +void WebRtcVideoReceiveChannel::StartReceive(uint32_t ssrc) { + RTC_DCHECK_RUN_ON(&thread_checker_); + WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc); + if(!stream) { + return; + } + stream->StartStream(); +} + +void WebRtcVideoReceiveChannel::StopReceive(uint32_t ssrc) { + RTC_DCHECK_RUN_ON(&thread_checker_); + WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc); + if(!stream) { + return; + } + stream->StopStream(); +} + void WebRtcVideoReceiveChannel::SetReceiverFeedbackParameters( bool lntf_enabled, bool nack_enabled, @@ -3478,6 +3496,17 @@ void WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::SetReceiverParameters( } } +void WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::StartStream(){ + if (stream_) { + stream_->Start(); + } +} +void WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::StopStream(){ + if (stream_) { + stream_->Stop(); + } +} + void WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream:: RecreateReceiveStream() { RTC_DCHECK_RUN_ON(&thread_checker_); diff --git a/media/engine/webrtc_video_engine.h b/media/engine/webrtc_video_engine.h index 11f1b99ac2..599b8063a6 100644 --- a/media/engine/webrtc_video_engine.h +++ b/media/engine/webrtc_video_engine.h @@ -294,7 +294,6 @@ class WebRtcVideoSendChannel : public MediaChannelUtil, } return send_codec()->rtx_time; } - private: struct ChangedSenderParameters { // These optionals are unset if not changed. @@ -643,6 +642,8 @@ class WebRtcVideoReceiveChannel : public MediaChannelUtil, webrtc::RtcpMode rtcp_mode, absl::optional rtx_time) override; + void StartReceive(uint32_t ssrc) override; + void StopReceive(uint32_t ssrc) override; private: class WebRtcVideoReceiveStream; struct ChangedReceiverParameters { @@ -744,6 +745,9 @@ class WebRtcVideoReceiveChannel : public MediaChannelUtil, rtc::scoped_refptr frame_transformer); + void StartStream(); + void StopStream(); + void SetLocalSsrc(uint32_t local_ssrc); void UpdateRtxSsrc(uint32_t ssrc); void StartReceiveStream(); diff --git a/pc/media_stream_track_proxy.h b/pc/media_stream_track_proxy.h index 2af3aedb22..fab23d17ec 100644 --- a/pc/media_stream_track_proxy.h +++ b/pc/media_stream_track_proxy.h @@ -55,6 +55,8 @@ PROXY_SECONDARY_METHOD2(void, PROXY_SECONDARY_METHOD1(void, RemoveSink, rtc::VideoSinkInterface*) PROXY_SECONDARY_METHOD0(void, RequestRefreshFrame) BYPASS_PROXY_CONSTMETHOD0(VideoTrackSourceInterface*, GetSource) +PROXY_CONSTMETHOD0(bool, should_receive) +PROXY_METHOD1(void, set_should_receive, bool) PROXY_METHOD1(void, RegisterObserver, ObserverInterface*) PROXY_METHOD1(void, UnregisterObserver, ObserverInterface*) diff --git a/pc/video_rtp_receiver.cc b/pc/video_rtp_receiver.cc index 152bc6b439..1ed13b9088 100644 --- a/pc/video_rtp_receiver.cc +++ b/pc/video_rtp_receiver.cc @@ -41,9 +41,12 @@ VideoRtpReceiver::VideoRtpReceiver( rtc::Thread::Current(), worker_thread, VideoTrack::Create(receiver_id, source_, worker_thread))), - attachment_id_(GenerateUniqueId()) { + cached_track_should_receive_(track_->should_receive()), + attachment_id_(GenerateUniqueId()), + worker_thread_safety_(PendingTaskSafetyFlag::CreateDetachedInactive()) { RTC_DCHECK(worker_thread_); SetStreams(streams); + track_->RegisterObserver(this); RTC_DCHECK_EQ(source_->state(), MediaSourceInterface::kInitializing); } @@ -116,6 +119,39 @@ void VideoRtpReceiver::Stop() { track_->internal()->set_ended(); } +void VideoRtpReceiver::OnChanged() { + RTC_DCHECK_RUN_ON(&signaling_thread_checker_); + if (cached_track_should_receive_ != track_->should_receive()) { + cached_track_should_receive_ = track_->should_receive(); + worker_thread_->PostTask( + [this, receive = cached_track_should_receive_]() { + RTC_DCHECK_RUN_ON(worker_thread_); + if(receive) { + StartMediaChannel(); + } else { + StopMediaChannel(); + } + }); + } +} + +void VideoRtpReceiver::StartMediaChannel() { + RTC_DCHECK_RUN_ON(worker_thread_); + if (!media_channel_) { + return; + } + media_channel_->StartReceive(signaled_ssrc_.value_or(0)); + OnGenerateKeyFrame(); +} + +void VideoRtpReceiver::StopMediaChannel() { + RTC_DCHECK_RUN_ON(worker_thread_); + if (!media_channel_) { + return; + } + media_channel_->StopReceive(signaled_ssrc_.value_or(0)); +} + void VideoRtpReceiver::RestartMediaChannel(absl::optional ssrc) { RTC_DCHECK_RUN_ON(&signaling_thread_checker_); MediaSourceInterface::SourceState state = source_->state(); @@ -211,6 +247,7 @@ void VideoRtpReceiver::set_transport( void VideoRtpReceiver::SetStreams( const std::vector>& streams) { RTC_DCHECK_RUN_ON(&signaling_thread_checker_); + // Remove remote track from any streams that are going away. for (const auto& existing_stream : streams_) { bool removed = true; diff --git a/pc/video_rtp_receiver.h b/pc/video_rtp_receiver.h index ef88016052..491efe2931 100644 --- a/pc/video_rtp_receiver.h +++ b/pc/video_rtp_receiver.h @@ -42,7 +42,8 @@ namespace webrtc { -class VideoRtpReceiver : public RtpReceiverInternal { +class VideoRtpReceiver : public RtpReceiverInternal, + public ObserverInterface { public: // An SSRC of 0 will create a receiver that will match the first SSRC it // sees. Must be called on signaling thread. @@ -60,6 +61,9 @@ class VideoRtpReceiver : public RtpReceiverInternal { rtc::scoped_refptr video_track() const { return track_; } + // ObserverInterface implementation + void OnChanged() override; + // RtpReceiverInterface implementation rtc::scoped_refptr track() const override { return track_; @@ -115,6 +119,8 @@ class VideoRtpReceiver : public RtpReceiverInternal { cricket::MediaReceiveChannelInterface* media_channel); private: + void StartMediaChannel(); + void StopMediaChannel(); void RestartMediaChannel(absl::optional ssrc) RTC_RUN_ON(&signaling_thread_checker_); void RestartMediaChannel_w(absl::optional ssrc, @@ -162,6 +168,8 @@ class VideoRtpReceiver : public RtpReceiverInternal { RTC_GUARDED_BY(&signaling_thread_checker_) = nullptr; bool received_first_packet_ RTC_GUARDED_BY(&signaling_thread_checker_) = false; + + bool cached_track_should_receive_ RTC_GUARDED_BY(&signaling_thread_checker_); const int attachment_id_; rtc::scoped_refptr frame_decryptor_ RTC_GUARDED_BY(worker_thread_); @@ -177,6 +185,7 @@ class VideoRtpReceiver : public RtpReceiverInternal { // or switched. bool saved_generate_keyframe_ RTC_GUARDED_BY(worker_thread_) = false; bool saved_encoded_sink_enabled_ RTC_GUARDED_BY(worker_thread_) = false; + const rtc::scoped_refptr worker_thread_safety_; }; } // namespace webrtc diff --git a/pc/video_track.cc b/pc/video_track.cc index 0bf8687af3..8922cdaf1f 100644 --- a/pc/video_track.cc +++ b/pc/video_track.cc @@ -76,6 +76,19 @@ VideoTrackSourceInterface* VideoTrack::GetSourceInternal() const { return video_source_->internal(); } +void VideoTrack::set_should_receive(bool receive) { + RTC_DCHECK_RUN_ON(&signaling_thread_); + if (should_receive_ == receive) + return; + should_receive_ = receive; + Notifier::FireOnChanged(); +} + +bool VideoTrack::should_receive() const { + RTC_DCHECK_RUN_ON(&signaling_thread_); + return should_receive_; +} + VideoTrackInterface::ContentHint VideoTrack::content_hint() const { RTC_DCHECK_RUN_ON(&signaling_thread_); return content_hint_; diff --git a/pc/video_track.h b/pc/video_track.h index 13a51c454b..b56c64ef20 100644 --- a/pc/video_track.h +++ b/pc/video_track.h @@ -48,6 +48,9 @@ class VideoTrack : public MediaStreamTrack, void RequestRefreshFrame() override; VideoTrackSourceInterface* GetSource() const override; + void set_should_receive(bool should_receive) override; + bool should_receive() const override; + ContentHint content_hint() const override; void set_content_hint(ContentHint hint) override; bool set_enabled(bool enable) override; @@ -81,6 +84,7 @@ class VideoTrack : public MediaStreamTrack, // be queried without blocking on the worker thread by callers that don't // use an api proxy to call the `enabled()` method. bool enabled_w_ RTC_GUARDED_BY(worker_thread_) = true; + bool should_receive_ RTC_GUARDED_BY(signaling_thread_) = true; }; } // namespace webrtc diff --git a/sdk/android/api/org/webrtc/VideoTrack.java b/sdk/android/api/org/webrtc/VideoTrack.java index 512e46c26e..1791592b56 100644 --- a/sdk/android/api/org/webrtc/VideoTrack.java +++ b/sdk/android/api/org/webrtc/VideoTrack.java @@ -54,6 +54,24 @@ public void removeSink(VideoSink sink) { } } + /** + * For a remote video track, starts/stops receiving the video stream. + * + * If this is a local video track, this is a no-op. + */ + public void setShouldReceive(boolean shouldReceive){ + nativeSetShouldReceive(getNativeMediaStreamTrack(), shouldReceive); + } + + /** + * The current receive status for a remote video track. + * + * This has no meaning for a local video track. + */ + public boolean shouldReceive(){ + return nativeGetShouldReceive(getNativeMediaStreamTrack()); + } + @Override public void dispose() { for (long nativeSink : sinks.values()) { @@ -73,4 +91,6 @@ public long getNativeVideoTrack() { private static native void nativeRemoveSink(long track, long nativeSink); private static native long nativeWrapSink(VideoSink sink); private static native void nativeFreeSink(long sink); + private static native void nativeSetShouldReceive(long track, boolean shouldReceive); + private static native boolean nativeGetShouldReceive(long track); } diff --git a/sdk/android/src/jni/video_track.cc b/sdk/android/src/jni/video_track.cc index eb343ebdb3..2078359cbc 100644 --- a/sdk/android/src/jni/video_track.cc +++ b/sdk/android/src/jni/video_track.cc @@ -44,5 +44,16 @@ static void JNI_VideoTrack_FreeSink(JNIEnv* jni, jlong j_native_sink) { delete reinterpret_cast*>(j_native_sink); } +static void JNI_VideoTrack_SetShouldReceive(JNIEnv* jni, + jlong j_native_track, + jboolean should_receive) { + reinterpret_cast(j_native_track)->set_should_receive(should_receive); +} + +static jboolean JNI_VideoTrack_GetShouldReceive(JNIEnv* jni, + jlong j_native_track) { + return reinterpret_cast(j_native_track)->should_receive(); +} + } // namespace jni } // namespace webrtc diff --git a/sdk/objc/api/peerconnection/RTCVideoTrack.h b/sdk/objc/api/peerconnection/RTCVideoTrack.h index 5382b7169f..56d25c1568 100644 --- a/sdk/objc/api/peerconnection/RTCVideoTrack.h +++ b/sdk/objc/api/peerconnection/RTCVideoTrack.h @@ -25,6 +25,9 @@ RTC_OBJC_EXPORT /** The video source for this video track. */ @property(nonatomic, readonly) RTC_OBJC_TYPE(RTCVideoSource) *source; +/** The receive state, if this is a remote video track. */ +@property(nonatomic, assign) BOOL shouldReceive; + - (instancetype)init NS_UNAVAILABLE; /** Register a renderer that will render all frames received on this track. */ diff --git a/sdk/objc/api/peerconnection/RTCVideoTrack.mm b/sdk/objc/api/peerconnection/RTCVideoTrack.mm index d3296f6279..df294d2f3e 100644 --- a/sdk/objc/api/peerconnection/RTCVideoTrack.mm +++ b/sdk/objc/api/peerconnection/RTCVideoTrack.mm @@ -70,6 +70,14 @@ - (void)dealloc { return _source; } +- (BOOL)shouldReceive { + return self.nativeVideoTrack->should_receive(); +} + +- (void)setShouldReceive:(BOOL)shouldReceive { + self.nativeVideoTrack->set_should_receive(shouldReceive); +} + - (void)addRenderer:(id)renderer { if (!_workerThread->IsCurrent()) { _workerThread->BlockingCall([renderer, self] { [self addRenderer:renderer]; });