diff --git a/api/video/video_stream_encoder_interface.h b/api/video/video_stream_encoder_interface.h index 69d0ad2363..f2d7e131e6 100644 --- a/api/video/video_stream_encoder_interface.h +++ b/api/video/video_stream_encoder_interface.h @@ -39,7 +39,7 @@ namespace webrtc { // // 2. Moving responsibility for simulcast and for software fallback into this // class. -class VideoStreamEncoderInterface : public rtc::VideoSinkInterface { +class VideoStreamEncoderInterface { public: // Interface for receiving encoded video frames and notifications about // configuration changes. @@ -58,6 +58,8 @@ class VideoStreamEncoderInterface : public rtc::VideoSinkInterface { VideoLayersAllocation allocation) = 0; }; + virtual ~VideoStreamEncoderInterface() = default; + // If the resource is overusing, the VideoStreamEncoder will try to reduce // resolution or frame rate until no resource is overusing. // TODO(https://crbug.com/webrtc/11565): When the ResourceAdaptationProcessor diff --git a/video/BUILD.gn b/video/BUILD.gn index e8cd0cfede..c108e0a0ea 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -53,6 +53,7 @@ rtc_library("video") { ] deps = [ + ":frame_cadence_adapter", ":frame_dumping_decoder", ":video_stream_encoder_impl", "../api:array_view", @@ -257,6 +258,25 @@ rtc_library("frame_dumping_decoder") { ] } +rtc_library("frame_cadence_adapter") { + visibility = [ "*" ] + sources = [ + "frame_cadence_adapter.cc", + "frame_cadence_adapter.h", + ] + + deps = [ + "../api:sequence_checker", + "../api/video:video_frame", + "../rtc_base:logging", + "../rtc_base:macromagic", + "../rtc_base:rtc_base_approved", + "../rtc_base/synchronization:mutex", + "../system_wrappers:field_trial", + "../system_wrappers:metrics", + ] +} + rtc_library("video_stream_encoder_impl") { visibility = [ "*" ] @@ -277,6 +297,7 @@ rtc_library("video_stream_encoder_impl") { ] deps = [ + ":frame_cadence_adapter", "../api:rtp_parameters", "../api:sequence_checker", "../api/adaptation:resource_adaptation_api", @@ -611,6 +632,7 @@ if (rtc_include_tests) { "end_to_end_tests/ssrc_tests.cc", "end_to_end_tests/stats_tests.cc", "end_to_end_tests/transport_feedback_tests.cc", + "frame_cadence_adapter_unittest.cc", "frame_encode_metadata_writer_unittest.cc", "picture_id_tests.cc", "quality_limitation_reason_tracker_unittest.cc", @@ -635,6 +657,7 @@ if (rtc_include_tests) { "video_stream_encoder_unittest.cc", ] deps = [ + ":frame_cadence_adapter", ":video", ":video_legacy", ":video_mocks", diff --git a/video/frame_cadence_adapter.cc b/video/frame_cadence_adapter.cc new file mode 100644 index 0000000000..7c7ba94bd5 --- /dev/null +++ b/video/frame_cadence_adapter.cc @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/frame_cadence_adapter.h" + +#include + +#include "api/sequence_checker.h" +#include "rtc_base/logging.h" +#include "rtc_base/race_checker.h" +#include "rtc_base/synchronization/mutex.h" +#include "system_wrappers/include/field_trial.h" +#include "system_wrappers/include/metrics.h" + +namespace webrtc { +namespace { + +class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface { + public: + FrameCadenceAdapterImpl(); + + // FrameCadenceAdapterInterface overrides. + void Initialize(Callback* callback) override; + void SetZeroHertzModeEnabled(bool enabled) override; + + // VideoFrameSink overrides. + void OnFrame(const VideoFrame& frame) override; + void OnDiscardedFrame() override { callback_->OnDiscardedFrame(); } + void OnConstraintsChanged( + const VideoTrackSourceConstraints& constraints) override; + + private: + // Called to report on constraint UMAs. + void MaybeReportFrameRateConstraintUmas() + RTC_RUN_ON(&incoming_frame_race_checker_) RTC_LOCKS_EXCLUDED(mutex_); + + // True if we support frame entry for screenshare with a minimum frequency of + // 0 Hz. + const bool zero_hertz_screenshare_enabled_; + + // Set up during Initialize. + Callback* callback_ = nullptr; + + // Lock protecting zero-hertz activation state. This is needed because the + // threading contexts of OnFrame, OnConstraintsChanged, and ConfigureEncoder + // are mutating it. + Mutex mutex_; + + // The source's constraints. + absl::optional source_constraints_ + RTC_GUARDED_BY(mutex_); + + // Whether zero-hertz and UMA reporting is enabled. + bool zero_hertz_and_uma_reporting_enabled_ RTC_GUARDED_BY(mutex_) = false; + + // Race checker for incoming frames. This is the network thread in chromium, + // but may vary from test contexts. + rtc::RaceChecker incoming_frame_race_checker_; + bool has_reported_screenshare_frame_rate_umas_ RTC_GUARDED_BY(mutex_) = false; +}; + +FrameCadenceAdapterImpl::FrameCadenceAdapterImpl() + : zero_hertz_screenshare_enabled_( + field_trial::IsEnabled("WebRTC-ZeroHertzScreenshare")) {} + +void FrameCadenceAdapterImpl::Initialize(Callback* callback) { + callback_ = callback; +} + +void FrameCadenceAdapterImpl::SetZeroHertzModeEnabled(bool enabled) { + // This method is called on the worker thread. + MutexLock lock(&mutex_); + if (enabled && !zero_hertz_and_uma_reporting_enabled_) + has_reported_screenshare_frame_rate_umas_ = false; + zero_hertz_and_uma_reporting_enabled_ = enabled; +} + +void FrameCadenceAdapterImpl::OnFrame(const VideoFrame& frame) { + // This method is called on the network thread under Chromium, or other + // various contexts in test. + RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_); + callback_->OnFrame(frame); + MaybeReportFrameRateConstraintUmas(); +} + +void FrameCadenceAdapterImpl::OnConstraintsChanged( + const VideoTrackSourceConstraints& constraints) { + RTC_LOG(LS_INFO) << __func__ << " min_fps " + << constraints.min_fps.value_or(-1) << " max_fps " + << constraints.max_fps.value_or(-1); + MutexLock lock(&mutex_); + source_constraints_ = constraints; +} + +// RTC_RUN_ON(&incoming_frame_race_checker_) +void FrameCadenceAdapterImpl::MaybeReportFrameRateConstraintUmas() { + MutexLock lock(&mutex_); + if (has_reported_screenshare_frame_rate_umas_) + return; + has_reported_screenshare_frame_rate_umas_ = true; + if (!zero_hertz_and_uma_reporting_enabled_) + return; + RTC_HISTOGRAM_BOOLEAN("WebRTC.Screenshare.FrameRateConstraints.Exists", + source_constraints_.has_value()); + if (!source_constraints_.has_value()) + return; + RTC_HISTOGRAM_BOOLEAN("WebRTC.Screenshare.FrameRateConstraints.Min.Exists", + source_constraints_->min_fps.has_value()); + if (source_constraints_->min_fps.has_value()) { + RTC_HISTOGRAM_COUNTS_100( + "WebRTC.Screenshare.FrameRateConstraints.Min.Value", + source_constraints_->min_fps.value()); + } + RTC_HISTOGRAM_BOOLEAN("WebRTC.Screenshare.FrameRateConstraints.Max.Exists", + source_constraints_->max_fps.has_value()); + if (source_constraints_->max_fps.has_value()) { + RTC_HISTOGRAM_COUNTS_100( + "WebRTC.Screenshare.FrameRateConstraints.Max.Value", + source_constraints_->max_fps.value()); + } + if (!source_constraints_->min_fps.has_value()) { + if (source_constraints_->max_fps.has_value()) { + RTC_HISTOGRAM_COUNTS_100( + "WebRTC.Screenshare.FrameRateConstraints.MinUnset.Max", + source_constraints_->max_fps.value()); + } + } else if (source_constraints_->max_fps.has_value()) { + if (source_constraints_->min_fps.value() < + source_constraints_->max_fps.value()) { + RTC_HISTOGRAM_COUNTS_100( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Min", + source_constraints_->min_fps.value()); + RTC_HISTOGRAM_COUNTS_100( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Max", + source_constraints_->max_fps.value()); + } + // Multi-dimensional histogram for min and max FPS making it possible to + // uncover min and max combinations. See + // https://chromium.googlesource.com/chromium/src.git/+/HEAD/tools/metrics/histograms/README.md#multidimensional-histograms + constexpr int kMaxBucketCount = + 60 * /*max min_fps=*/60 + /*max max_fps=*/60 - 1; + RTC_HISTOGRAM_ENUMERATION_SPARSE( + "WebRTC.Screenshare.FrameRateConstraints.60MinPlusMaxMinusOne", + source_constraints_->min_fps.value() * 60 + + source_constraints_->max_fps.value() - 1, + /*boundary=*/kMaxBucketCount); + } +} + +} // namespace + +std::unique_ptr +FrameCadenceAdapterInterface::Create() { + return std::make_unique(); +} + +} // namespace webrtc diff --git a/video/frame_cadence_adapter.h b/video/frame_cadence_adapter.h new file mode 100644 index 0000000000..c0348c742e --- /dev/null +++ b/video/frame_cadence_adapter.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_FRAME_CADENCE_ADAPTER_H_ +#define VIDEO_FRAME_CADENCE_ADAPTER_H_ + +#include + +#include "api/video/video_frame.h" +#include "api/video/video_sink_interface.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" + +namespace webrtc { + +// A sink adapter implementing mutations to the received frame cadence. +// With the exception of construction & destruction which has to happen on the +// same sequence, this class is thread-safe because three different execution +// contexts call into it. +class FrameCadenceAdapterInterface + : public rtc::VideoSinkInterface { + public: + // Callback interface used to inform instance owners. + class Callback { + public: + virtual ~Callback() = default; + + // Called when a frame arrives. + virtual void OnFrame(const VideoFrame& frame) = 0; + + // Called when the source has discarded a frame. + virtual void OnDiscardedFrame() = 0; + }; + + // Factory function creating a production instance. Deletion of the returned + // instance needs to happen on the same sequence that Create() was called on. + static std::unique_ptr Create(); + + // Call before using the rest of the API. + virtual void Initialize(Callback* callback) = 0; + + // Pass true in |enabled| as a prerequisite to enable zero-hertz operation. + virtual void SetZeroHertzModeEnabled(bool enabled) = 0; +}; + +} // namespace webrtc + +#endif // VIDEO_FRAME_CADENCE_ADAPTER_H_ diff --git a/video/frame_cadence_adapter_unittest.cc b/video/frame_cadence_adapter_unittest.cc new file mode 100644 index 0000000000..475425ac06 --- /dev/null +++ b/video/frame_cadence_adapter_unittest.cc @@ -0,0 +1,328 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/frame_cadence_adapter.h" + +#include +#include + +#include "api/video/nv12_buffer.h" +#include "api/video/video_frame.h" +#include "rtc_base/ref_counted_object.h" +#include "system_wrappers/include/metrics.h" +#include "test/field_trial.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +using ::testing::ElementsAre; +using ::testing::Mock; +using ::testing::Pair; +using ::testing::Ref; +using ::testing::UnorderedElementsAre; + +VideoFrame CreateFrame() { + return VideoFrame::Builder() + .set_video_frame_buffer( + rtc::make_ref_counted(/*width=*/16, /*height=*/16)) + .build(); +} + +class MockCallback : public FrameCadenceAdapterInterface::Callback { + public: + MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override)); + MOCK_METHOD(void, OnDiscardedFrame, (), (override)); +}; + +class ZeroHertzFieldTrialDisabler : public test::ScopedFieldTrials { + public: + ZeroHertzFieldTrialDisabler() + : test::ScopedFieldTrials("WebRTC-ZeroHertzScreenshare/Disabled/") {} +}; + +TEST(FrameCadenceAdapterTest, + ForwardsFramesOnConstructionAndUnderDisabledFieldTrial) { + auto disabler = std::make_unique(); + for (int i = 0; i != 2; i++) { + MockCallback callback; + auto adapter = FrameCadenceAdapterInterface::Create(); + adapter->Initialize(&callback); + VideoFrame frame = CreateFrame(); + EXPECT_CALL(callback, OnFrame(Ref(frame))).Times(1); + adapter->OnFrame(frame); + Mock::VerifyAndClearExpectations(&callback); + EXPECT_CALL(callback, OnDiscardedFrame).Times(1); + adapter->OnDiscardedFrame(); + Mock::VerifyAndClearExpectations(&callback); + + disabler = nullptr; + } +} + +class FrameCadenceAdapterMetricsTest : public ::testing::Test { + public: + FrameCadenceAdapterMetricsTest() { metrics::Reset(); } +}; + +TEST_F(FrameCadenceAdapterMetricsTest, RecordsNoUmasWithNoFrameTransfer) { + MockCallback callback; + auto adapter = FrameCadenceAdapterInterface::Create(); + adapter->Initialize(&callback); + adapter->OnConstraintsChanged( + VideoTrackSourceConstraints{absl::nullopt, absl::nullopt}); + adapter->OnConstraintsChanged(VideoTrackSourceConstraints{absl::nullopt, 1}); + adapter->OnConstraintsChanged(VideoTrackSourceConstraints{2, 3}); + adapter->OnConstraintsChanged(VideoTrackSourceConstraints{4, 4}); + adapter->OnConstraintsChanged(VideoTrackSourceConstraints{5, absl::nullopt}); + EXPECT_TRUE(metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Exists") + .empty()); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Exists") + .empty()); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Value") + .empty()); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Exists") + .empty()); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Value") + .empty()); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.MinUnset.Max") + .empty()); + EXPECT_TRUE(metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Min") + .empty()); + EXPECT_TRUE(metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Max") + .empty()); + EXPECT_TRUE( + metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.60MinPlusMaxMinusOne") + .empty()); +} + +TEST_F(FrameCadenceAdapterMetricsTest, RecordsNoUmasWithoutEnabledContentType) { + MockCallback callback; + auto adapter = FrameCadenceAdapterInterface::Create(); + adapter->Initialize(&callback); + adapter->OnFrame(CreateFrame()); + adapter->OnConstraintsChanged( + VideoTrackSourceConstraints{absl::nullopt, absl::nullopt}); + adapter->OnConstraintsChanged(VideoTrackSourceConstraints{absl::nullopt, 1}); + adapter->OnConstraintsChanged(VideoTrackSourceConstraints{2, 3}); + adapter->OnConstraintsChanged(VideoTrackSourceConstraints{4, 4}); + adapter->OnConstraintsChanged(VideoTrackSourceConstraints{5, absl::nullopt}); + EXPECT_TRUE(metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Exists") + .empty()); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Exists") + .empty()); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Value") + .empty()); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Exists") + .empty()); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Value") + .empty()); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.MinUnset.Max") + .empty()); + EXPECT_TRUE(metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Min") + .empty()); + EXPECT_TRUE(metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Max") + .empty()); + EXPECT_TRUE( + metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.60MinPlusMaxMinusOne") + .empty()); +} + +TEST_F(FrameCadenceAdapterMetricsTest, RecordsNoConstraintsIfUnsetOnFrame) { + MockCallback callback; + auto adapter = FrameCadenceAdapterInterface::Create(); + adapter->Initialize(&callback); + adapter->SetZeroHertzModeEnabled(true); + adapter->OnFrame(CreateFrame()); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Exists"), + ElementsAre(Pair(false, 1))); +} + +TEST_F(FrameCadenceAdapterMetricsTest, RecordsEmptyConstraintsIfSetOnFrame) { + MockCallback callback; + auto adapter = FrameCadenceAdapterInterface::Create(); + adapter->Initialize(&callback); + adapter->SetZeroHertzModeEnabled(true); + adapter->OnConstraintsChanged( + VideoTrackSourceConstraints{absl::nullopt, absl::nullopt}); + adapter->OnFrame(CreateFrame()); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Exists"), + ElementsAre(Pair(true, 1))); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Exists"), + ElementsAre(Pair(false, 1))); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Value") + .empty()); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Exists"), + ElementsAre(Pair(false, 1))); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Value") + .empty()); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.MinUnset.Max") + .empty()); + EXPECT_TRUE(metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Min") + .empty()); + EXPECT_TRUE(metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Max") + .empty()); + EXPECT_TRUE( + metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.60MinPlusMaxMinusOne") + .empty()); +} + +TEST_F(FrameCadenceAdapterMetricsTest, RecordsMaxConstraintIfSetOnFrame) { + MockCallback callback; + auto adapter = FrameCadenceAdapterInterface::Create(); + adapter->Initialize(&callback); + adapter->SetZeroHertzModeEnabled(true); + adapter->OnConstraintsChanged( + VideoTrackSourceConstraints{absl::nullopt, 2.0}); + adapter->OnFrame(CreateFrame()); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Exists"), + ElementsAre(Pair(false, 1))); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Value") + .empty()); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Exists"), + ElementsAre(Pair(true, 1))); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Value"), + ElementsAre(Pair(2.0, 1))); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.MinUnset.Max"), + ElementsAre(Pair(2.0, 1))); + EXPECT_TRUE(metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Min") + .empty()); + EXPECT_TRUE(metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Max") + .empty()); + EXPECT_TRUE( + metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.60MinPlusMaxMinusOne") + .empty()); +} + +TEST_F(FrameCadenceAdapterMetricsTest, RecordsMinConstraintIfSetOnFrame) { + MockCallback callback; + auto adapter = FrameCadenceAdapterInterface::Create(); + adapter->Initialize(&callback); + adapter->SetZeroHertzModeEnabled(true); + adapter->OnConstraintsChanged( + VideoTrackSourceConstraints{3.0, absl::nullopt}); + adapter->OnFrame(CreateFrame()); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Exists"), + ElementsAre(Pair(true, 1))); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Value"), + ElementsAre(Pair(3.0, 1))); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Exists"), + ElementsAre(Pair(false, 1))); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Value") + .empty()); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.MinUnset.Max") + .empty()); + EXPECT_TRUE(metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Min") + .empty()); + EXPECT_TRUE(metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Max") + .empty()); + EXPECT_TRUE( + metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.60MinPlusMaxMinusOne") + .empty()); +} + +TEST_F(FrameCadenceAdapterMetricsTest, RecordsMinGtMaxConstraintIfSetOnFrame) { + MockCallback callback; + auto adapter = FrameCadenceAdapterInterface::Create(); + adapter->Initialize(&callback); + adapter->SetZeroHertzModeEnabled(true); + adapter->OnConstraintsChanged(VideoTrackSourceConstraints{5.0, 4.0}); + adapter->OnFrame(CreateFrame()); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Exists"), + ElementsAre(Pair(true, 1))); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Value"), + ElementsAre(Pair(5.0, 1))); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Exists"), + ElementsAre(Pair(true, 1))); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Value"), + ElementsAre(Pair(4.0, 1))); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.MinUnset.Max") + .empty()); + EXPECT_TRUE(metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Min") + .empty()); + EXPECT_TRUE(metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Max") + .empty()); + EXPECT_THAT( + metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.60MinPlusMaxMinusOne"), + ElementsAre(Pair(60 * 5.0 + 4.0 - 1, 1))); +} + +TEST_F(FrameCadenceAdapterMetricsTest, RecordsMinLtMaxConstraintIfSetOnFrame) { + MockCallback callback; + auto adapter = FrameCadenceAdapterInterface::Create(); + adapter->Initialize(&callback); + adapter->SetZeroHertzModeEnabled(true); + adapter->OnConstraintsChanged(VideoTrackSourceConstraints{4.0, 5.0}); + adapter->OnFrame(CreateFrame()); + EXPECT_THAT(metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Min"), + ElementsAre(Pair(4.0, 1))); + EXPECT_THAT(metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Max"), + ElementsAre(Pair(5.0, 1))); + EXPECT_THAT( + metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.60MinPlusMaxMinusOne"), + ElementsAre(Pair(60 * 4.0 + 5.0 - 1, 1))); +} + +} // namespace +} // namespace webrtc diff --git a/video/test/mock_video_stream_encoder.h b/video/test/mock_video_stream_encoder.h index 2af613e3ad..8ea87acc0f 100644 --- a/video/test/mock_video_stream_encoder.h +++ b/video/test/mock_video_stream_encoder.h @@ -43,7 +43,6 @@ class MockVideoStreamEncoder : public VideoStreamEncoderInterface { OnBitrateUpdated, (DataRate, DataRate, DataRate, uint8_t, int64_t, double), (override)); - MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override)); MOCK_METHOD(void, SetFecControllerOverride, (FecControllerOverride*), diff --git a/video/video_send_stream.cc b/video/video_send_stream.cc index 10c209f143..e62a666802 100644 --- a/video/video_send_stream.cc +++ b/video/video_send_stream.cc @@ -23,6 +23,7 @@ #include "system_wrappers/include/clock.h" #include "system_wrappers/include/field_trial.h" #include "video/adaptation/overuse_frame_detector.h" +#include "video/frame_cadence_adapter.h" #include "video/video_stream_encoder.h" namespace webrtc { @@ -135,6 +136,7 @@ VideoSendStream::VideoSendStream( &stats_proxy_, config_.encoder_settings, std::make_unique(&stats_proxy_), + FrameCadenceAdapterInterface::Create(), task_queue_factory, network_queue, GetBitrateAllocationCallbackType(config_))), diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 7e0e7fc63e..010c1d1ffb 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -592,6 +592,7 @@ VideoStreamEncoder::VideoStreamEncoder( VideoStreamEncoderObserver* encoder_stats_observer, const VideoStreamEncoderSettings& settings, std::unique_ptr overuse_detector, + std::unique_ptr frame_cadence_adapter, TaskQueueFactory* task_queue_factory, TaskQueueBase* network_queue, BitrateAllocationCallbackType allocation_cb_type) @@ -604,6 +605,8 @@ VideoStreamEncoder::VideoStreamEncoder( rate_control_settings_(RateControlSettings::ParseFromFieldTrials()), encoder_selector_(settings.encoder_factory->GetEncoderSelector()), encoder_stats_observer_(encoder_stats_observer), + cadence_callback_(*this), + frame_cadence_adapter_(std::move(frame_cadence_adapter)), encoder_initialized_(false), max_framerate_(-1), pending_encoder_reconfiguration_(false), @@ -657,7 +660,7 @@ VideoStreamEncoder::VideoStreamEncoder( settings_.experiment_cpu_load_estimator, std::move(overuse_detector), degradation_preference_manager_.get()), - video_source_sink_controller_(/*sink=*/this, + video_source_sink_controller_(/*sink=*/frame_cadence_adapter_.get(), /*source=*/nullptr), default_limits_allowed_( !field_trial::IsEnabled("WebRTC-DefaultBitrateLimitsKillSwitch")), @@ -671,6 +674,7 @@ VideoStreamEncoder::VideoStreamEncoder( RTC_DCHECK(encoder_stats_observer); RTC_DCHECK_GE(number_of_cores, 1); + frame_cadence_adapter_->Initialize(&cadence_callback_); stream_resource_manager_.Initialize(&encoder_queue_); rtc::Event initialize_processor_event; @@ -821,6 +825,8 @@ void VideoStreamEncoder::SetStartBitrate(int start_bitrate_bps) { void VideoStreamEncoder::ConfigureEncoder(VideoEncoderConfig config, size_t max_data_payload_length) { RTC_DCHECK_RUN_ON(worker_queue_); + frame_cadence_adapter_->SetZeroHertzModeEnabled( + config.content_type == VideoEncoderConfig::ContentType::kScreen); encoder_queue_.PostTask( [this, config = std::move(config), max_data_payload_length]() mutable { RTC_DCHECK_RUN_ON(&encoder_queue_); @@ -830,8 +836,6 @@ void VideoStreamEncoder::ConfigureEncoder(VideoEncoderConfig config, pending_encoder_creation_ = (!encoder_ || encoder_config_.video_format != config.video_format || max_data_payload_length_ != max_data_payload_length); - if (encoder_config_.content_type != config.content_type) - has_reported_screenshare_frame_rate_umas_ = false; encoder_config_ = std::move(config); max_data_payload_length_ = max_data_payload_length; pending_encoder_reconfiguration_ = true; @@ -1331,7 +1335,6 @@ void VideoStreamEncoder::OnFrame(const VideoFrame& video_frame) { encoder_queue_.PostTask( [this, incoming_frame, post_time_us, log_stats]() { RTC_DCHECK_RUN_ON(&encoder_queue_); - MaybeReportFrameRateConstraintUmas(); encoder_stats_observer_->OnIncomingFrame(incoming_frame.width(), incoming_frame.height()); ++captured_frame_count_; @@ -1382,18 +1385,6 @@ void VideoStreamEncoder::OnDiscardedFrame() { VideoStreamEncoderObserver::DropReason::kSource); } -void VideoStreamEncoder::OnConstraintsChanged( - const webrtc::VideoTrackSourceConstraints& constraints) { - RTC_DCHECK_RUN_ON(network_queue_); - RTC_LOG(LS_INFO) << __func__ << " min_fps " - << constraints.min_fps.value_or(-1) << " max_fps " - << constraints.max_fps.value_or(-1); - worker_queue_->PostTask(ToQueuedTask(task_safety_, [this, constraints] { - RTC_DCHECK_RUN_ON(worker_queue_); - source_constraints_ = constraints; - })); -} - bool VideoStreamEncoder::EncoderPaused() const { RTC_DCHECK_RUN_ON(&encoder_queue_); // Pause video if paused by caller or as long as the network is down or the @@ -2364,64 +2355,6 @@ void VideoStreamEncoder::QueueRequestEncoderSwitch( })); } -// RTC_RUN_ON(&encoder_queue_) -void VideoStreamEncoder::MaybeReportFrameRateConstraintUmas() { - if (has_reported_screenshare_frame_rate_umas_) - return; - has_reported_screenshare_frame_rate_umas_ = true; - bool is_screenshare = - encoder_config_.content_type == VideoEncoderConfig::ContentType::kScreen; - if (!is_screenshare) - return; - worker_queue_->PostTask(ToQueuedTask(task_safety_, [this] { - RTC_DCHECK_RUN_ON(worker_queue_); - RTC_HISTOGRAM_BOOLEAN("WebRTC.Screenshare.FrameRateConstraints.Exists", - source_constraints_.has_value()); - if (source_constraints_.has_value()) { - RTC_HISTOGRAM_BOOLEAN( - "WebRTC.Screenshare.FrameRateConstraints.Min.Exists", - source_constraints_->min_fps.has_value()); - if (source_constraints_->min_fps.has_value()) { - RTC_HISTOGRAM_COUNTS_100( - "WebRTC.Screenshare.FrameRateConstraints.Min.Value", - source_constraints_->min_fps.value()); - } - RTC_HISTOGRAM_BOOLEAN( - "WebRTC.Screenshare.FrameRateConstraints.Max.Exists", - source_constraints_->max_fps.has_value()); - if (source_constraints_->max_fps.has_value()) { - RTC_HISTOGRAM_COUNTS_100( - "WebRTC.Screenshare.FrameRateConstraints.Max.Value", - source_constraints_->max_fps.value()); - } - if (!source_constraints_->min_fps.has_value()) { - if (source_constraints_->max_fps.has_value()) { - RTC_HISTOGRAM_COUNTS_100( - "WebRTC.Screenshare.FrameRateConstraints.MinUnset.Max", - source_constraints_->max_fps.value()); - } - } else if (source_constraints_->max_fps.has_value()) { - if (source_constraints_->min_fps.value() < - source_constraints_->max_fps.value()) { - RTC_HISTOGRAM_COUNTS_100( - "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Min", - source_constraints_->min_fps.value()); - RTC_HISTOGRAM_COUNTS_100( - "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Max", - source_constraints_->max_fps.value()); - } - constexpr int kMaxBucketCount = - 60 * /*max min_fps=*/60 + /*max max_fps=*/60 - 1; - RTC_HISTOGRAM_ENUMERATION_SPARSE( - "WebRTC.Screenshare.FrameRateConstraints.60MinPlusMaxMinusOne", - source_constraints_->min_fps.value() * 60 + - source_constraints_->max_fps.value() - 1, - /*boundary=*/kMaxBucketCount); - } - } - })); -} - void VideoStreamEncoder::InjectAdaptationResource( rtc::scoped_refptr resource, VideoAdaptationReason reason) { diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 0738e1d2f5..9d6bb4bc2f 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -45,6 +45,7 @@ #include "system_wrappers/include/clock.h" #include "video/adaptation/video_stream_encoder_resource_manager.h" #include "video/encoder_bitrate_adjuster.h" +#include "video/frame_cadence_adapter.h" #include "video/frame_encode_metadata_writer.h" #include "video/video_source_sink_controller.h" @@ -69,14 +70,16 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, kVideoBitrateAllocationWhenScreenSharing, kVideoLayersAllocation }; - VideoStreamEncoder(Clock* clock, - uint32_t number_of_cores, - VideoStreamEncoderObserver* encoder_stats_observer, - const VideoStreamEncoderSettings& settings, - std::unique_ptr overuse_detector, - TaskQueueFactory* task_queue_factory, - TaskQueueBase* network_queue, - BitrateAllocationCallbackType allocation_cb_type); + VideoStreamEncoder( + Clock* clock, + uint32_t number_of_cores, + VideoStreamEncoderObserver* encoder_stats_observer, + const VideoStreamEncoderSettings& settings, + std::unique_ptr overuse_detector, + std::unique_ptr frame_cadence_adapter, + TaskQueueFactory* task_queue_factory, + TaskQueueBase* network_queue, + BitrateAllocationCallbackType allocation_cb_type); ~VideoStreamEncoder() override; void AddAdaptationResource(rtc::scoped_refptr resource) override; @@ -138,6 +141,22 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, VideoSourceRestrictionsListener* restrictions_listener); private: + class CadenceCallback : public FrameCadenceAdapterInterface::Callback { + public: + explicit CadenceCallback(VideoStreamEncoder& video_stream_encoder) + : video_stream_encoder_(video_stream_encoder) {} + // FrameCadenceAdapterInterface::Callback overrides. + void OnFrame(const VideoFrame& frame) override { + video_stream_encoder_.OnFrame(frame); + } + void OnDiscardedFrame() override { + video_stream_encoder_.OnDiscardedFrame(); + } + + private: + VideoStreamEncoder& video_stream_encoder_; + }; + class VideoFrameInfo { public: VideoFrameInfo(int width, int height, bool is_texture) @@ -173,12 +192,8 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, void ReconfigureEncoder() RTC_RUN_ON(&encoder_queue_); void OnEncoderSettingsChanged() RTC_RUN_ON(&encoder_queue_); - - // Implements VideoSinkInterface. - void OnFrame(const VideoFrame& video_frame) override; - void OnDiscardedFrame() override; - void OnConstraintsChanged( - const webrtc::VideoTrackSourceConstraints& constraints) override; + void OnFrame(const VideoFrame& video_frame); + void OnDiscardedFrame(); void MaybeEncodeVideoFrame(const VideoFrame& frame, int64_t time_when_posted_in_ms); @@ -229,9 +244,6 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, void QueueRequestEncoderSwitch(const webrtc::SdpVideoFormat& format) RTC_RUN_ON(&encoder_queue_); - // Reports UMAs on frame rate constraints usage on the first call. - void MaybeReportFrameRateConstraintUmas() RTC_RUN_ON(&encoder_queue_); - TaskQueueBase* const worker_queue_; TaskQueueBase* const network_queue_; @@ -245,12 +257,12 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, std::unique_ptr const encoder_selector_; VideoStreamEncoderObserver* const encoder_stats_observer_; - - // The source's constraints. - absl::optional source_constraints_ - RTC_GUARDED_BY(worker_queue_); - bool has_reported_screenshare_frame_rate_umas_ - RTC_GUARDED_BY(&encoder_queue_) = false; + // Adapter that avoids public inheritance of the cadence adapter's callback + // interface. + CadenceCallback cadence_callback_; + // Frame cadence encoder adapter. Frames enter this adapter first, and it then + // forwards them to our OnFrame method. + const std::unique_ptr frame_cadence_adapter_; VideoEncoderConfig encoder_config_ RTC_GUARDED_BY(&encoder_queue_); std::unique_ptr encoder_ RTC_GUARDED_BY(&encoder_queue_) diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index b41b4b0d1d..d143a639c5 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -18,6 +18,7 @@ #include "absl/memory/memory.h" #include "api/task_queue/default_task_queue_factory.h" +#include "api/task_queue/task_queue_factory.h" #include "api/test/mock_fec_controller_override.h" #include "api/test/mock_video_encoder.h" #include "api/test/mock_video_encoder_factory.h" @@ -62,6 +63,7 @@ #include "test/mappable_native_buffer.h" #include "test/time_controller/simulated_time_controller.h" #include "test/video_encoder_proxy_factory.h" +#include "video/frame_cadence_adapter.h" #include "video/send_statistics_proxy.h" namespace webrtc { @@ -76,7 +78,9 @@ using ::testing::Gt; using ::testing::Le; using ::testing::Lt; using ::testing::Matcher; +using ::testing::Mock; using ::testing::NiceMock; +using ::testing::Optional; using ::testing::Return; using ::testing::SizeIs; using ::testing::StrictMock; @@ -355,6 +359,7 @@ class VideoStreamEncoderUnderTest : public VideoStreamEncoder { std::unique_ptr( overuse_detector_proxy_ = new CpuOveruseDetectorProxy(stats_proxy)), + FrameCadenceAdapterInterface::Create(), task_queue_factory, TaskQueueBase::Current(), allocation_callback_type), @@ -626,6 +631,79 @@ class MockableSendStatisticsProxy : public SendStatisticsProxy { std::function on_frame_dropped_; }; +class SimpleVideoStreamEncoderFactory { + public: + class AdaptedVideoStreamEncoder : public VideoStreamEncoder { + public: + using VideoStreamEncoder::VideoStreamEncoder; + ~AdaptedVideoStreamEncoder() { Stop(); } + }; + + SimpleVideoStreamEncoderFactory() + : time_controller_(Timestamp::Millis(0)), + task_queue_factory_(time_controller_.CreateTaskQueueFactory()), + stats_proxy_(std::make_unique( + time_controller_.GetClock(), + VideoSendStream::Config(nullptr), + webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)), + encoder_settings_( + VideoEncoder::Capabilities(/*loss_notification=*/false)), + fake_encoder_(time_controller_.GetClock()), + encoder_factory_(&fake_encoder_) { + encoder_settings_.encoder_factory = &encoder_factory_; + } + + std::unique_ptr Create( + std::unique_ptr zero_hertz_adapter) { + auto result = std::make_unique( + time_controller_.GetClock(), + /*number_of_cores=*/1, + /*stats_proxy=*/stats_proxy_.get(), encoder_settings_, + std::make_unique(/*stats_proxy=*/nullptr), + std::move(zero_hertz_adapter), task_queue_factory_.get(), + TaskQueueBase::Current(), + VideoStreamEncoder::BitrateAllocationCallbackType:: + kVideoBitrateAllocation); + result->SetSink(&sink_, /*rotation_applied=*/false); + return result; + } + + private: + class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink { + public: + ~NullEncoderSink() override = default; + void OnEncoderConfigurationChanged( + std::vector streams, + bool is_svc, + VideoEncoderConfig::ContentType content_type, + int min_transmit_bitrate_bps) override {} + void OnBitrateAllocationUpdated( + const VideoBitrateAllocation& allocation) override {} + void OnVideoLayersAllocationUpdated( + VideoLayersAllocation allocation) override {} + Result OnEncodedImage( + const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info) override { + return Result(EncodedImageCallback::Result::OK); + } + }; + + GlobalSimulatedTimeController time_controller_; + std::unique_ptr task_queue_factory_; + std::unique_ptr stats_proxy_; + VideoStreamEncoderSettings encoder_settings_; + test::FakeEncoder fake_encoder_; + test::VideoEncoderProxyFactory encoder_factory_; + NullEncoderSink sink_; +}; + +class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface { + public: + MOCK_METHOD(void, Initialize, (Callback * callback), (override)); + MOCK_METHOD(void, SetZeroHertzModeEnabled, (bool), (override)); + MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override)); +}; + class MockEncoderSelector : public VideoEncoderFactory::EncoderSelectorInterface { public: @@ -8709,4 +8787,43 @@ TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) { RunTest({config1, config2}, /*expected_num_init_encode=*/2); } +TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) { + auto adapter = std::make_unique(); + auto* adapter_ptr = adapter.get(); + SimpleVideoStreamEncoderFactory factory; + auto video_stream_encoder = factory.Create(std::move(adapter)); + + EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(true)); + VideoEncoderConfig config; + config.content_type = VideoEncoderConfig::ContentType::kScreen; + video_stream_encoder->ConfigureEncoder(std::move(config), 0); + Mock::VerifyAndClearExpectations(adapter_ptr); + + EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(false)); + VideoEncoderConfig config2; + config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo; + video_stream_encoder->ConfigureEncoder(std::move(config2), 0); +} + +TEST(VideoStreamEncoderFrameCadenceTest, + ForwardsFramesIntoFrameCadenceAdapter) { + auto adapter = std::make_unique(); + auto* adapter_ptr = adapter.get(); + test::FrameForwarder video_source; + SimpleVideoStreamEncoderFactory factory; + auto video_stream_encoder = factory.Create(std::move(adapter)); + video_stream_encoder->SetSource( + &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE); + + EXPECT_CALL(*adapter_ptr, OnFrame); + auto buffer = rtc::make_ref_counted(/*width=*/16, /*height=*/16); + video_source.IncomingCapturedFrame( + VideoFrame::Builder() + .set_video_frame_buffer(std::move(buffer)) + .set_ntp_time_ms(0) + .set_timestamp_ms(0) + .set_rotation(kVideoRotation_0) + .build()); +} + } // namespace webrtc