diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 8b328647d6..fcae1d6865 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -185,6 +185,31 @@ std::array GetExperimentGroups() { } return experiment_groups; } + +// Limit allocation across TLs in bitrate allocation according to number of TLs +// in EncoderInfo. +VideoBitrateAllocation UpdateAllocationFromEncoderInfo( + const VideoBitrateAllocation& allocation, + const VideoEncoder::EncoderInfo& encoder_info) { + if (allocation.get_sum_bps() == 0) { + return allocation; + } + VideoBitrateAllocation new_allocation; + for (int si = 0; si < kMaxSpatialLayers; ++si) { + if (encoder_info.fps_allocation[si].size() == 1 && + allocation.IsSpatialLayerUsed(si)) { + // One TL is signalled to be used by the encoder. Do not distribute + // bitrate allocation across TLs (use sum at ti:0). + new_allocation.SetBitrate(si, 0, allocation.GetSpatialLayerSum(si)); + } else { + for (int ti = 0; ti < kMaxTemporalStreams; ++ti) { + if (allocation.HasBitrate(si, ti)) + new_allocation.SetBitrate(si, ti, allocation.GetBitrate(si, ti)); + } + } + } + return new_allocation; +} } // namespace // VideoSourceProxy is responsible ensuring thread safety between calls to @@ -722,6 +747,7 @@ void VideoStreamEncoder::ReconfigureEncoder() { } frame_encoder_timer_.Reset(); + last_encode_info_ms_ = absl::nullopt; } if (success) { @@ -977,7 +1003,20 @@ VideoStreamEncoder::GetBitrateAllocationAndNotifyObserver( } if (bitrate_observer_ && bitrate_allocation.get_sum_bps() > 0) { - bitrate_observer_->OnBitrateAllocationUpdated(bitrate_allocation); + if (encoder_ && encoder_initialized_) { + // Avoid too old encoder_info_. + const int64_t kMaxDiffMs = 100; + const bool updated_recently = + (last_encode_info_ms_ && ((clock_->TimeInMilliseconds() - + *last_encode_info_ms_) < kMaxDiffMs)); + // Update allocation according to info from encoder. + bitrate_observer_->OnBitrateAllocationUpdated( + UpdateAllocationFromEncoderInfo( + bitrate_allocation, + updated_recently ? encoder_info_ : encoder_->GetEncoderInfo())); + } else { + bitrate_observer_->OnBitrateAllocationUpdated(bitrate_allocation); + } } if (bitrate_adjuster_) { @@ -1238,6 +1277,7 @@ void VideoStreamEncoder::EncodeVideoFrame(const VideoFrame& video_frame, } encoder_info_ = info; + last_encode_info_ms_ = clock_->TimeInMilliseconds(); RTC_DCHECK_EQ(send_codec_.width, out_frame.width()); RTC_DCHECK_EQ(send_codec_.height, out_frame.height()); const VideoFrameBuffer::Type buffer_type = diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 663a020fb9..41e865cf59 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -290,6 +290,7 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, RTC_GUARDED_BY(&encoder_queue_); absl::optional last_parameters_update_ms_ RTC_GUARDED_BY(&encoder_queue_); + absl::optional last_encode_info_ms_ RTC_GUARDED_BY(&encoder_queue_); VideoEncoder::EncoderInfo encoder_info_ RTC_GUARDED_BY(&encoder_queue_); VideoEncoderFactory::CodecInfo codec_info_ RTC_GUARDED_BY(&encoder_queue_); diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index 5193af5e0b..278e566666 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -23,6 +23,7 @@ #include "media/base/video_adapter.h" #include "modules/video_coding/codecs/vp9/include/vp9_globals.h" #include "modules/video_coding/utility/default_video_bitrate_allocator.h" +#include "modules/video_coding/utility/simulcast_rate_allocator.h" #include "rtc_base/fake_clock.h" #include "rtc_base/logging.h" #include "rtc_base/ref_counted_object.h" @@ -407,6 +408,20 @@ class VideoStreamEncoderTest : public ::testing::Test { return frame; } + void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) { + MockBitrateObserver bitrate_observer; + video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer); + + EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate)) + .Times(1); + video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kTargetBitrateBps), + DataRate::Zero(), 0, 0); + + video_source_.IncomingCapturedFrame( + CreateFrame(1, codec_width_, codec_height_)); + WaitForEncodedFrame(1); + } + void VerifyNoLimitation(const rtc::VideoSinkWants& wants) { EXPECT_EQ(std::numeric_limits::max(), wants.max_framerate_fps); EXPECT_EQ(std::numeric_limits::max(), wants.max_pixel_count); @@ -555,6 +570,12 @@ class VideoStreamEncoderTest : public ::testing::Test { VideoEncoder::ScalingSettings(1, 2, kMinPixelsPerFrame); } info.is_hardware_accelerated = is_hardware_accelerated_; + for (int i = 0; i < kMaxSpatialLayers; ++i) { + if (temporal_layers_supported_[i]) { + int num_layers = temporal_layers_supported_[i].value() ? 2 : 1; + info.fps_allocation[i].resize(num_layers); + } + } } return info; } @@ -585,6 +606,12 @@ class VideoStreamEncoderTest : public ::testing::Test { is_hardware_accelerated_ = is_hardware_accelerated; } + void SetTemporalLayersSupported(size_t spatial_idx, bool supported) { + RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers); + rtc::CritScope lock(&local_crit_sect_); + temporal_layers_supported_[spatial_idx] = supported; + } + void ForceInitEncodeFailure(bool force_failure) { rtc::CritScope lock(&local_crit_sect_); force_init_encode_failed_ = force_failure; @@ -735,6 +762,9 @@ class VideoStreamEncoderTest : public ::testing::Test { bool is_hardware_accelerated_ RTC_GUARDED_BY(local_crit_sect_) = false; std::unique_ptr frame_buffer_controller_ RTC_GUARDED_BY(local_crit_sect_); + absl::optional + temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY( + local_crit_sect_); bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false; double rate_factor_ RTC_GUARDED_BY(local_crit_sect_) = 1.0; uint32_t last_framerate_ RTC_GUARDED_BY(local_crit_sect_) = 0; @@ -2326,6 +2356,65 @@ TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) { video_stream_encoder_->Stop(); } +TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) { + // 2 TLs configured, temporal layers supported by encoder. + const int kNumTemporalLayers = 2; + ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false); + fake_encoder_.SetTemporalLayersSupported(0, true); + + // Bitrate allocated across temporal layers. + const int kTl0Bps = kTargetBitrateBps * + webrtc::SimulcastRateAllocator::GetTemporalRateAllocation( + kNumTemporalLayers, /*temporal_id*/ 0); + const int kTl1Bps = kTargetBitrateBps * + webrtc::SimulcastRateAllocator::GetTemporalRateAllocation( + kNumTemporalLayers, /*temporal_id*/ 1); + VideoBitrateAllocation expected_bitrate; + expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps); + expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps); + + VerifyAllocatedBitrate(expected_bitrate); + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) { + // 2 TLs configured, temporal layers not supported by encoder. + ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false); + fake_encoder_.SetTemporalLayersSupported(0, false); + + // Temporal layers not supported by the encoder. + // Total bitrate should be at ti:0. + VideoBitrateAllocation expected_bitrate; + expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps); + + VerifyAllocatedBitrate(expected_bitrate); + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) { + // 2 TLs configured, temporal layers only supported for first stream. + ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false); + fake_encoder_.SetTemporalLayersSupported(0, true); + fake_encoder_.SetTemporalLayersSupported(1, false); + + const int kS0Bps = 150000; + const int kS0Tl0Bps = + kS0Bps * webrtc::SimulcastRateAllocator::GetTemporalRateAllocation( + /*num_layers*/ 2, /*temporal_id*/ 0); + const int kS0Tl1Bps = + kS0Bps * webrtc::SimulcastRateAllocator::GetTemporalRateAllocation( + /*num_layers*/ 2, /*temporal_id*/ 1); + const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps; + // Temporal layers not supported by si:1. + VideoBitrateAllocation expected_bitrate; + expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps); + expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps); + expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps); + + VerifyAllocatedBitrate(expected_bitrate); + video_stream_encoder_->Stop(); +} + TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) { const int kFrameWidth = 1280; const int kFrameHeight = 720;