admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ZeroHertzAdapterMode: activate earlier.

Browse Source 
Careful analysis of logs related to the requesting of refresh
frames from the source revealed an uncomfortable truth:
zero-hertz mode activates first when the first frame has been
received in the VideoStreamEncoder, because the number of simulcast
layers can only be computed when frame dimensions are known. This
fact means that the currently implemented logic for requesting
refresh frames is noneffective.

Fix this by
1. Activating zero-hertz mode prior of knowing the final layer
count. This causes refresh frame requests to happen without any
frames received from the source.
2. Making layer count dynamically configurable. Zero-hertz mode
considers layer quality unconverged after such a reconfiguration.

go/rtc-0hz-present

Bug: chromium:1255737
Change-Id: I0ecea4d2a8442a00e3b79b146dd39a5f4ac561d9
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/242860
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Markus Handell <handellm@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#35593}
This commit is contained in:
Markus Handell 2021-12-23 09:29:23 +01:00 committed by WebRTC LUCI CQ
parent 5065e5b922
commit e59fee87fb
4 changed files with 131 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
video/frame_cadence_adapter.cc
View File

@ -96,12 +96,15 @@ class PassthroughAdapterMode : public AdapterMode {
// Implements a frame cadence adapter supporting zero-hertz input.
class ZeroHertzAdapterMode : public AdapterMode {
public:
ZeroHertzAdapterMode(
TaskQueueBase* queue,
Clock* clock,
FrameCadenceAdapterInterface::Callback* callback,
double max_fps,
FrameCadenceAdapterInterface::ZeroHertzModeParams params);
ZeroHertzAdapterMode(TaskQueueBase* queue,
Clock* clock,
FrameCadenceAdapterInterface::Callback* callback,
double max_fps);
// Reconfigures according to parameters.
// All spatial layer trackers are initialized as unconverged by this method.
void ReconfigureParameters(
const FrameCadenceAdapterInterface::ZeroHertzModeParams& params);
// Updates spatial layer quality convergence status.
void UpdateLayerQualityConvergence(int spatial_index, bool quality_converged);
@ -143,6 +146,9 @@ class ZeroHertzAdapterMode : public AdapterMode {
// Convergence means QP has dropped to a low-enough level to warrant ceasing
// to send identical frames at high frequency.
bool HasQualityConverged() const RTC_RUN_ON(sequence_checker_);
// Resets quality convergence information. HasQualityConverged() returns false
// after this call.
void ResetQualityConvergenceInfo() RTC_RUN_ON(sequence_checker_);
// Processes incoming frames on a delayed cadence.
void ProcessOnDelayedCadence() RTC_RUN_ON(sequence_checker_);
// Schedules a later repeat with delay depending on state of layer trackers.
@ -191,6 +197,7 @@ class ZeroHertzAdapterMode : public AdapterMode {
class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface {
public:
FrameCadenceAdapterImpl(Clock* clock, TaskQueueBase* queue);
~FrameCadenceAdapterImpl();
// FrameCadenceAdapterInterface overrides.
void Initialize(Callback* callback) override;
@ -266,22 +273,29 @@ ZeroHertzAdapterMode::ZeroHertzAdapterMode(
TaskQueueBase* queue,
Clock* clock,
FrameCadenceAdapterInterface::Callback* callback,
double max_fps,
FrameCadenceAdapterInterface::ZeroHertzModeParams params)
: queue_(queue),
clock_(clock),
callback_(callback),
max_fps_(max_fps),
layer_trackers_(params.num_simulcast_layers) {
double max_fps)
: queue_(queue), clock_(clock), callback_(callback), max_fps_(max_fps) {
sequence_checker_.Detach();
}
void ZeroHertzAdapterMode::ReconfigureParameters(
const FrameCadenceAdapterInterface::ZeroHertzModeParams& params) {
RTC_DCHECK_RUN_ON(&sequence_checker_);
RTC_LOG(LS_INFO) << __func__ << " this " << this << " num_simulcast_layers "
<< params.num_simulcast_layers;
// Start as unconverged.
layer_trackers_.clear();
layer_trackers_.resize(params.num_simulcast_layers,
SpatialLayerTracker{false});
}
void ZeroHertzAdapterMode::UpdateLayerQualityConvergence(
int spatial_index,
bool quality_converged) {
RTC_DCHECK_RUN_ON(&sequence_checker_);
RTC_DCHECK_LT(spatial_index, layer_trackers_.size());
RTC_LOG(LS_INFO) << __func__ << " layer " << spatial_index
RTC_LOG(LS_INFO) << __func__ << " this " << this << " layer " << spatial_index
<< " quality has converged: " << quality_converged;
if (layer_trackers_[spatial_index].quality_converged.has_value())
layer_trackers_[spatial_index].quality_converged = quality_converged;
@ -299,7 +313,7 @@ void ZeroHertzAdapterMode::UpdateLayerStatus(int spatial_index, bool enabled) {
layer_trackers_[spatial_index].quality_converged = absl::nullopt;
}
RTC_LOG(LS_INFO)
<< __func__ << " layer " << spatial_index
<< __func__ << " this " << this << " layer " << spatial_index
<< (enabled
? (layer_trackers_[spatial_index].quality_converged.has_value()
? " enabled."
@ -311,13 +325,11 @@ void ZeroHertzAdapterMode::OnFrame(Timestamp post_time,
int frames_scheduled_for_processing,
const VideoFrame& frame) {
RTC_DCHECK_RUN_ON(&sequence_checker_);
RTC_DLOG(LS_VERBOSE) << __func__ << " this " << this;
RTC_DLOG(LS_VERBOSE) << "ZeroHertzAdapterMode::" << __func__ << " this "
<< this;
// Assume all enabled layers are unconverged after frame entry.
for (auto& layer_tracker : layer_trackers_) {
if (layer_tracker.quality_converged.has_value())
layer_tracker.quality_converged = false;
}
ResetQualityConvergenceInfo();
// Remove stored repeating frame if needed.
if (scheduled_repeat_.has_value()) {
@ -349,9 +361,9 @@ bool ZeroHertzAdapterMode::ProcessKeyFrameRequest() {
// If no frame was ever passed to us, request a refresh frame from the source.
if (current_frame_id_ == 0) {
RTC_LOG(LS_INFO) << __func__ << " this " << this
<< " recommending requesting refresh frame due to no "
"frames received yet.";
RTC_LOG(LS_INFO)
<< __func__ << " this " << this
<< " requesting refresh frame due to no frames received yet.";
return true;
}
@ -359,7 +371,7 @@ bool ZeroHertzAdapterMode::ProcessKeyFrameRequest() {
// very soon send out a frame and don't need a refresh frame.
if (!scheduled_repeat_.has_value() || !scheduled_repeat_->idle) {
RTC_LOG(LS_INFO) << __func__ << " this " << this
<< " ignoring key frame request because of recently "
<< " not requesting refresh frame because of recently "
"incoming frame or short repeating.";
return false;
}
@ -370,16 +382,17 @@ bool ZeroHertzAdapterMode::ProcessKeyFrameRequest() {
if (scheduled_repeat_->scheduled + RepeatDuration(/*idle_repeat=*/true) -
now <=
frame_delay_) {
RTC_LOG(LS_INFO)
<< __func__ << " this " << this
<< " ignoring key frame request because of soon happening idle repeat";
RTC_LOG(LS_INFO) << __func__ << " this " << this
<< " not requesting refresh frame because of soon "
"happening idle repeat";
return false;
}
// Cancel the current repeat and reschedule a short repeat now. No need for a
// new refresh frame.
RTC_LOG(LS_INFO) << __func__ << " this " << this
<< " scheduling a short repeat due to key frame request";
<< " not requesting refresh frame and scheduling a short "
"repeat due to key frame request";
ScheduleRepeat(++current_frame_id_, /*idle_repeat=*/false);
return false;
}
@ -393,6 +406,15 @@ bool ZeroHertzAdapterMode::HasQualityConverged() const {
return quality_converged;
}
// RTC_RUN_ON(&sequence_checker_)
void ZeroHertzAdapterMode::ResetQualityConvergenceInfo() {
RTC_DLOG(LS_INFO) << __func__ << " this " << this;
for (auto& layer_tracker : layer_trackers_) {
if (layer_tracker.quality_converged.has_value())
layer_tracker.quality_converged = false;
}
}
// RTC_RUN_ON(&sequence_checker_)
void ZeroHertzAdapterMode::ProcessOnDelayedCadence() {
RTC_DCHECK(!queued_frames_.empty());
@ -484,6 +506,10 @@ FrameCadenceAdapterImpl::FrameCadenceAdapterImpl(Clock* clock,
zero_hertz_screenshare_enabled_(
field_trial::IsEnabled("WebRTC-ZeroHertzScreenshare")) {}
FrameCadenceAdapterImpl::~FrameCadenceAdapterImpl() {
RTC_DLOG(LS_VERBOSE) << __func__ << " this " << this;
}
void FrameCadenceAdapterImpl::Initialize(Callback* callback) {
callback_ = callback;
passthrough_adapter_.emplace(clock_, callback);
@ -539,6 +565,8 @@ void FrameCadenceAdapterImpl::OnFrame(const VideoFrame& frame) {
// This method is called on the network thread under Chromium, or other
// various contexts in test.
RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_);
RTC_DLOG(LS_VERBOSE) << "FrameCadenceAdapterImpl::" << __func__ << " this "
<< this;
// Local time in webrtc time base.
Timestamp post_time = clock_->CurrentTime();
@ -556,7 +584,7 @@ void FrameCadenceAdapterImpl::OnFrame(const VideoFrame& frame) {
void FrameCadenceAdapterImpl::OnConstraintsChanged(
const VideoTrackSourceConstraints& constraints) {
RTC_LOG(LS_INFO) << __func__ << " min_fps "
RTC_LOG(LS_INFO) << __func__ << " this " << this << " min_fps "
<< constraints.min_fps.value_or(-1) << " max_fps "
<< constraints.max_fps.value_or(-1);
queue_->PostTask(ToQueuedTask(safety_.flag(), [this, constraints] {
@ -591,10 +619,10 @@ void FrameCadenceAdapterImpl::MaybeReconfigureAdapters(
if (is_zero_hertz_enabled) {
if (!was_zero_hertz_enabled) {
zero_hertz_adapter_.emplace(queue_, clock_, callback_,
source_constraints_->max_fps.value(),
zero_hertz_params_.value());
RTC_LOG(LS_INFO) << "FrameCadenceAdapterImpl: Zero hertz mode activated.";
source_constraints_->max_fps.value());
RTC_LOG(LS_INFO) << "Zero hertz mode activated.";
}
zero_hertz_adapter_->ReconfigureParameters(zero_hertz_params_.value());
current_adapter_mode_ = &zero_hertz_adapter_.value();
} else {
if (was_zero_hertz_enabled)

37
video/frame_cadence_adapter_unittest.cc
View File

@ -457,6 +457,28 @@ TEST(FrameCadenceAdapterTest,
time_controller.AdvanceTime(kIdleFrameDelay);
}
TEST(FrameCadenceAdapterTest, LayerReconfigurationResetsConvergenceInfo) {
ZeroHertzFieldTrialEnabler enabler;
MockCallback callback;
GlobalSimulatedTimeController time_controller(Timestamp::Millis(0));
auto adapter = CreateAdapter(time_controller.GetClock());
adapter->Initialize(&callback);
adapter->SetZeroHertzModeEnabled(
FrameCadenceAdapterInterface::ZeroHertzModeParams{});
constexpr int kMaxFpsHz = 10;
constexpr TimeDelta kMinFrameDelay = TimeDelta::Millis(1000 / kMaxFpsHz);
adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, kMaxFpsHz});
time_controller.AdvanceTime(TimeDelta::Zero());
// Now setup 2 simulcast layers. The state should be unconverged.
adapter->SetZeroHertzModeEnabled(
FrameCadenceAdapterInterface::ZeroHertzModeParams{
/*num_simulcast_layers=*/2});
adapter->OnFrame(CreateFrame());
EXPECT_CALL(callback, OnFrame).Times(kMaxFpsHz);
time_controller.AdvanceTime(kMaxFpsHz * kMinFrameDelay);
}
class ZeroHertzLayerQualityConvergenceTest : public ::testing::Test {
public:
static constexpr TimeDelta kMinFrameDelay = TimeDelta::Millis(100);
@ -499,18 +521,13 @@ class ZeroHertzLayerQualityConvergenceTest : public ::testing::Test {
CreateAdapter(time_controller_.GetClock())};
};
TEST_F(ZeroHertzLayerQualityConvergenceTest, InitialStateConverged) {
// We start out assuming we're disabled in all layers, therefore converged. In
// reality we expect layer enabledness to be set up very early on
// initialization, but to cover this case we prefer being converged over being
// unconverged due to lower CPU usage.
TEST_F(ZeroHertzLayerQualityConvergenceTest, InitialStateUnconverged) {
// As the layer count is just configured, assume we start out as unconverged.
PassFrame();
ExpectFrameEntriesAtDelaysFromNow({
kMinFrameDelay, // Original frame emitted
kMinFrameDelay +
kIdleFrameDelay, // Idle repeats after convergence at 100.
kMinFrameDelay + 2 * kIdleFrameDelay, // ...
kMinFrameDelay + 3 * kIdleFrameDelay, // ...
1 * kMinFrameDelay, // Original frame emitted
2 * kMinFrameDelay, // Short repeats.
3 * kMinFrameDelay, // ...
});
}

42
video/video_stream_encoder.cc
View File

@ -822,8 +822,21 @@ void VideoStreamEncoder::ConfigureEncoder(VideoEncoderConfig config,
[this, config = std::move(config), max_data_payload_length]() mutable {
RTC_DCHECK_RUN_ON(&encoder_queue_);
RTC_DCHECK(sink_);
RTC_LOG(LS_ERROR) << "ConfigureEncoder requested. simulcast_layers = "
<< config.simulcast_layers.size();
RTC_LOG(LS_INFO) << "ConfigureEncoder requested.";
// Set up the frame cadence adapter according to if we're going to do
// screencast. The final number of spatial layers is based on info
// in `send_codec_`, which is computed based on incoming frame
// dimensions which can only be determined later.
//
// Note: zero-hertz mode isn't enabled by this alone. Constraints also
// have to be set up with min_fps = 0 and max_fps > 0.
if (config.content_type == VideoEncoderConfig::ContentType::kScreen) {
frame_cadence_adapter_->SetZeroHertzModeEnabled(
FrameCadenceAdapterInterface::ZeroHertzModeParams{});
} else {
frame_cadence_adapter_->SetZeroHertzModeEnabled(absl::nullopt);
}
pending_encoder_creation_ =
(!encoder_ || encoder_config_.video_format != config.video_format ||
@ -1256,8 +1269,6 @@ void VideoStreamEncoder::OnEncoderSettingsChanged() {
frame_cadence_adapter_->SetZeroHertzModeEnabled(
FrameCadenceAdapterInterface::ZeroHertzModeParams{
send_codec_.numberOfSimulcastStreams});
} else {
frame_cadence_adapter_->SetZeroHertzModeEnabled(absl::nullopt);
}
}
@ -1815,15 +1826,20 @@ void VideoStreamEncoder::SendKeyFrame() {
TRACE_EVENT0("webrtc", "OnKeyFrameRequest");
RTC_DCHECK(!next_frame_types_.empty());
if (!encoder_)
return; // Shutting down.
if (frame_cadence_adapter_ &&
frame_cadence_adapter_->ProcessKeyFrameRequest()) {
worker_queue_->PostTask(ToQueuedTask(task_safety_, [this] {
RTC_DCHECK_RUN_ON(worker_queue_);
video_source_sink_controller_.RequestRefreshFrame();
}));
if (frame_cadence_adapter_) {
if (frame_cadence_adapter_->ProcessKeyFrameRequest()) {
RTC_DLOG(LS_INFO) << __func__ << " RequestRefreshFrame().";
worker_queue_->PostTask(ToQueuedTask(task_safety_, [this] {
RTC_DCHECK_RUN_ON(worker_queue_);
video_source_sink_controller_.RequestRefreshFrame();
}));
} else {
RTC_DLOG(LS_INFO) << __func__ << " No RequestRefreshFrame().";
}
}
if (!encoder_) {
RTC_DLOG(LS_INFO) << __func__ << " no encoder.";
return; // Shutting down, or not configured yet.
}
// TODO(webrtc:10615): Map keyframe request to spatial layer.

18
video/video_stream_encoder_unittest.cc
View File

@ -72,7 +72,6 @@ namespace webrtc {
using ::testing::_;
using ::testing::AllOf;
using ::testing::AtLeast;
using ::testing::Eq;
using ::testing::Field;
using ::testing::Ge;
@ -83,7 +82,6 @@ using ::testing::Lt;
using ::testing::Matcher;
using ::testing::Mock;
using ::testing::NiceMock;
using ::testing::Not;
using ::testing::Optional;
using ::testing::Return;
using ::testing::SizeIs;
@ -8781,15 +8779,29 @@ TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) {
auto video_stream_encoder =
factory.Create(std::move(adapter), &encoder_queue);
EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Not(Eq(absl::nullopt))));
// First a call before we know the frame size and hence cannot compute the
// number of simulcast layers.
EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
&FrameCadenceAdapterInterface::
ZeroHertzModeParams::num_simulcast_layers,
Eq(0)))));
VideoEncoderConfig config;
test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
config.content_type = VideoEncoderConfig::ContentType::kScreen;
video_stream_encoder->ConfigureEncoder(std::move(config), 0);
factory.DepleteTaskQueues();
// Then a call as we've computed the number of simulcast layers after a passed
// frame.
EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
&FrameCadenceAdapterInterface::
ZeroHertzModeParams::num_simulcast_layers,
Gt(0)))));
PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
factory.DepleteTaskQueues();
Mock::VerifyAndClearExpectations(adapter_ptr);
// Expect a disabled zero-hertz mode after passing realtime video.
EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Eq(absl::nullopt)));
VideoEncoderConfig config2;
test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config2);