2016-05-19 12:19:35 +02:00
|
|
|
/*
|
|
|
|
|
* Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
|
|
|
|
|
*
|
|
|
|
|
* Use of this source code is governed by a BSD-style license
|
|
|
|
|
* that can be found in the LICENSE file in the root of the source
|
|
|
|
|
* tree. An additional intellectual property rights grant can be found
|
|
|
|
|
* in the file PATENTS. All contributing project authors may
|
|
|
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
|
|
|
*/
|
|
|
|
|
|
2017-09-15 06:47:31 +02:00
|
|
|
#include "modules/video_coding/frame_buffer2.h"
|
2016-05-19 12:19:35 +02:00
|
|
|
|
|
|
|
|
#include <algorithm>
|
2018-11-28 16:47:49 +01:00
|
|
|
#include <cstdlib>
|
|
|
|
|
#include <iterator>
|
2021-11-10 11:41:14 +01:00
|
|
|
#include <memory>
|
2016-09-28 10:23:49 +02:00
|
|
|
#include <queue>
|
2018-11-28 16:47:49 +01:00
|
|
|
#include <utility>
|
2018-06-11 13:10:14 +02:00
|
|
|
#include <vector>
|
2016-05-19 12:19:35 +02:00
|
|
|
|
2021-12-15 15:05:15 +01:00
|
|
|
#include "absl/container/inlined_vector.h"
|
2022-03-07 13:21:51 +01:00
|
|
|
#include "api/units/data_size.h"
|
|
|
|
|
#include "api/units/time_delta.h"
|
2018-11-28 16:47:49 +01:00
|
|
|
#include "api/video/encoded_image.h"
|
|
|
|
|
#include "api/video/video_timing.h"
|
2021-12-15 15:05:15 +01:00
|
|
|
#include "modules/video_coding/frame_helpers.h"
|
2017-09-15 06:47:31 +02:00
|
|
|
#include "modules/video_coding/include/video_coding_defines.h"
|
|
|
|
|
#include "modules/video_coding/timing.h"
|
2022-05-25 10:12:42 +02:00
|
|
|
#include "modules/video_coding/timing/jitter_estimator.h"
|
2017-09-15 06:47:31 +02:00
|
|
|
#include "rtc_base/checks.h"
|
2018-11-28 16:47:49 +01:00
|
|
|
#include "rtc_base/experiments/rtt_mult_experiment.h"
|
2017-09-15 06:47:31 +02:00
|
|
|
#include "rtc_base/logging.h"
|
2018-11-28 16:47:49 +01:00
|
|
|
#include "rtc_base/numerics/sequence_number_util.h"
|
2017-09-15 06:47:31 +02:00
|
|
|
#include "rtc_base/trace_event.h"
|
|
|
|
|
#include "system_wrappers/include/clock.h"
|
2016-05-19 12:19:35 +02:00
|
|
|
|
|
|
|
|
namespace webrtc {
|
|
|
|
|
namespace video_coding {
|
|
|
|
|
|
|
|
|
|
namespace {
|
2016-09-28 10:23:49 +02:00
|
|
|
// Max number of frames the buffer will hold.
|
2019-01-14 13:24:22 +01:00
|
|
|
constexpr size_t kMaxFramesBuffered = 800;
|
2016-05-19 12:19:35 +02:00
|
|
|
|
2021-09-28 21:31:46 +02:00
|
|
|
// Default value for the maximum decode queue size that is used when the
|
|
|
|
|
// low-latency renderer is used.
|
|
|
|
|
constexpr size_t kZeroPlayoutDelayDefaultMaxDecodeQueueSize = 8;
|
|
|
|
|
|
2016-09-28 10:23:49 +02:00
|
|
|
// Max number of decoded frame info that will be saved.
|
2019-01-14 13:24:22 +01:00
|
|
|
constexpr int kMaxFramesHistory = 1 << 13;
|
2017-07-24 08:26:53 -07:00
|
|
|
|
2018-02-27 15:49:47 +01:00
|
|
|
// The time it's allowed for a frame to be late to its rendering prediction and
|
|
|
|
|
// still be rendered.
|
2018-02-28 09:59:26 +01:00
|
|
|
constexpr int kMaxAllowedFrameDelayMs = 5;
|
2018-02-27 15:49:47 +01:00
|
|
|
|
2017-07-24 08:26:53 -07:00
|
|
|
constexpr int64_t kLogNonDecodedIntervalMs = 5000;
|
2016-05-19 12:19:35 +02:00
|
|
|
} // namespace
|
|
|
|
|
|
|
|
|
|
FrameBuffer::FrameBuffer(Clock* clock,
|
2017-02-22 05:30:39 -08:00
|
|
|
VCMTiming* timing,
|
2022-03-25 12:43:14 +01:00
|
|
|
VCMReceiveStatisticsCallback* stats_callback,
|
2022-03-29 11:04:48 +02:00
|
|
|
const FieldTrialsView& field_trials)
|
2019-01-14 13:24:22 +01:00
|
|
|
: decoded_frames_history_(kMaxFramesHistory),
|
|
|
|
|
clock_(clock),
|
2019-04-11 12:39:34 +02:00
|
|
|
callback_queue_(nullptr),
|
2022-03-25 12:43:14 +01:00
|
|
|
jitter_estimator_(clock, field_trials),
|
2016-05-19 12:19:35 +02:00
|
|
|
timing_(timing),
|
2017-03-15 08:10:08 -07:00
|
|
|
stopped_(false),
|
2017-02-22 05:30:39 -08:00
|
|
|
protection_mode_(kProtectionNack),
|
2017-07-24 08:26:53 -07:00
|
|
|
stats_callback_(stats_callback),
|
2019-01-14 18:56:14 +01:00
|
|
|
last_log_non_decoded_ms_(-kLogNonDecodedIntervalMs),
|
2021-08-10 16:56:12 +02:00
|
|
|
rtt_mult_settings_(RttMultExperiment::GetRttMultValue()),
|
2021-09-28 21:31:46 +02:00
|
|
|
zero_playout_delay_max_decode_queue_size_(
|
|
|
|
|
"max_decode_queue_size",
|
|
|
|
|
kZeroPlayoutDelayDefaultMaxDecodeQueueSize) {
|
2021-08-10 16:56:12 +02:00
|
|
|
ParseFieldTrial({&zero_playout_delay_max_decode_queue_size_},
|
2022-03-25 12:43:14 +01:00
|
|
|
field_trials.Lookup("WebRTC-ZeroPlayoutDelay"));
|
2020-05-19 23:27:29 +02:00
|
|
|
callback_checker_.Detach();
|
|
|
|
|
}
|
2016-11-28 08:49:07 -08:00
|
|
|
|
2020-05-19 23:27:29 +02:00
|
|
|
FrameBuffer::~FrameBuffer() {
|
|
|
|
|
RTC_DCHECK_RUN_ON(&construction_checker_);
|
|
|
|
|
}
|
2016-05-19 12:19:35 +02:00
|
|
|
|
2021-12-07 14:11:45 +01:00
|
|
|
void FrameBuffer::NextFrame(int64_t max_wait_time_ms,
|
|
|
|
|
bool keyframe_required,
|
|
|
|
|
rtc::TaskQueue* callback_queue,
|
|
|
|
|
NextFrameCallback handler) {
|
2020-05-19 23:27:29 +02:00
|
|
|
RTC_DCHECK_RUN_ON(&callback_checker_);
|
|
|
|
|
RTC_DCHECK(callback_queue->IsCurrent());
|
2019-04-11 12:39:34 +02:00
|
|
|
TRACE_EVENT0("webrtc", "FrameBuffer::NextFrame");
|
|
|
|
|
int64_t latest_return_time_ms =
|
|
|
|
|
clock_->TimeInMilliseconds() + max_wait_time_ms;
|
2020-05-19 23:27:29 +02:00
|
|
|
|
2020-07-07 12:17:12 +02:00
|
|
|
MutexLock lock(&mutex_);
|
2019-04-11 12:39:34 +02:00
|
|
|
if (stopped_) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
latest_return_time_ms_ = latest_return_time_ms;
|
|
|
|
|
keyframe_required_ = keyframe_required;
|
|
|
|
|
frame_handler_ = handler;
|
|
|
|
|
callback_queue_ = callback_queue;
|
|
|
|
|
StartWaitForNextFrameOnQueue();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void FrameBuffer::StartWaitForNextFrameOnQueue() {
|
|
|
|
|
RTC_DCHECK(callback_queue_);
|
|
|
|
|
RTC_DCHECK(!callback_task_.Running());
|
2022-03-02 15:13:55 +01:00
|
|
|
int64_t wait_ms = FindNextFrame(clock_->CurrentTime());
|
2019-04-11 12:39:34 +02:00
|
|
|
callback_task_ = RepeatingTaskHandle::DelayedStart(
|
2022-01-25 08:15:50 +01:00
|
|
|
callback_queue_->Get(), TimeDelta::Millis(wait_ms),
|
|
|
|
|
[this] {
|
2020-05-19 23:27:29 +02:00
|
|
|
RTC_DCHECK_RUN_ON(&callback_checker_);
|
2019-04-11 12:39:34 +02:00
|
|
|
// If this task has not been cancelled, we did not get any new frames
|
|
|
|
|
// while waiting. Continue with frame delivery.
|
2020-12-14 00:08:11 +01:00
|
|
|
std::unique_ptr<EncodedFrame> frame;
|
2021-12-07 14:11:45 +01:00
|
|
|
NextFrameCallback frame_handler;
|
2020-12-14 00:08:11 +01:00
|
|
|
{
|
|
|
|
|
MutexLock lock(&mutex_);
|
|
|
|
|
if (!frames_to_decode_.empty()) {
|
|
|
|
|
// We have frames, deliver!
|
2021-11-10 11:41:14 +01:00
|
|
|
frame = GetNextFrame();
|
2022-03-02 15:13:55 +01:00
|
|
|
timing_->SetLastDecodeScheduledTimestamp(clock_->CurrentTime());
|
2020-12-14 00:08:11 +01:00
|
|
|
} else if (clock_->TimeInMilliseconds() < latest_return_time_ms_) {
|
|
|
|
|
// If there's no frames to decode and there is still time left, it
|
|
|
|
|
// means that the frame buffer was cleared between creation and
|
|
|
|
|
// execution of this task. Continue waiting for the remaining time.
|
2022-03-02 15:13:55 +01:00
|
|
|
int64_t wait_ms = FindNextFrame(clock_->CurrentTime());
|
2020-12-14 00:08:11 +01:00
|
|
|
return TimeDelta::Millis(wait_ms);
|
|
|
|
|
}
|
|
|
|
|
frame_handler = std::move(frame_handler_);
|
2019-04-11 12:39:34 +02:00
|
|
|
CancelCallback();
|
|
|
|
|
}
|
2020-12-14 00:08:11 +01:00
|
|
|
// Deliver frame, if any. Otherwise signal timeout.
|
2021-12-07 14:11:45 +01:00
|
|
|
frame_handler(std::move(frame));
|
2020-12-14 00:08:11 +01:00
|
|
|
return TimeDelta::Zero(); // Ignored.
|
2022-01-25 08:15:50 +01:00
|
|
|
},
|
|
|
|
|
TaskQueueBase::DelayPrecision::kHigh);
|
2019-04-11 12:39:34 +02:00
|
|
|
}
|
|
|
|
|
|
2022-03-02 15:13:55 +01:00
|
|
|
int64_t FrameBuffer::FindNextFrame(Timestamp now) {
|
|
|
|
|
int64_t wait_ms = latest_return_time_ms_ - now.ms();
|
2019-04-04 13:01:39 +02:00
|
|
|
frames_to_decode_.clear();
|
|
|
|
|
|
2021-08-09 13:02:57 +02:00
|
|
|
// `last_continuous_frame_` may be empty below, but nullopt is smaller
|
2019-04-04 13:01:39 +02:00
|
|
|
// than everything else and loop will immediately terminate as expected.
|
|
|
|
|
for (auto frame_it = frames_.begin();
|
|
|
|
|
frame_it != frames_.end() && frame_it->first <= last_continuous_frame_;
|
|
|
|
|
++frame_it) {
|
|
|
|
|
if (!frame_it->second.continuous ||
|
|
|
|
|
frame_it->second.num_missing_decodable > 0) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
2019-01-18 13:33:38 +01:00
|
|
|
|
2019-04-04 13:01:39 +02:00
|
|
|
EncodedFrame* frame = frame_it->second.frame.get();
|
2019-01-18 13:33:38 +01:00
|
|
|
|
2019-04-04 13:01:39 +02:00
|
|
|
if (keyframe_required_ && !frame->is_keyframe())
|
|
|
|
|
continue;
|
2019-01-18 13:33:38 +01:00
|
|
|
|
2019-04-04 13:01:39 +02:00
|
|
|
auto last_decoded_frame_timestamp =
|
|
|
|
|
decoded_frames_history_.GetLastDecodedFrameTimestamp();
|
2019-01-18 13:33:38 +01:00
|
|
|
|
2019-04-04 13:01:39 +02:00
|
|
|
// TODO(https://bugs.webrtc.org/9974): consider removing this check
|
|
|
|
|
// as it may make a stream undecodable after a very long delay between
|
|
|
|
|
// frames.
|
|
|
|
|
if (last_decoded_frame_timestamp &&
|
|
|
|
|
AheadOf(*last_decoded_frame_timestamp, frame->Timestamp())) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Gather all remaining frames for the same superframe.
|
|
|
|
|
std::vector<FrameMap::iterator> current_superframe;
|
|
|
|
|
current_superframe.push_back(frame_it);
|
|
|
|
|
bool last_layer_completed = frame_it->second.frame->is_last_spatial_layer;
|
|
|
|
|
FrameMap::iterator next_frame_it = frame_it;
|
2020-11-20 17:49:24 +01:00
|
|
|
while (!last_layer_completed) {
|
2019-04-04 13:01:39 +02:00
|
|
|
++next_frame_it;
|
2020-11-20 17:49:24 +01:00
|
|
|
|
|
|
|
|
if (next_frame_it == frames_.end() || !next_frame_it->second.frame) {
|
2019-04-04 13:01:39 +02:00
|
|
|
break;
|
|
|
|
|
}
|
2020-11-20 17:49:24 +01:00
|
|
|
|
|
|
|
|
if (next_frame_it->second.frame->Timestamp() != frame->Timestamp() ||
|
|
|
|
|
!next_frame_it->second.continuous) {
|
2019-04-04 13:01:39 +02:00
|
|
|
break;
|
2019-04-03 10:27:36 +00:00
|
|
|
}
|
2020-11-20 17:49:24 +01:00
|
|
|
|
|
|
|
|
if (next_frame_it->second.num_missing_decodable > 0) {
|
2020-12-10 10:49:20 +01:00
|
|
|
bool has_inter_layer_dependency = false;
|
|
|
|
|
for (size_t i = 0; i < EncodedFrame::kMaxFrameReferences &&
|
2020-11-20 17:49:24 +01:00
|
|
|
i < next_frame_it->second.frame->num_references;
|
|
|
|
|
++i) {
|
2021-02-15 13:31:29 +01:00
|
|
|
if (next_frame_it->second.frame->references[i] >= frame_it->first) {
|
2020-11-20 17:49:24 +01:00
|
|
|
has_inter_layer_dependency = true;
|
2020-12-10 10:49:20 +01:00
|
|
|
break;
|
2020-11-20 17:49:24 +01:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// If the frame has an undecoded dependency that is not within the same
|
2020-12-10 10:49:20 +01:00
|
|
|
// temporal unit then this frame is not yet ready to be decoded. If it
|
2020-11-20 17:49:24 +01:00
|
|
|
// is within the same temporal unit then the not yet decoded dependency
|
|
|
|
|
// is just a lower spatial frame, which is ok.
|
|
|
|
|
if (!has_inter_layer_dependency ||
|
|
|
|
|
next_frame_it->second.num_missing_decodable > 1) {
|
|
|
|
|
break;
|
|
|
|
|
}
|
2019-04-04 13:01:39 +02:00
|
|
|
}
|
2020-11-20 17:49:24 +01:00
|
|
|
|
2019-04-04 13:01:39 +02:00
|
|
|
current_superframe.push_back(next_frame_it);
|
|
|
|
|
last_layer_completed = next_frame_it->second.frame->is_last_spatial_layer;
|
|
|
|
|
}
|
|
|
|
|
// Check if the current superframe is complete.
|
|
|
|
|
// TODO(bugs.webrtc.org/10064): consider returning all available to
|
|
|
|
|
// decode frames even if the superframe is not complete yet.
|
|
|
|
|
if (!last_layer_completed) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
frames_to_decode_ = std::move(current_superframe);
|
2016-09-28 10:23:49 +02:00
|
|
|
|
2022-03-02 15:13:55 +01:00
|
|
|
absl::optional<Timestamp> render_time = frame->RenderTimestamp();
|
|
|
|
|
if (!render_time) {
|
|
|
|
|
render_time = timing_->RenderTime(frame->Timestamp(), now);
|
|
|
|
|
frame->SetRenderTime(render_time->ms());
|
2018-12-04 15:54:52 +01:00
|
|
|
}
|
2021-08-10 16:56:12 +02:00
|
|
|
bool too_many_frames_queued =
|
|
|
|
|
frames_.size() > zero_playout_delay_max_decode_queue_size_ ? true
|
|
|
|
|
: false;
|
2022-03-02 15:13:55 +01:00
|
|
|
wait_ms =
|
|
|
|
|
timing_->MaxWaitingTime(*render_time, now, too_many_frames_queued).ms();
|
2019-04-04 13:01:39 +02:00
|
|
|
|
|
|
|
|
// This will cause the frame buffer to prefer high framerate rather
|
|
|
|
|
// than high resolution in the case of the decoder not decoding fast
|
|
|
|
|
// enough and the stream has multiple spatial and temporal layers.
|
|
|
|
|
// For multiple temporal layers it may cause non-base layer frames to be
|
|
|
|
|
// skipped if they are late.
|
|
|
|
|
if (wait_ms < -kMaxAllowedFrameDelayMs)
|
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
break;
|
|
|
|
|
}
|
2022-03-02 15:13:55 +01:00
|
|
|
wait_ms = std::min<int64_t>(wait_ms, latest_return_time_ms_ - now.ms());
|
2019-04-04 13:01:39 +02:00
|
|
|
wait_ms = std::max<int64_t>(wait_ms, 0);
|
|
|
|
|
return wait_ms;
|
|
|
|
|
}
|
|
|
|
|
|
2021-11-10 11:41:14 +01:00
|
|
|
std::unique_ptr<EncodedFrame> FrameBuffer::GetNextFrame() {
|
2020-05-19 23:27:29 +02:00
|
|
|
RTC_DCHECK_RUN_ON(&callback_checker_);
|
2022-03-02 15:13:55 +01:00
|
|
|
Timestamp now = clock_->CurrentTime();
|
2021-08-09 13:02:57 +02:00
|
|
|
// TODO(ilnik): remove `frames_out` use frames_to_decode_ directly.
|
2021-11-10 11:41:14 +01:00
|
|
|
std::vector<std::unique_ptr<EncodedFrame>> frames_out;
|
2019-04-04 13:01:39 +02:00
|
|
|
|
|
|
|
|
RTC_DCHECK(!frames_to_decode_.empty());
|
|
|
|
|
bool superframe_delayed_by_retransmission = false;
|
2022-03-07 13:21:51 +01:00
|
|
|
DataSize superframe_size = DataSize::Zero();
|
2021-11-01 10:28:31 +01:00
|
|
|
const EncodedFrame& first_frame = *frames_to_decode_[0]->second.frame;
|
2022-03-02 15:13:55 +01:00
|
|
|
absl::optional<Timestamp> render_time = first_frame.RenderTimestamp();
|
2021-11-01 10:28:31 +01:00
|
|
|
int64_t receive_time_ms = first_frame.ReceivedTime();
|
2019-04-04 13:01:39 +02:00
|
|
|
// Gracefully handle bad RTP timestamps and render time issues.
|
2022-03-02 15:13:55 +01:00
|
|
|
if (!render_time ||
|
|
|
|
|
FrameHasBadRenderTiming(*render_time, now, timing_->TargetVideoDelay())) {
|
2019-04-30 09:16:36 +02:00
|
|
|
jitter_estimator_.Reset();
|
2019-04-04 13:01:39 +02:00
|
|
|
timing_->Reset();
|
2022-03-02 15:13:55 +01:00
|
|
|
render_time = timing_->RenderTime(first_frame.Timestamp(), now);
|
2019-04-04 13:01:39 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (FrameMap::iterator& frame_it : frames_to_decode_) {
|
|
|
|
|
RTC_DCHECK(frame_it != frames_.end());
|
2021-11-10 11:41:14 +01:00
|
|
|
std::unique_ptr<EncodedFrame> frame = std::move(frame_it->second.frame);
|
2019-04-04 13:01:39 +02:00
|
|
|
|
2022-03-02 15:13:55 +01:00
|
|
|
frame->SetRenderTime(render_time->ms());
|
2019-04-04 13:01:39 +02:00
|
|
|
|
|
|
|
|
superframe_delayed_by_retransmission |= frame->delayed_by_retransmission();
|
|
|
|
|
receive_time_ms = std::max(receive_time_ms, frame->ReceivedTime());
|
2022-03-07 13:21:51 +01:00
|
|
|
superframe_size += DataSize::Bytes(frame->size());
|
2019-04-04 13:01:39 +02:00
|
|
|
|
|
|
|
|
PropagateDecodability(frame_it->second);
|
|
|
|
|
decoded_frames_history_.InsertDecoded(frame_it->first, frame->Timestamp());
|
|
|
|
|
|
|
|
|
|
// Remove decoded frame and all undecoded frames before it.
|
2019-08-26 15:04:43 +02:00
|
|
|
if (stats_callback_) {
|
2021-02-15 13:31:29 +01:00
|
|
|
unsigned int dropped_frames =
|
|
|
|
|
std::count_if(frames_.begin(), frame_it,
|
|
|
|
|
[](const std::pair<const int64_t, FrameInfo>& frame) {
|
|
|
|
|
return frame.second.frame != nullptr;
|
|
|
|
|
});
|
2019-08-26 15:04:43 +02:00
|
|
|
if (dropped_frames > 0) {
|
|
|
|
|
stats_callback_->OnDroppedFrames(dropped_frames);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2019-04-04 13:01:39 +02:00
|
|
|
frames_.erase(frames_.begin(), ++frame_it);
|
|
|
|
|
|
2021-11-10 11:41:14 +01:00
|
|
|
frames_out.emplace_back(std::move(frame));
|
2019-04-04 13:01:39 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!superframe_delayed_by_retransmission) {
|
2022-03-07 14:50:51 +01:00
|
|
|
auto frame_delay = inter_frame_delay_.CalculateDelay(
|
|
|
|
|
first_frame.Timestamp(), Timestamp::Millis(receive_time_ms));
|
2019-04-04 13:01:39 +02:00
|
|
|
|
2022-03-07 14:50:51 +01:00
|
|
|
if (frame_delay) {
|
|
|
|
|
jitter_estimator_.UpdateEstimate(*frame_delay, superframe_size);
|
2017-03-15 08:10:08 -07:00
|
|
|
}
|
2019-04-02 15:08:14 +02:00
|
|
|
|
2019-04-04 13:01:39 +02:00
|
|
|
float rtt_mult = protection_mode_ == kProtectionNackFEC ? 0.0 : 1.0;
|
2022-03-07 13:21:51 +01:00
|
|
|
absl::optional<TimeDelta> rtt_mult_add_cap_ms = absl::nullopt;
|
2019-06-07 03:55:01 -05:00
|
|
|
if (rtt_mult_settings_.has_value()) {
|
|
|
|
|
rtt_mult = rtt_mult_settings_->rtt_mult_setting;
|
2022-03-07 13:21:51 +01:00
|
|
|
rtt_mult_add_cap_ms =
|
|
|
|
|
TimeDelta::Millis(rtt_mult_settings_->rtt_mult_add_cap_ms);
|
2019-04-04 13:01:39 +02:00
|
|
|
}
|
2022-03-07 13:21:51 +01:00
|
|
|
timing_->SetJitterDelay(
|
|
|
|
|
jitter_estimator_.GetJitterEstimate(rtt_mult, rtt_mult_add_cap_ms));
|
2022-03-02 15:13:55 +01:00
|
|
|
timing_->UpdateCurrentDelay(*render_time, now);
|
2019-04-04 13:01:39 +02:00
|
|
|
} else {
|
2021-10-13 11:19:48 +02:00
|
|
|
if (RttMultExperiment::RttMultEnabled())
|
2019-04-30 09:16:36 +02:00
|
|
|
jitter_estimator_.FrameNacked();
|
2019-04-04 13:01:39 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
UpdateJitterDelay();
|
|
|
|
|
UpdateTimingFrameInfo();
|
|
|
|
|
|
|
|
|
|
if (frames_out.size() == 1) {
|
2021-11-10 11:41:14 +01:00
|
|
|
return std::move(frames_out[0]);
|
2019-04-04 13:01:39 +02:00
|
|
|
} else {
|
2021-11-10 11:41:14 +01:00
|
|
|
return CombineAndDeleteFrames(std::move(frames_out));
|
2016-05-19 12:19:35 +02:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2016-08-03 10:59:32 +02:00
|
|
|
void FrameBuffer::SetProtectionMode(VCMVideoProtection mode) {
|
2017-03-03 07:21:18 -08:00
|
|
|
TRACE_EVENT0("webrtc", "FrameBuffer::SetProtectionMode");
|
2020-07-07 12:17:12 +02:00
|
|
|
MutexLock lock(&mutex_);
|
2019-04-03 10:27:36 +00:00
|
|
|
protection_mode_ = mode;
|
2016-08-03 10:59:32 +02:00
|
|
|
}
|
|
|
|
|
|
2016-06-30 17:33:02 +02:00
|
|
|
void FrameBuffer::Stop() {
|
2017-03-03 07:21:18 -08:00
|
|
|
TRACE_EVENT0("webrtc", "FrameBuffer::Stop");
|
2020-07-07 12:17:12 +02:00
|
|
|
MutexLock lock(&mutex_);
|
2020-05-19 23:27:29 +02:00
|
|
|
if (stopped_)
|
|
|
|
|
return;
|
2019-04-03 10:27:36 +00:00
|
|
|
stopped_ = true;
|
2020-05-19 23:27:29 +02:00
|
|
|
|
2019-04-11 12:39:34 +02:00
|
|
|
CancelCallback();
|
2016-06-30 17:33:02 +02:00
|
|
|
}
|
|
|
|
|
|
2018-11-07 14:32:28 +01:00
|
|
|
void FrameBuffer::Clear() {
|
2020-07-07 12:17:12 +02:00
|
|
|
MutexLock lock(&mutex_);
|
2019-04-03 10:27:36 +00:00
|
|
|
ClearFramesAndHistory();
|
2018-11-07 14:32:28 +01:00
|
|
|
}
|
|
|
|
|
|
2020-10-26 13:54:40 +01:00
|
|
|
int FrameBuffer::Size() {
|
|
|
|
|
MutexLock lock(&mutex_);
|
|
|
|
|
return frames_.size();
|
|
|
|
|
}
|
|
|
|
|
|
2017-09-25 06:37:12 -07:00
|
|
|
void FrameBuffer::UpdateRtt(int64_t rtt_ms) {
|
2020-07-07 12:17:12 +02:00
|
|
|
MutexLock lock(&mutex_);
|
2022-03-07 13:21:51 +01:00
|
|
|
jitter_estimator_.UpdateRtt(TimeDelta::Millis(rtt_ms));
|
2017-09-25 06:37:12 -07:00
|
|
|
}
|
|
|
|
|
|
2018-02-22 14:35:06 +01:00
|
|
|
bool FrameBuffer::ValidReferences(const EncodedFrame& frame) const {
|
2017-06-15 09:06:21 -07:00
|
|
|
for (size_t i = 0; i < frame.num_references; ++i) {
|
2021-02-15 13:31:29 +01:00
|
|
|
if (frame.references[i] >= frame.Id())
|
2017-06-15 09:06:21 -07:00
|
|
|
return false;
|
2017-09-11 09:38:36 -07:00
|
|
|
|
2017-06-15 09:06:21 -07:00
|
|
|
for (size_t j = i + 1; j < frame.num_references; ++j) {
|
|
|
|
|
if (frame.references[i] == frame.references[j])
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
2019-04-11 12:39:34 +02:00
|
|
|
void FrameBuffer::CancelCallback() {
|
2020-05-19 23:27:29 +02:00
|
|
|
// Called from the callback queue or from within Stop().
|
2019-04-11 12:39:34 +02:00
|
|
|
frame_handler_ = {};
|
|
|
|
|
callback_task_.Stop();
|
|
|
|
|
callback_queue_ = nullptr;
|
2020-05-19 23:27:29 +02:00
|
|
|
callback_checker_.Detach();
|
2019-04-11 12:39:34 +02:00
|
|
|
}
|
|
|
|
|
|
2018-02-22 14:35:06 +01:00
|
|
|
int64_t FrameBuffer::InsertFrame(std::unique_ptr<EncodedFrame> frame) {
|
2017-03-03 07:21:18 -08:00
|
|
|
TRACE_EVENT0("webrtc", "FrameBuffer::InsertFrame");
|
2016-10-06 12:25:13 +02:00
|
|
|
RTC_DCHECK(frame);
|
2019-03-25 11:40:34 +01:00
|
|
|
|
2020-07-07 12:17:12 +02:00
|
|
|
MutexLock lock(&mutex_);
|
2019-03-25 11:40:34 +01:00
|
|
|
|
2021-02-15 13:31:29 +01:00
|
|
|
int64_t last_continuous_frame_id = last_continuous_frame_.value_or(-1);
|
2016-05-19 12:19:35 +02:00
|
|
|
|
2017-06-15 09:06:21 -07:00
|
|
|
if (!ValidReferences(*frame)) {
|
2021-02-15 13:31:29 +01:00
|
|
|
RTC_LOG(LS_WARNING) << "Frame " << frame->Id()
|
2021-02-11 15:25:08 +01:00
|
|
|
<< " has invalid frame references, dropping frame.";
|
2021-02-15 13:31:29 +01:00
|
|
|
return last_continuous_frame_id;
|
2016-09-28 10:23:49 +02:00
|
|
|
}
|
2016-05-19 12:19:35 +02:00
|
|
|
|
2019-01-10 15:16:47 +01:00
|
|
|
if (frames_.size() >= kMaxFramesBuffered) {
|
2018-03-02 11:06:27 +01:00
|
|
|
if (frame->is_keyframe()) {
|
2021-02-15 13:31:29 +01:00
|
|
|
RTC_LOG(LS_WARNING) << "Inserting keyframe " << frame->Id()
|
2021-02-11 15:25:08 +01:00
|
|
|
<< " but buffer is full, clearing"
|
2020-01-14 12:11:31 +01:00
|
|
|
" buffer and inserting the frame.";
|
2018-03-02 11:06:27 +01:00
|
|
|
ClearFramesAndHistory();
|
|
|
|
|
} else {
|
2021-02-15 13:31:29 +01:00
|
|
|
RTC_LOG(LS_WARNING) << "Frame " << frame->Id()
|
2021-02-11 15:25:08 +01:00
|
|
|
<< " could not be inserted due to the frame "
|
2020-01-14 12:11:31 +01:00
|
|
|
"buffer being full, dropping frame.";
|
2021-02-15 13:31:29 +01:00
|
|
|
return last_continuous_frame_id;
|
2018-03-02 11:06:27 +01:00
|
|
|
}
|
2016-09-28 10:23:49 +02:00
|
|
|
}
|
2016-05-19 12:19:35 +02:00
|
|
|
|
2019-01-14 13:24:22 +01:00
|
|
|
auto last_decoded_frame = decoded_frames_history_.GetLastDecodedFrameId();
|
|
|
|
|
auto last_decoded_frame_timestamp =
|
|
|
|
|
decoded_frames_history_.GetLastDecodedFrameTimestamp();
|
2021-02-15 13:31:29 +01:00
|
|
|
if (last_decoded_frame && frame->Id() <= *last_decoded_frame) {
|
2019-01-14 13:24:22 +01:00
|
|
|
if (AheadOf(frame->Timestamp(), *last_decoded_frame_timestamp) &&
|
Reland of quest keyframes more frequently on stream start/decoding error. (patchset #1 id:1 of https://codereview.chromium.org/2995153002/ )
Reason for revert:
iOS workaround.
Original issue's description:
> Revert of quest keyframes more frequently on stream start/decoding error. (patchset #2 id:170001 of https://codereview.webrtc.org/2996823002/ )
>
> Reason for revert:
> Causes iOS H264 calls received in the background to have increased delay before being able to decode stream from sender due to not having a keyframe.
>
> Original issue's description:
> > Reland of quest keyframes more frequently on stream start/decoding error. (patchset #1 id:1 of https://codereview.chromium.org/2994043002/ )
> >
> > Reason for revert:
> > Create fix CL.
> >
> > Original issue's description:
> > > Revert of Request keyframes more frequently on stream start/decoding error. (patchset #1 id:1 of https://codereview.webrtc.org/2993793002/ )
> > >
> > > Reason for revert:
> > > Broke downstream test that was waiting for 5 keyframes to be received within 10 seconds. Maybe the issue is that "stats_callback_->OnCompleteFrame(frame->num_references == 0, ..." was changed to "frame->is_keyframe()"?
> > >
> > > Original issue's description:
> > > > Request keyframes more frequently on stream start/decoding error.
> > > >
> > > > In this CL:
> > > > - Added FrameObject::is_keyframe() convinience function.
> > > > - Moved logic to request keyframes on decoding error from VideoReceived to
> > > > VideoReceiveStream.
> > > > - Added keyframe_required as a parameter to FrameBuffer::NextFrame.
> > > >
> > > > BUG=webrtc:8074
> > > >
> > > > Review-Url: https://codereview.webrtc.org/2993793002
> > > > Cr-Commit-Position: refs/heads/master@{#19280}
> > > > Committed: https://chromium.googlesource.com/external/webrtc/+/26b48043581735eed6e36b95fae6f5b1bcf8cfb5
> > >
> > > TBR=terelius@webrtc.org,stefan@webrtc.org,noahric@chromium.org,philipel@webrtc.org
> > > # Skipping CQ checks because original CL landed less than 1 days ago.
> > > NOPRESUBMIT=true
> > > NOTREECHECKS=true
> > > NOTRY=true
> > > BUG=webrtc:8074
> > >
> > > Review-Url: https://codereview.webrtc.org/2994043002
> > > Cr-Commit-Position: refs/heads/master@{#19295}
> > > Committed: https://chromium.googlesource.com/external/webrtc/+/77a983185f57628cd5955bd2c0a1bf71c30439bb
> >
> > TBR=terelius@webrtc.org,stefan@webrtc.org,noahric@chromium.org,deadbeef@webrtc.org
> > # Skipping CQ checks because original CL landed less than 1 days ago.
> > BUG=webrtc:8074
> >
> > Review-Url: https://codereview.webrtc.org/2996823002
> > Cr-Commit-Position: refs/heads/master@{#19324}
> > Committed: https://chromium.googlesource.com/external/webrtc/+/628ac5964e32e66083a6ab14dceac6cb2cabe345
>
> TBR=terelius@webrtc.org,stefan@webrtc.org,noahric@chromium.org,deadbeef@webrtc.org,philipel@webrtc.org
> # Not skipping CQ checks because original CL landed more than 1 days ago.
> BUG=webrtc:8074
>
> Review-Url: https://codereview.webrtc.org/2995153002
> Cr-Commit-Position: refs/heads/master@{#19392}
> Committed: https://chromium.googlesource.com/external/webrtc/+/53959fcc2ba580e7c87231708e5b4af7906f6836
TBR=terelius@webrtc.org,stefan@webrtc.org,noahric@chromium.org,deadbeef@webrtc.org,tkchin@webrtc.org
# Skipping CQ checks because original CL landed less than 1 days ago.
BUG=webrtc:8074
Review-Url: https://codereview.webrtc.org/2996153003
Cr-Commit-Position: refs/heads/master@{#19410}
2017-08-18 04:55:02 -07:00
|
|
|
frame->is_keyframe()) {
|
2021-02-15 13:31:29 +01:00
|
|
|
// If this frame has a newer timestamp but an earlier frame id then we
|
|
|
|
|
// assume there has been a jump in the frame id due to some encoder
|
2017-01-18 05:35:20 -08:00
|
|
|
// reconfiguration or some other reason. Even though this is not according
|
|
|
|
|
// to spec we can still continue to decode from this frame if it is a
|
|
|
|
|
// keyframe.
|
2017-11-09 11:09:25 +01:00
|
|
|
RTC_LOG(LS_WARNING)
|
2021-02-15 13:31:29 +01:00
|
|
|
<< "A jump in frame id was detected, clearing buffer.";
|
2017-01-18 05:35:20 -08:00
|
|
|
ClearFramesAndHistory();
|
2021-02-15 13:31:29 +01:00
|
|
|
last_continuous_frame_id = -1;
|
2017-01-18 05:35:20 -08:00
|
|
|
} else {
|
2021-02-15 13:31:29 +01:00
|
|
|
RTC_LOG(LS_WARNING) << "Frame " << frame->Id() << " inserted after frame "
|
|
|
|
|
<< *last_decoded_frame
|
2021-02-11 15:25:08 +01:00
|
|
|
<< " was handed off for decoding, dropping frame.";
|
2021-02-15 13:31:29 +01:00
|
|
|
return last_continuous_frame_id;
|
2017-01-18 05:35:20 -08:00
|
|
|
}
|
2016-09-28 10:23:49 +02:00
|
|
|
}
|
2016-05-19 12:19:35 +02:00
|
|
|
|
2017-04-20 04:04:38 -07:00
|
|
|
// Test if inserting this frame would cause the order of the frames to become
|
|
|
|
|
// ambiguous (covering more than half the interval of 2^16). This can happen
|
2021-02-15 13:31:29 +01:00
|
|
|
// when the frame id make large jumps mid stream.
|
|
|
|
|
if (!frames_.empty() && frame->Id() < frames_.begin()->first &&
|
|
|
|
|
frames_.rbegin()->first < frame->Id()) {
|
|
|
|
|
RTC_LOG(LS_WARNING) << "A jump in frame id was detected, clearing buffer.";
|
2017-04-20 04:04:38 -07:00
|
|
|
ClearFramesAndHistory();
|
2021-02-15 13:31:29 +01:00
|
|
|
last_continuous_frame_id = -1;
|
2017-04-20 04:04:38 -07:00
|
|
|
}
|
|
|
|
|
|
2021-02-15 13:31:29 +01:00
|
|
|
auto info = frames_.emplace(frame->Id(), FrameInfo()).first;
|
2016-09-28 10:23:49 +02:00
|
|
|
|
2016-10-06 12:25:13 +02:00
|
|
|
if (info->second.frame) {
|
2021-02-15 13:31:29 +01:00
|
|
|
return last_continuous_frame_id;
|
2016-09-28 10:23:49 +02:00
|
|
|
}
|
|
|
|
|
|
2016-10-06 12:25:13 +02:00
|
|
|
if (!UpdateFrameInfoWithIncomingFrame(*frame, info))
|
2021-02-15 13:31:29 +01:00
|
|
|
return last_continuous_frame_id;
|
2019-02-27 15:32:48 +01:00
|
|
|
|
2022-03-02 15:13:55 +01:00
|
|
|
// If ReceiveTime is negative then it is not a valid timestamp.
|
|
|
|
|
if (!frame->delayed_by_retransmission() && frame->ReceivedTime() >= 0)
|
|
|
|
|
timing_->IncomingTimestamp(frame->Timestamp(),
|
|
|
|
|
Timestamp::Millis(frame->ReceivedTime()));
|
2018-02-28 11:29:47 +01:00
|
|
|
|
2020-12-08 17:36:53 +01:00
|
|
|
// It can happen that a frame will be reported as fully received even if a
|
|
|
|
|
// lower spatial layer frame is missing.
|
|
|
|
|
if (stats_callback_ && frame->is_last_spatial_layer) {
|
2019-08-22 13:16:44 +02:00
|
|
|
stats_callback_->OnCompleteFrame(frame->is_keyframe(), frame->size(),
|
|
|
|
|
frame->contentType());
|
|
|
|
|
}
|
|
|
|
|
|
2016-09-28 10:23:49 +02:00
|
|
|
info->second.frame = std::move(frame);
|
|
|
|
|
|
|
|
|
|
if (info->second.num_missing_continuous == 0) {
|
|
|
|
|
info->second.continuous = true;
|
|
|
|
|
PropagateContinuity(info);
|
2021-02-15 13:31:29 +01:00
|
|
|
last_continuous_frame_id = *last_continuous_frame_;
|
2016-09-28 10:23:49 +02:00
|
|
|
|
|
|
|
|
// Since we now have new continuous frames there might be a better frame
|
2019-04-11 12:39:34 +02:00
|
|
|
// to return from NextFrame.
|
|
|
|
|
if (callback_queue_) {
|
|
|
|
|
callback_queue_->PostTask([this] {
|
2020-07-07 12:17:12 +02:00
|
|
|
MutexLock lock(&mutex_);
|
2019-04-11 12:39:34 +02:00
|
|
|
if (!callback_task_.Running())
|
|
|
|
|
return;
|
|
|
|
|
RTC_CHECK(frame_handler_);
|
|
|
|
|
callback_task_.Stop();
|
|
|
|
|
StartWaitForNextFrameOnQueue();
|
|
|
|
|
});
|
|
|
|
|
}
|
2016-09-28 10:23:49 +02:00
|
|
|
}
|
|
|
|
|
|
2021-02-15 13:31:29 +01:00
|
|
|
return last_continuous_frame_id;
|
2016-05-19 12:19:35 +02:00
|
|
|
}
|
|
|
|
|
|
2016-09-28 10:23:49 +02:00
|
|
|
void FrameBuffer::PropagateContinuity(FrameMap::iterator start) {
|
2017-03-03 07:21:18 -08:00
|
|
|
TRACE_EVENT0("webrtc", "FrameBuffer::PropagateContinuity");
|
2016-09-28 10:23:49 +02:00
|
|
|
RTC_DCHECK(start->second.continuous);
|
|
|
|
|
|
|
|
|
|
std::queue<FrameMap::iterator> continuous_frames;
|
|
|
|
|
continuous_frames.push(start);
|
|
|
|
|
|
|
|
|
|
// A simple BFS to traverse continuous frames.
|
|
|
|
|
while (!continuous_frames.empty()) {
|
|
|
|
|
auto frame = continuous_frames.front();
|
|
|
|
|
continuous_frames.pop();
|
|
|
|
|
|
2019-01-10 15:16:47 +01:00
|
|
|
if (!last_continuous_frame_ || *last_continuous_frame_ < frame->first) {
|
|
|
|
|
last_continuous_frame_ = frame->first;
|
|
|
|
|
}
|
2016-09-28 10:23:49 +02:00
|
|
|
|
|
|
|
|
// Loop through all dependent frames, and if that frame no longer has
|
|
|
|
|
// any unfulfilled dependencies then that frame is continuous as well.
|
2019-01-10 15:02:54 +01:00
|
|
|
for (size_t d = 0; d < frame->second.dependent_frames.size(); ++d) {
|
2016-09-28 10:23:49 +02:00
|
|
|
auto frame_ref = frames_.find(frame->second.dependent_frames[d]);
|
2017-06-15 09:06:21 -07:00
|
|
|
RTC_DCHECK(frame_ref != frames_.end());
|
|
|
|
|
|
|
|
|
|
// TODO(philipel): Look into why we've seen this happen.
|
|
|
|
|
if (frame_ref != frames_.end()) {
|
|
|
|
|
--frame_ref->second.num_missing_continuous;
|
|
|
|
|
if (frame_ref->second.num_missing_continuous == 0) {
|
|
|
|
|
frame_ref->second.continuous = true;
|
|
|
|
|
continuous_frames.push(frame_ref);
|
|
|
|
|
}
|
2016-09-28 10:23:49 +02:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void FrameBuffer::PropagateDecodability(const FrameInfo& info) {
|
2017-03-03 07:21:18 -08:00
|
|
|
TRACE_EVENT0("webrtc", "FrameBuffer::PropagateDecodability");
|
2019-01-10 15:02:54 +01:00
|
|
|
for (size_t d = 0; d < info.dependent_frames.size(); ++d) {
|
2016-09-28 10:23:49 +02:00
|
|
|
auto ref_info = frames_.find(info.dependent_frames[d]);
|
2016-10-06 12:25:13 +02:00
|
|
|
RTC_DCHECK(ref_info != frames_.end());
|
2017-05-14 07:23:11 -07:00
|
|
|
// TODO(philipel): Look into why we've seen this happen.
|
|
|
|
|
if (ref_info != frames_.end()) {
|
|
|
|
|
RTC_DCHECK_GT(ref_info->second.num_missing_decodable, 0U);
|
|
|
|
|
--ref_info->second.num_missing_decodable;
|
|
|
|
|
}
|
2016-09-28 10:23:49 +02:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2018-02-22 14:35:06 +01:00
|
|
|
bool FrameBuffer::UpdateFrameInfoWithIncomingFrame(const EncodedFrame& frame,
|
2016-09-28 10:23:49 +02:00
|
|
|
FrameMap::iterator info) {
|
2017-03-03 07:21:18 -08:00
|
|
|
TRACE_EVENT0("webrtc", "FrameBuffer::UpdateFrameInfoWithIncomingFrame");
|
2019-01-14 13:24:22 +01:00
|
|
|
auto last_decoded_frame = decoded_frames_history_.GetLastDecodedFrameId();
|
|
|
|
|
RTC_DCHECK(!last_decoded_frame || *last_decoded_frame < info->first);
|
2016-09-28 10:23:49 +02:00
|
|
|
|
2021-08-09 13:02:57 +02:00
|
|
|
// In this function we determine how many missing dependencies this `frame`
|
|
|
|
|
// has to become continuous/decodable. If a frame that this `frame` depend
|
2018-06-11 13:10:14 +02:00
|
|
|
// on has already been decoded then we can ignore that dependency since it has
|
|
|
|
|
// already been fulfilled.
|
|
|
|
|
//
|
2021-08-09 13:02:57 +02:00
|
|
|
// For all other frames we will register a backwards reference to this `frame`
|
|
|
|
|
// so that `num_missing_continuous` and `num_missing_decodable` can be
|
2018-06-11 13:10:14 +02:00
|
|
|
// decremented as frames become continuous/are decoded.
|
|
|
|
|
struct Dependency {
|
2021-02-15 13:31:29 +01:00
|
|
|
int64_t frame_id;
|
2018-06-11 13:10:14 +02:00
|
|
|
bool continuous;
|
|
|
|
|
};
|
|
|
|
|
std::vector<Dependency> not_yet_fulfilled_dependencies;
|
|
|
|
|
|
|
|
|
|
// Find all dependencies that have not yet been fulfilled.
|
2016-09-28 10:23:49 +02:00
|
|
|
for (size_t i = 0; i < frame.num_references; ++i) {
|
2021-08-09 13:02:57 +02:00
|
|
|
// Does `frame` depend on a frame earlier than the last decoded one?
|
2021-02-15 13:31:29 +01:00
|
|
|
if (last_decoded_frame && frame.references[i] <= *last_decoded_frame) {
|
2021-08-09 13:02:57 +02:00
|
|
|
// Was that frame decoded? If not, this `frame` will never become
|
2018-06-11 13:10:14 +02:00
|
|
|
// decodable.
|
2021-02-15 13:31:29 +01:00
|
|
|
if (!decoded_frames_history_.WasDecoded(frame.references[i])) {
|
2017-07-24 08:26:53 -07:00
|
|
|
int64_t now_ms = clock_->TimeInMilliseconds();
|
|
|
|
|
if (last_log_non_decoded_ms_ + kLogNonDecodedIntervalMs < now_ms) {
|
2017-11-09 11:09:25 +01:00
|
|
|
RTC_LOG(LS_WARNING)
|
2021-02-15 13:31:29 +01:00
|
|
|
<< "Frame " << frame.Id()
|
2021-02-11 15:25:08 +01:00
|
|
|
<< " depends on a non-decoded frame more previous than the last "
|
|
|
|
|
"decoded frame, dropping frame.";
|
2017-07-24 08:26:53 -07:00
|
|
|
last_log_non_decoded_ms_ = now_ms;
|
|
|
|
|
}
|
2016-09-28 10:23:49 +02:00
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
} else {
|
2021-02-15 13:31:29 +01:00
|
|
|
auto ref_info = frames_.find(frame.references[i]);
|
2018-06-11 13:10:14 +02:00
|
|
|
bool ref_continuous =
|
|
|
|
|
ref_info != frames_.end() && ref_info->second.continuous;
|
2021-02-15 13:31:29 +01:00
|
|
|
not_yet_fulfilled_dependencies.push_back(
|
|
|
|
|
{frame.references[i], ref_continuous});
|
2016-09-28 10:23:49 +02:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2018-06-11 13:10:14 +02:00
|
|
|
info->second.num_missing_continuous = not_yet_fulfilled_dependencies.size();
|
|
|
|
|
info->second.num_missing_decodable = not_yet_fulfilled_dependencies.size();
|
|
|
|
|
|
|
|
|
|
for (const Dependency& dep : not_yet_fulfilled_dependencies) {
|
|
|
|
|
if (dep.continuous)
|
2016-09-28 10:23:49 +02:00
|
|
|
--info->second.num_missing_continuous;
|
|
|
|
|
|
2021-02-15 13:31:29 +01:00
|
|
|
frames_[dep.frame_id].dependent_frames.push_back(frame.Id());
|
2016-05-19 12:19:35 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
2016-11-30 01:31:40 -08:00
|
|
|
void FrameBuffer::UpdateJitterDelay() {
|
2017-03-03 07:21:18 -08:00
|
|
|
TRACE_EVENT0("webrtc", "FrameBuffer::UpdateJitterDelay");
|
2017-02-22 05:30:39 -08:00
|
|
|
if (!stats_callback_)
|
|
|
|
|
return;
|
|
|
|
|
|
2022-03-22 10:55:15 +01:00
|
|
|
auto timings = timing_->GetTimings();
|
|
|
|
|
if (timings.num_decoded_frames > 0) {
|
2017-02-22 05:30:39 -08:00
|
|
|
stats_callback_->OnFrameBufferTimingsUpdated(
|
2022-03-22 10:55:15 +01:00
|
|
|
timings.max_decode_duration.ms(), timings.current_delay.ms(),
|
|
|
|
|
timings.target_delay.ms(), timings.jitter_buffer_delay.ms(),
|
|
|
|
|
timings.min_playout_delay.ms(), timings.render_delay.ms());
|
2016-11-30 01:31:40 -08:00
|
|
|
}
|
2016-11-28 08:49:07 -08:00
|
|
|
}
|
|
|
|
|
|
2017-07-06 03:06:50 -07:00
|
|
|
void FrameBuffer::UpdateTimingFrameInfo() {
|
|
|
|
|
TRACE_EVENT0("webrtc", "FrameBuffer::UpdateTimingFrameInfo");
|
2018-06-18 10:48:16 +02:00
|
|
|
absl::optional<TimingFrameInfo> info = timing_->GetTimingFrameInfo();
|
2018-03-23 10:43:21 +01:00
|
|
|
if (info && stats_callback_)
|
2017-07-06 03:06:50 -07:00
|
|
|
stats_callback_->OnTimingFrameInfoUpdated(*info);
|
|
|
|
|
}
|
|
|
|
|
|
2017-01-18 05:35:20 -08:00
|
|
|
void FrameBuffer::ClearFramesAndHistory() {
|
2017-07-06 03:06:50 -07:00
|
|
|
TRACE_EVENT0("webrtc", "FrameBuffer::ClearFramesAndHistory");
|
2019-08-26 15:04:43 +02:00
|
|
|
if (stats_callback_) {
|
2021-02-15 13:31:29 +01:00
|
|
|
unsigned int dropped_frames =
|
|
|
|
|
std::count_if(frames_.begin(), frames_.end(),
|
|
|
|
|
[](const std::pair<const int64_t, FrameInfo>& frame) {
|
|
|
|
|
return frame.second.frame != nullptr;
|
|
|
|
|
});
|
2019-08-26 15:04:43 +02:00
|
|
|
if (dropped_frames > 0) {
|
|
|
|
|
stats_callback_->OnDroppedFrames(dropped_frames);
|
|
|
|
|
}
|
|
|
|
|
}
|
2017-01-18 05:35:20 -08:00
|
|
|
frames_.clear();
|
2019-01-10 15:16:47 +01:00
|
|
|
last_continuous_frame_.reset();
|
2018-12-04 15:54:52 +01:00
|
|
|
frames_to_decode_.clear();
|
2019-01-14 13:24:22 +01:00
|
|
|
decoded_frames_history_.Clear();
|
2017-01-18 05:35:20 -08:00
|
|
|
}
|
|
|
|
|
|
2019-09-27 10:29:30 +02:00
|
|
|
// TODO(philipel): Avoid the concatenation of frames here, by replacing
|
|
|
|
|
// NextFrame and GetNextFrame with methods returning multiple frames.
|
2021-11-10 11:41:14 +01:00
|
|
|
std::unique_ptr<EncodedFrame> FrameBuffer::CombineAndDeleteFrames(
|
|
|
|
|
std::vector<std::unique_ptr<EncodedFrame>> frames) const {
|
2018-12-04 15:54:52 +01:00
|
|
|
RTC_DCHECK(!frames.empty());
|
2021-12-15 15:05:15 +01:00
|
|
|
absl::InlinedVector<std::unique_ptr<EncodedFrame>, 4> inlined;
|
|
|
|
|
for (auto& frame : frames) {
|
|
|
|
|
inlined.push_back(std::move(frame));
|
2018-12-04 15:54:52 +01:00
|
|
|
}
|
2021-12-15 15:05:15 +01:00
|
|
|
return webrtc::CombineAndDeleteFrames(std::move(inlined));
|
2018-12-04 15:54:52 +01:00
|
|
|
}
|
|
|
|
|
|
2018-03-27 08:31:45 +02:00
|
|
|
FrameBuffer::FrameInfo::FrameInfo() = default;
|
|
|
|
|
FrameBuffer::FrameInfo::FrameInfo(FrameInfo&&) = default;
|
|
|
|
|
FrameBuffer::FrameInfo::~FrameInfo() = default;
|
|
|
|
|
|
2016-05-19 12:19:35 +02:00
|
|
|
} // namespace video_coding
|
|
|
|
|
} // namespace webrtc
|