admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

VideoAdapter: Drop frames based on actual fps instead of expected fps

Browse Source 
Pass timestamps to VideoAdapter instead of setting expected input frame rate, and use that to calculate when frames should be dropped.

BUG=webrtc:4938
TEST=Enable quality slider and HUD in debug settings. Request low fps with the quality slider and observe dropped frames.

Review-Url: https://codereview.webrtc.org/1982983003
Cr-Commit-Position: refs/heads/master@{#12811}
This commit is contained in:
magjed 2016-05-19 06:05:40 -07:00 committed by Commit bot
parent 0026dd85de
commit 604abe09f1
5 changed files with 268 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
webrtc/media/base/videoadapter.cc
View File

@ -11,6 +11,7 @@
#include "webrtc/media/base/videoadapter.h"
#include <algorithm>
#include <cstdlib>
#include <limits>
#include "webrtc/base/checks.h"
@ -118,22 +119,43 @@ VideoAdapter::VideoAdapter()
adaption_changes_(0),
previous_width_(0),
previous_height_(0),
input_interval_(0),
interval_next_frame_(0),
resolution_request_max_pixel_count_(std::numeric_limits<int>::max()),
resolution_request_max_pixel_count_step_up_(0) {}
VideoAdapter::~VideoAdapter() {}
void VideoAdapter::SetExpectedInputFrameInterval(int64_t interval) {
// TODO(perkj): Consider measuring input frame rate instead.
// Frame rate typically varies depending on lighting.
bool VideoAdapter::KeepFrame(int64_t in_timestamp_ns) {
rtc::CritScope cs(&critical_section_);
input_interval_ = interval;
if (!requested_format_ || requested_format_->interval == 0)
return true;
if (next_frame_timestamp_ns_) {
// Time until next frame should be outputted.
const int64_t time_until_next_frame_ns =
(*next_frame_timestamp_ns_ - in_timestamp_ns);
// Continue if timestamp is withing expected range.
if (std::abs(time_until_next_frame_ns) < 2 * requested_format_->interval) {
// Drop if a frame shouldn't be outputted yet.
if (time_until_next_frame_ns > 0)
return false;
// Time to output new frame.
*next_frame_timestamp_ns_ += requested_format_->interval;
return true;
}
}
// First timestamp received or timestamp is way outside expected range, so
// reset. Set first timestamp target to just half the interval to prefer
// keeping frames in case of jitter.
next_frame_timestamp_ns_ =
rtc::Optional<int64_t>(in_timestamp_ns + requested_format_->interval / 2);
return true;
}
void VideoAdapter::AdaptFrameResolution(int in_width,
int in_height,
int64_t in_timestamp_ns,
int* cropped_width,
int* cropped_height,
int* out_width,
@ -150,25 +172,7 @@ void VideoAdapter::AdaptFrameResolution(int in_width,
}
// Drop the input frame if necessary.
bool should_drop = false;
if (max_pixel_count == 0) {
// Drop all frames as the output format is 0x0.
should_drop = true;
} else if (requested_format_ && requested_format_->interval > 0) {
// Drop some frames based on input fps and output fps.
// Normally output fps is less than input fps.
interval_next_frame_ += input_interval_;
if (interval_next_frame_ >= requested_format_->interval) {
interval_next_frame_ -= requested_format_->interval;
// Reset |interval_next_frame_| if it accumulates too much to avoid
// "catching up" behaviour.
if (interval_next_frame_ >= requested_format_->interval)
interval_next_frame_ = 0;
} else {
should_drop = true;
}
}
if (should_drop) {
if (max_pixel_count == 0 || !KeepFrame(in_timestamp_ns)) {
// Show VAdapt log every 90 frames dropped. (3 seconds)
if ((frames_in_ - frames_out_) % 90 == 0) {
// TODO(fbarchard): Reduce to LS_VERBOSE when adapter info is not needed
@ -179,7 +183,7 @@ void VideoAdapter::AdaptFrameResolution(int in_width,
<< " Changes: " << adaption_changes_
<< " Input: " << in_width
<< "x" << in_height
<< " i" << input_interval_
<< " timestamp: " << in_timestamp_ns
<< " Output: i"
<< (requested_format_ ? requested_format_->interval : 0);
}
@ -238,7 +242,7 @@ void VideoAdapter::AdaptFrameResolution(int in_width,
LOG(LS_INFO) << "Frame size changed: scaled " << frames_scaled_ << " / out "
<< frames_out_ << " / in " << frames_in_
<< " Changes: " << adaption_changes_ << " Input: " << in_width
<< "x" << in_height << " i" << input_interval_
<< "x" << in_height
<< " Scale: " << scale.numerator << "/" << scale.denominator
<< " Output: " << *out_width << "x" << *out_height << " i"
<< (requested_format_ ? requested_format_->interval : 0);
@ -251,7 +255,7 @@ void VideoAdapter::AdaptFrameResolution(int in_width,
void VideoAdapter::OnOutputFormatRequest(const VideoFormat& format) {
rtc::CritScope cs(&critical_section_);
requested_format_ = rtc::Optional<VideoFormat>(format);
interval_next_frame_ = 0;
next_frame_timestamp_ns_ = rtc::Optional<int64_t>();
}
void VideoAdapter::OnResolutionRequest(

13
webrtc/media/base/videoadapter.h
View File

@ -27,17 +27,13 @@ class VideoAdapter {
VideoAdapter();
virtual ~VideoAdapter();
// Sets the expected frame interval. This controls how often frames should
// be dropped if |OnOutputFormatRequest| is called with a lower frame
// interval.
void SetExpectedInputFrameInterval(int64_t interval);
// Return the adapted resolution given the input resolution. The input
// resolution should first be cropped to the specified resolution, and then
// scaled to the final output resolution. The output resolution will be 0x0 if
// the frame should be dropped.
void AdaptFrameResolution(int in_width,
int in_height,
int64_t in_timestamp_ns,
int* cropped_width,
int* cropped_height,
int* out_width,
@ -58,14 +54,17 @@ class VideoAdapter {
rtc::Optional<int> max_pixel_count_step_up);
private:
// Determine if frame should be dropped based on input fps and requested fps.
bool KeepFrame(int64_t in_timestamp_ns);
int frames_in_; // Number of input frames.
int frames_out_; // Number of output frames.
int frames_scaled_; // Number of frames scaled.
int adaption_changes_; // Number of changes in scale factor.
int previous_width_; // Previous adapter output width.
int previous_height_; // Previous adapter output height.
int input_interval_ GUARDED_BY(critical_section_);
int64_t interval_next_frame_ GUARDED_BY(critical_section_);
// The target timestamp for the next frame based on requested format.
rtc::Optional<int64_t> next_frame_timestamp_ns_ GUARDED_BY(critical_section_);
// Max number of pixels requested via calls to OnOutputFormatRequest,
// OnResolutionRequest respectively.

311
webrtc/media/base/videoadapter_unittest.cc
View File

@ -28,8 +28,7 @@ class VideoAdapterTest : public testing::Test {
virtual void SetUp() {
capturer_.reset(new FakeVideoCapturer);
capture_format_ = capturer_->GetSupportedFormats()->at(0);
capture_format_.interval = VideoFormat::FpsToInterval(50);
adapter_.SetExpectedInputFrameInterval(capture_format_.interval);
capture_format_.interval = VideoFormat::FpsToInterval(30);
listener_.reset(new VideoCapturerListener(&adapter_));
capturer_->SignalFrameCaptured.connect(
@ -73,6 +72,7 @@ class VideoAdapterTest : public testing::Test {
int out_width;
int out_height;
video_adapter_->AdaptFrameResolution(in_width, in_height,
captured_frame->time_stamp,
&cropped_width, &cropped_height,
&out_width, &out_height);
if (out_width != 0 && out_height != 0) {
@ -170,39 +170,193 @@ TEST_F(VideoAdapterTest, AdaptZeroInterval) {
// Adapt the frame rate to be half of the capture rate at the beginning. Expect
// the number of dropped frames to be half of the number the captured frames.
TEST_F(VideoAdapterTest, AdaptFramerate) {
TEST_F(VideoAdapterTest, AdaptFramerateToHalf) {
VideoFormat request_format = capture_format_;
request_format.interval *= 2;
adapter_.OnOutputFormatRequest(request_format);
EXPECT_EQ(CS_RUNNING, capturer_->Start(capture_format_));
// Capture 10 frames and verify that every other frame is dropped. The first
// frame should not be dropped.
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 1);
EXPECT_EQ(0, listener_->GetStats().dropped_frames);
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 2);
EXPECT_EQ(0, listener_->GetStats().dropped_frames);
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 3);
EXPECT_EQ(1, listener_->GetStats().dropped_frames);
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 4);
EXPECT_EQ(1, listener_->GetStats().dropped_frames);
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 5);
EXPECT_EQ(2, listener_->GetStats().dropped_frames);
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 6);
EXPECT_EQ(2, listener_->GetStats().dropped_frames);
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 7);
EXPECT_EQ(3, listener_->GetStats().dropped_frames);
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 8);
EXPECT_EQ(3, listener_->GetStats().dropped_frames);
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 9);
EXPECT_EQ(4, listener_->GetStats().dropped_frames);
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 10);
EXPECT_EQ(4, listener_->GetStats().dropped_frames);
}
// Adapt the frame rate to be two thirds of the capture rate at the beginning.
// Expect the number of dropped frames to be one thirds of the number the
// captured frames.
TEST_F(VideoAdapterTest, AdaptFramerateToTwoThirds) {
VideoFormat request_format = capture_format_;
request_format.interval = request_format.interval * 3 / 2;
adapter_.OnOutputFormatRequest(request_format);
EXPECT_EQ(CS_RUNNING, capturer_->Start(capture_format_));
// Capture 10 frames and verify that every third frame is dropped. The first
// frame should not be dropped.
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 1);
EXPECT_EQ(0, listener_->GetStats().dropped_frames);
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 2);
EXPECT_EQ(0, listener_->GetStats().dropped_frames);
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 3);
EXPECT_EQ(1, listener_->GetStats().dropped_frames);
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 4);
EXPECT_EQ(1, listener_->GetStats().dropped_frames);
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 5);
EXPECT_EQ(1, listener_->GetStats().dropped_frames);
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 6);
EXPECT_EQ(2, listener_->GetStats().dropped_frames);
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 7);
EXPECT_EQ(2, listener_->GetStats().dropped_frames);
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 8);
EXPECT_EQ(2, listener_->GetStats().dropped_frames);
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 9);
EXPECT_EQ(3, listener_->GetStats().dropped_frames);
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 10);
EXPECT_EQ(3, listener_->GetStats().dropped_frames);
}
// Request frame rate twice as high as captured frame rate. Expect no frame
// drop.
TEST_F(VideoAdapterTest, AdaptFramerateHighLimit) {
VideoFormat request_format = capture_format_;
request_format.interval /= 2;
adapter_.OnOutputFormatRequest(request_format);
EXPECT_EQ(CS_RUNNING, capturer_->Start(capture_format_));
for (int i = 0; i < 10; ++i)
capturer_->CaptureFrame();
// Verify frame drop and no resolution change.
VideoCapturerListener::Stats stats = listener_->GetStats();
EXPECT_GE(stats.captured_frames, 10);
EXPECT_EQ(stats.captured_frames / 2, stats.dropped_frames);
VerifyAdaptedResolution(stats, capture_format_.width, capture_format_.height,
capture_format_.width, capture_format_.height);
// Verify no frame drop.
EXPECT_EQ(0, listener_->GetStats().dropped_frames);
}
// Adapt the frame rate to be half of the capture rate at the beginning. Expect
// the number of dropped frames to be half of the number the captured frames.
TEST_F(VideoAdapterTest, AdaptFramerateVariable) {
VideoFormat request_format = capture_format_;
request_format.interval = request_format.interval * 3 / 2;
adapter_.OnOutputFormatRequest(request_format);
EXPECT_EQ(CS_RUNNING, capturer_->Start(capture_format_));
for (int i = 0; i < 30; ++i)
capturer_->CaptureFrame();
// After the first timestamp, add a big offset to the timestamps. Expect that
// the adapter is conservative and resets to the new offset and does not drop
// any frame.
TEST_F(VideoAdapterTest, AdaptFramerateTimestampOffset) {
const int64_t capture_interval = VideoFormat::FpsToInterval(30);
adapter_.OnOutputFormatRequest(
VideoFormat(640, 480, capture_interval, cricket::FOURCC_ANY));
// Verify frame drop and no resolution change.
VideoCapturerListener::Stats stats = listener_->GetStats();
EXPECT_GE(stats.captured_frames, 30);
// Verify 2 / 3 kept (20) and 1 / 3 dropped (10).
EXPECT_EQ(stats.captured_frames * 1 / 3, stats.dropped_frames);
VerifyAdaptedResolution(stats, capture_format_.width, capture_format_.height,
capture_format_.width, capture_format_.height);
const int64_t first_timestamp = 0;
adapter_.AdaptFrameResolution(640, 480, first_timestamp,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_GT(out_width_, 0);
EXPECT_GT(out_height_, 0);
const int64_t big_offset = -987654321LL * 1000;
const int64_t second_timestamp = big_offset;
adapter_.AdaptFrameResolution(640, 480, second_timestamp,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_GT(out_width_, 0);
EXPECT_GT(out_height_, 0);
const int64_t third_timestamp = big_offset + capture_interval;
adapter_.AdaptFrameResolution(640, 480, third_timestamp,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_GT(out_width_, 0);
EXPECT_GT(out_height_, 0);
}
// Request 30 fps and send 30 fps with jitter. Expect that no frame is dropped.
TEST_F(VideoAdapterTest, AdaptFramerateTimestampJitter) {
const int64_t capture_interval = VideoFormat::FpsToInterval(30);
adapter_.OnOutputFormatRequest(
VideoFormat(640, 480, capture_interval, cricket::FOURCC_ANY));
adapter_.AdaptFrameResolution(640, 480, capture_interval * 0 / 10,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_GT(out_width_, 0);
EXPECT_GT(out_height_, 0);
adapter_.AdaptFrameResolution(640, 480, capture_interval * 10 / 10 - 1,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_GT(out_width_, 0);
EXPECT_GT(out_height_, 0);
adapter_.AdaptFrameResolution(640, 480, capture_interval * 25 / 10,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_GT(out_width_, 0);
EXPECT_GT(out_height_, 0);
adapter_.AdaptFrameResolution(640, 480, capture_interval * 30 / 10,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_GT(out_width_, 0);
EXPECT_GT(out_height_, 0);
adapter_.AdaptFrameResolution(640, 480, capture_interval * 35 / 10,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_GT(out_width_, 0);
EXPECT_GT(out_height_, 0);
adapter_.AdaptFrameResolution(640, 480, capture_interval * 50 / 10,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_GT(out_width_, 0);
EXPECT_GT(out_height_, 0);
}
// Adapt the frame rate to be half of the capture rate after capturing no less
@ -237,7 +391,7 @@ TEST_F(VideoAdapterTest, AdaptFrameResolutionHighLimit) {
output_format.height *= 10;
adapter_.OnOutputFormatRequest(output_format);
adapter_.AdaptFrameResolution(capture_format_.width, capture_format_.height,
&cropped_width_, &cropped_height_,
0, &cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(capture_format_.width, cropped_width_);
EXPECT_EQ(capture_format_.height, cropped_height_);
@ -250,7 +404,7 @@ TEST_F(VideoAdapterTest, AdaptFrameResolutionHighLimit) {
TEST_F(VideoAdapterTest, AdaptFrameResolutionIdentical) {
adapter_.OnOutputFormatRequest(capture_format_);
adapter_.AdaptFrameResolution(capture_format_.width, capture_format_.height,
&cropped_width_, &cropped_height_,
0, &cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(capture_format_.width, cropped_width_);
EXPECT_EQ(capture_format_.height, cropped_height_);
@ -266,7 +420,7 @@ TEST_F(VideoAdapterTest, AdaptFrameResolutionQuarter) {
request_format.height /= 2;
adapter_.OnOutputFormatRequest(request_format);
adapter_.AdaptFrameResolution(capture_format_.width, capture_format_.height,
&cropped_width_, &cropped_height_,
0, &cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(capture_format_.width, cropped_width_);
EXPECT_EQ(capture_format_.height, cropped_height_);
@ -281,7 +435,7 @@ TEST_F(VideoAdapterTest, AdaptFrameResolutionDrop) {
output_format.height = 0;
adapter_.OnOutputFormatRequest(output_format);
adapter_.AdaptFrameResolution(capture_format_.width, capture_format_.height,
&cropped_width_, &cropped_height_,
0, &cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(0, out_width_);
EXPECT_EQ(0, out_height_);
@ -348,9 +502,8 @@ TEST_F(VideoAdapterTest, DropAllFrames) {
}
TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) {
VideoFormat format(640, 400, VideoFormat::FpsToInterval(30), 0);
adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30));
adapter_.AdaptFrameResolution(640, 400,
VideoFormat format(640, 400, 0, 0);
adapter_.AdaptFrameResolution(640, 400, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
@ -361,7 +514,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) {
// Format request 640x400.
format.height = 400;
adapter_.OnOutputFormatRequest(format);
adapter_.AdaptFrameResolution(640, 400,
adapter_.AdaptFrameResolution(640, 400, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
@ -374,7 +527,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) {
format.width = 1280;
format.height = 720;
adapter_.OnOutputFormatRequest(format);
adapter_.AdaptFrameResolution(640, 400,
adapter_.AdaptFrameResolution(640, 400, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
@ -386,7 +539,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) {
format.width = 0;
format.height = 0;
adapter_.OnOutputFormatRequest(format);
adapter_.AdaptFrameResolution(640, 400,
adapter_.AdaptFrameResolution(640, 400, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(0, out_width_);
@ -396,7 +549,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) {
format.width = 320;
format.height = 200;
adapter_.OnOutputFormatRequest(format);
adapter_.AdaptFrameResolution(640, 400,
adapter_.AdaptFrameResolution(640, 400, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
@ -410,7 +563,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) {
format.width = 424;
format.height = 265;
adapter_.OnOutputFormatRequest(format);
adapter_.AdaptFrameResolution(640, 400,
adapter_.AdaptFrameResolution(640, 400, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
@ -422,7 +575,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) {
format.width = 640 * 3 / 8;
format.height = 400 * 3 / 8;
adapter_.OnOutputFormatRequest(format);
adapter_.AdaptFrameResolution(640, 400,
adapter_.AdaptFrameResolution(640, 400, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
@ -434,7 +587,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) {
format.width = 320;
format.height = 200;
adapter_.OnOutputFormatRequest(format);
adapter_.AdaptFrameResolution(640, 400,
adapter_.AdaptFrameResolution(640, 400, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
@ -446,7 +599,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) {
format.width = 480;
format.height = 300;
adapter_.OnOutputFormatRequest(format);
adapter_.AdaptFrameResolution(640, 400,
adapter_.AdaptFrameResolution(640, 400, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
@ -457,9 +610,8 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) {
TEST_F(VideoAdapterTest, TestViewRequestPlusCameraSwitch) {
// Start at HD.
VideoFormat format(1280, 720, VideoFormat::FpsToInterval(30), 0);
adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30));
adapter_.AdaptFrameResolution(1280, 720,
VideoFormat format(1280, 720, 0, 0);
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
@ -471,7 +623,7 @@ TEST_F(VideoAdapterTest, TestViewRequestPlusCameraSwitch) {
format.width = 640;
format.height = 360;
adapter_.OnOutputFormatRequest(format);
adapter_.AdaptFrameResolution(1280, 720,
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
@ -481,7 +633,7 @@ TEST_F(VideoAdapterTest, TestViewRequestPlusCameraSwitch) {
// Now, the camera reopens at VGA.
// Both the frame and the output format should be 640x360.
adapter_.AdaptFrameResolution(640, 360,
adapter_.AdaptFrameResolution(640, 360, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
@ -492,7 +644,7 @@ TEST_F(VideoAdapterTest, TestViewRequestPlusCameraSwitch) {
// And another view request comes in for 640x360, which should have no
// real impact.
adapter_.OnOutputFormatRequest(format);
adapter_.AdaptFrameResolution(640, 360,
adapter_.AdaptFrameResolution(640, 360, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
@ -503,11 +655,10 @@ TEST_F(VideoAdapterTest, TestViewRequestPlusCameraSwitch) {
TEST_F(VideoAdapterTest, TestVGAWidth) {
// Reqeuested Output format is 640x360.
VideoFormat format(640, 360, VideoFormat::FpsToInterval(30), FOURCC_I420);
adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30));
VideoFormat format(640, 360, 0, FOURCC_I420);
adapter_.OnOutputFormatRequest(format);
adapter_.AdaptFrameResolution(640, 480,
adapter_.AdaptFrameResolution(640, 480, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
// Expect cropping.
@ -517,7 +668,7 @@ TEST_F(VideoAdapterTest, TestVGAWidth) {
EXPECT_EQ(360, out_height_);
// But if frames come in at 640x360, we shouldn't adapt them down.
adapter_.AdaptFrameResolution(640, 360,
adapter_.AdaptFrameResolution(640, 360, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
@ -525,7 +676,7 @@ TEST_F(VideoAdapterTest, TestVGAWidth) {
EXPECT_EQ(640, out_width_);
EXPECT_EQ(360, out_height_);
adapter_.AdaptFrameResolution(640, 480,
adapter_.AdaptFrameResolution(640, 480, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
@ -535,7 +686,7 @@ TEST_F(VideoAdapterTest, TestVGAWidth) {
}
TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) {
adapter_.AdaptFrameResolution(1280, 720,
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
@ -546,7 +697,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) {
// Adapt down one step.
adapter_.OnResolutionRequest(rtc::Optional<int>(1280 * 720 - 1),
rtc::Optional<int>());
adapter_.AdaptFrameResolution(1280, 720,
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
@ -557,7 +708,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) {
// Adapt down one step more.
adapter_.OnResolutionRequest(rtc::Optional<int>(960 * 540 - 1),
rtc::Optional<int>());
adapter_.AdaptFrameResolution(1280, 720,
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
@ -568,7 +719,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) {
// Adapt down one step more.
adapter_.OnResolutionRequest(rtc::Optional<int>(640 * 360 - 1),
rtc::Optional<int>());
adapter_.AdaptFrameResolution(1280, 720,
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
@ -579,7 +730,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) {
// Adapt up one step.
adapter_.OnResolutionRequest(rtc::Optional<int>(),
rtc::Optional<int>(480 * 270));
adapter_.AdaptFrameResolution(1280, 720,
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
@ -590,7 +741,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) {
// Adapt up one step more.
adapter_.OnResolutionRequest(rtc::Optional<int>(),
rtc::Optional<int>(640 * 360));
adapter_.AdaptFrameResolution(1280, 720,
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
@ -601,7 +752,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) {
// Adapt up one step more.
adapter_.OnResolutionRequest(rtc::Optional<int>(),
rtc::Optional<int>(960 * 720));
adapter_.AdaptFrameResolution(1280, 720,
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
@ -611,7 +762,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) {
}
TEST_F(VideoAdapterTest, TestOnResolutionRequestMaxZero) {
adapter_.AdaptFrameResolution(1280, 720,
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
@ -620,7 +771,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestMaxZero) {
EXPECT_EQ(720, out_height_);
adapter_.OnResolutionRequest(rtc::Optional<int>(0), rtc::Optional<int>());
adapter_.AdaptFrameResolution(1280, 720,
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(0, out_width_);
@ -630,7 +781,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestMaxZero) {
TEST_F(VideoAdapterTest, TestOnResolutionRequestInLargeSteps) {
adapter_.OnResolutionRequest(rtc::Optional<int>(640 * 360 - 1),
rtc::Optional<int>());
adapter_.AdaptFrameResolution(1280, 720,
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
@ -640,7 +791,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInLargeSteps) {
adapter_.OnResolutionRequest(rtc::Optional<int>(),
rtc::Optional<int>(960 * 720));
adapter_.AdaptFrameResolution(1280, 720,
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
@ -652,7 +803,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInLargeSteps) {
TEST_F(VideoAdapterTest, TestOnOutputFormatRequestCapsMaxResolution) {
adapter_.OnResolutionRequest(rtc::Optional<int>(640 * 360 - 1),
rtc::Optional<int>());
adapter_.AdaptFrameResolution(1280, 720,
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
@ -660,10 +811,9 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequestCapsMaxResolution) {
EXPECT_EQ(480, out_width_);
EXPECT_EQ(270, out_height_);
VideoFormat new_format(640, 360, VideoFormat::FpsToInterval(30), FOURCC_I420);
adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30));
VideoFormat new_format(640, 360, 0, FOURCC_I420);
adapter_.OnOutputFormatRequest(new_format);
adapter_.AdaptFrameResolution(1280, 720,
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
@ -673,7 +823,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequestCapsMaxResolution) {
adapter_.OnResolutionRequest(rtc::Optional<int>(),
rtc::Optional<int>(960 * 720));
adapter_.AdaptFrameResolution(1280, 720,
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
@ -683,7 +833,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequestCapsMaxResolution) {
}
TEST_F(VideoAdapterTest, TestOnResolutionRequestReset) {
adapter_.AdaptFrameResolution(1280, 720,
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
@ -693,7 +843,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestReset) {
adapter_.OnResolutionRequest(rtc::Optional<int>(640 * 360 - 1),
rtc::Optional<int>());
adapter_.AdaptFrameResolution(1280, 720,
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
@ -702,7 +852,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestReset) {
EXPECT_EQ(270, out_height_);
adapter_.OnResolutionRequest(rtc::Optional<int>(), rtc::Optional<int>());
adapter_.AdaptFrameResolution(1280, 720,
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(1280, cropped_width_);
@ -713,11 +863,9 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestReset) {
TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) {
// Ask for 640x360 (16:9 aspect).
adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30));
adapter_.OnOutputFormatRequest(
VideoFormat(640, 360, VideoFormat::FpsToInterval(30), FOURCC_I420));
adapter_.OnOutputFormatRequest(VideoFormat(640, 360, 0, FOURCC_I420));
// Send 640x480 (4:3 aspect).
adapter_.AdaptFrameResolution(640, 480,
adapter_.AdaptFrameResolution(640, 480, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
// Expect cropping to 16:9 format and no scaling.
@ -730,7 +878,7 @@ TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) {
adapter_.OnResolutionRequest(rtc::Optional<int>(640 * 360 - 1),
rtc::Optional<int>());
// Expect cropping to 16:9 format and 3/4 scaling.
adapter_.AdaptFrameResolution(640, 480,
adapter_.AdaptFrameResolution(640, 480, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
@ -742,7 +890,7 @@ TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) {
adapter_.OnResolutionRequest(rtc::Optional<int>(480 * 270 - 1),
rtc::Optional<int>());
// Expect cropping to 16:9 format and 1/2 scaling.
adapter_.AdaptFrameResolution(640, 480,
adapter_.AdaptFrameResolution(640, 480, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
@ -754,7 +902,7 @@ TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) {
adapter_.OnResolutionRequest(rtc::Optional<int>(),
rtc::Optional<int>(320 * 180));
// Expect cropping to 16:9 format and 3/4 scaling.
adapter_.AdaptFrameResolution(640, 480,
adapter_.AdaptFrameResolution(640, 480, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
@ -766,7 +914,7 @@ TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) {
adapter_.OnResolutionRequest(rtc::Optional<int>(),
rtc::Optional<int>(480 * 270));
// Expect cropping to 16:9 format and no scaling.
adapter_.AdaptFrameResolution(640, 480,
adapter_.AdaptFrameResolution(640, 480, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
@ -778,7 +926,7 @@ TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) {
adapter_.OnResolutionRequest(rtc::Optional<int>(),
rtc::Optional<int>(640 * 360));
// Expect cropping to 16:9 format and no scaling.
adapter_.AdaptFrameResolution(640, 480,
adapter_.AdaptFrameResolution(640, 480, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(640, cropped_width_);
@ -789,14 +937,13 @@ TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) {
TEST_F(VideoAdapterTest, TestCroppingOddResolution) {
// Ask for 640x360 (16:9 aspect), with 3/16 scaling.
adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30));
adapter_.OnOutputFormatRequest(
VideoFormat(640, 360, VideoFormat::FpsToInterval(30), FOURCC_I420));
VideoFormat(640, 360, 0, FOURCC_I420));
adapter_.OnResolutionRequest(rtc::Optional<int>(640 * 360 * 3 / 16 * 3 / 16),
rtc::Optional<int>());
// Send 640x480 (4:3 aspect).
adapter_.AdaptFrameResolution(640, 480,
adapter_.AdaptFrameResolution(640, 480, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);

1
webrtc/media/base/videocapturer.cc
View File

@ -227,6 +227,7 @@ void VideoCapturer::OnFrameCaptured(VideoCapturer*,
if (enable_video_adapter_ && !IsScreencast()) {
video_adapter_.AdaptFrameResolution(
captured_frame->width, captured_frame->height,
captured_frame->time_stamp,
&cropped_width, &cropped_height,
&out_width, &out_height);
if (out_width == 0 || out_height == 0) {

5
webrtc/media/base/videocapturer.h
View File

@ -246,11 +246,6 @@ class VideoCapturer : public sigslot::has_slots<>,
void SetCaptureFormat(const VideoFormat* format) {
capture_format_.reset(format ? new VideoFormat(*format) : NULL);
if (capture_format_) {
ASSERT(capture_format_->interval > 0 &&
"Capture format expected to have positive interval.");
video_adapter_.SetExpectedInputFrameInterval(capture_format_->interval);
}
}
void SetSupportedFormats(const std::vector<VideoFormat>& formats);