eaaae9e91b
TBR=kwiberg@webrtc.org BUG=webrtc:7634 Review-Url: https://codereview.webrtc.org/2974613003 Cr-Commit-Position: refs/heads/master@{#18926}
766 lines
29 KiB
Plaintext
766 lines
29 KiB
Plaintext
/*
|
|
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*
|
|
*/
|
|
|
|
#include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.h"
|
|
|
|
#include <memory>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#if defined(WEBRTC_IOS)
|
|
#import "Common/RTCUIApplicationStatusObserver.h"
|
|
#import "WebRTC/UIDevice+RTCDevice.h"
|
|
#endif
|
|
#import "WebRTC/RTCVideoFrameBuffer.h"
|
|
#include "libyuv/convert_from.h"
|
|
#include "webrtc/common_video/h264/profile_level_id.h"
|
|
#include "webrtc/rtc_base/checks.h"
|
|
#include "webrtc/rtc_base/logging.h"
|
|
#include "webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h"
|
|
#include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/nalu_rewriter.h"
|
|
#include "webrtc/system_wrappers/include/clock.h"
|
|
|
|
namespace internal {
|
|
|
|
// The ratio between kVTCompressionPropertyKey_DataRateLimits and
|
|
// kVTCompressionPropertyKey_AverageBitRate. The data rate limit is set higher
|
|
// than the average bit rate to avoid undershooting the target.
|
|
const float kLimitToAverageBitRateFactor = 1.5f;
|
|
// These thresholds deviate from the default h264 QP thresholds, as they
|
|
// have been found to work better on devices that support VideoToolbox
|
|
const int kLowH264QpThreshold = 28;
|
|
const int kHighH264QpThreshold = 39;
|
|
|
|
// Convenience function for creating a dictionary.
|
|
inline CFDictionaryRef CreateCFDictionary(CFTypeRef* keys,
|
|
CFTypeRef* values,
|
|
size_t size) {
|
|
return CFDictionaryCreate(kCFAllocatorDefault, keys, values, size,
|
|
&kCFTypeDictionaryKeyCallBacks,
|
|
&kCFTypeDictionaryValueCallBacks);
|
|
}
|
|
|
|
// Copies characters from a CFStringRef into a std::string.
|
|
std::string CFStringToString(const CFStringRef cf_string) {
|
|
RTC_DCHECK(cf_string);
|
|
std::string std_string;
|
|
// Get the size needed for UTF8 plus terminating character.
|
|
size_t buffer_size =
|
|
CFStringGetMaximumSizeForEncoding(CFStringGetLength(cf_string),
|
|
kCFStringEncodingUTF8) +
|
|
1;
|
|
std::unique_ptr<char[]> buffer(new char[buffer_size]);
|
|
if (CFStringGetCString(cf_string, buffer.get(), buffer_size,
|
|
kCFStringEncodingUTF8)) {
|
|
// Copy over the characters.
|
|
std_string.assign(buffer.get());
|
|
}
|
|
return std_string;
|
|
}
|
|
|
|
// Convenience function for setting a VT property.
|
|
void SetVTSessionProperty(VTSessionRef session,
|
|
CFStringRef key,
|
|
int32_t value) {
|
|
CFNumberRef cfNum =
|
|
CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &value);
|
|
OSStatus status = VTSessionSetProperty(session, key, cfNum);
|
|
CFRelease(cfNum);
|
|
if (status != noErr) {
|
|
std::string key_string = CFStringToString(key);
|
|
LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string
|
|
<< " to " << value << ": " << status;
|
|
}
|
|
}
|
|
|
|
// Convenience function for setting a VT property.
|
|
void SetVTSessionProperty(VTSessionRef session,
|
|
CFStringRef key,
|
|
uint32_t value) {
|
|
int64_t value_64 = value;
|
|
CFNumberRef cfNum =
|
|
CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &value_64);
|
|
OSStatus status = VTSessionSetProperty(session, key, cfNum);
|
|
CFRelease(cfNum);
|
|
if (status != noErr) {
|
|
std::string key_string = CFStringToString(key);
|
|
LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string
|
|
<< " to " << value << ": " << status;
|
|
}
|
|
}
|
|
|
|
// Convenience function for setting a VT property.
|
|
void SetVTSessionProperty(VTSessionRef session, CFStringRef key, bool value) {
|
|
CFBooleanRef cf_bool = (value) ? kCFBooleanTrue : kCFBooleanFalse;
|
|
OSStatus status = VTSessionSetProperty(session, key, cf_bool);
|
|
if (status != noErr) {
|
|
std::string key_string = CFStringToString(key);
|
|
LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string
|
|
<< " to " << value << ": " << status;
|
|
}
|
|
}
|
|
|
|
// Convenience function for setting a VT property.
|
|
void SetVTSessionProperty(VTSessionRef session,
|
|
CFStringRef key,
|
|
CFStringRef value) {
|
|
OSStatus status = VTSessionSetProperty(session, key, value);
|
|
if (status != noErr) {
|
|
std::string key_string = CFStringToString(key);
|
|
std::string val_string = CFStringToString(value);
|
|
LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string
|
|
<< " to " << val_string << ": " << status;
|
|
}
|
|
}
|
|
|
|
// Struct that we pass to the encoder per frame to encode. We receive it again
|
|
// in the encoder callback.
|
|
struct FrameEncodeParams {
|
|
FrameEncodeParams(webrtc::H264VideoToolboxEncoder* e,
|
|
const webrtc::CodecSpecificInfo* csi,
|
|
int32_t w,
|
|
int32_t h,
|
|
int64_t rtms,
|
|
uint32_t ts,
|
|
webrtc::VideoRotation r)
|
|
: encoder(e),
|
|
width(w),
|
|
height(h),
|
|
render_time_ms(rtms),
|
|
timestamp(ts),
|
|
rotation(r) {
|
|
if (csi) {
|
|
codec_specific_info = *csi;
|
|
} else {
|
|
codec_specific_info.codecType = webrtc::kVideoCodecH264;
|
|
}
|
|
}
|
|
|
|
webrtc::H264VideoToolboxEncoder* encoder;
|
|
webrtc::CodecSpecificInfo codec_specific_info;
|
|
int32_t width;
|
|
int32_t height;
|
|
int64_t render_time_ms;
|
|
uint32_t timestamp;
|
|
webrtc::VideoRotation rotation;
|
|
};
|
|
|
|
// We receive I420Frames as input, but we need to feed CVPixelBuffers into the
|
|
// encoder. This performs the copy and format conversion.
|
|
// TODO(tkchin): See if encoder will accept i420 frames and compare performance.
|
|
bool CopyVideoFrameToPixelBuffer(const rtc::scoped_refptr<webrtc::I420BufferInterface>& frame,
|
|
CVPixelBufferRef pixel_buffer) {
|
|
RTC_DCHECK(pixel_buffer);
|
|
RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(pixel_buffer),
|
|
kCVPixelFormatType_420YpCbCr8BiPlanarFullRange);
|
|
RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixel_buffer, 0),
|
|
static_cast<size_t>(frame->height()));
|
|
RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixel_buffer, 0),
|
|
static_cast<size_t>(frame->width()));
|
|
|
|
CVReturn cvRet = CVPixelBufferLockBaseAddress(pixel_buffer, 0);
|
|
if (cvRet != kCVReturnSuccess) {
|
|
LOG(LS_ERROR) << "Failed to lock base address: " << cvRet;
|
|
return false;
|
|
}
|
|
uint8_t* dst_y = reinterpret_cast<uint8_t*>(
|
|
CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 0));
|
|
int dst_stride_y = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 0);
|
|
uint8_t* dst_uv = reinterpret_cast<uint8_t*>(
|
|
CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 1));
|
|
int dst_stride_uv = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 1);
|
|
// Convert I420 to NV12.
|
|
int ret = libyuv::I420ToNV12(
|
|
frame->DataY(), frame->StrideY(),
|
|
frame->DataU(), frame->StrideU(),
|
|
frame->DataV(), frame->StrideV(),
|
|
dst_y, dst_stride_y, dst_uv, dst_stride_uv,
|
|
frame->width(), frame->height());
|
|
CVPixelBufferUnlockBaseAddress(pixel_buffer, 0);
|
|
if (ret) {
|
|
LOG(LS_ERROR) << "Error converting I420 VideoFrame to NV12 :" << ret;
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
CVPixelBufferRef CreatePixelBuffer(CVPixelBufferPoolRef pixel_buffer_pool) {
|
|
if (!pixel_buffer_pool) {
|
|
LOG(LS_ERROR) << "Failed to get pixel buffer pool.";
|
|
return nullptr;
|
|
}
|
|
CVPixelBufferRef pixel_buffer;
|
|
CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(nullptr, pixel_buffer_pool,
|
|
&pixel_buffer);
|
|
if (ret != kCVReturnSuccess) {
|
|
LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret;
|
|
// We probably want to drop frames here, since failure probably means
|
|
// that the pool is empty.
|
|
return nullptr;
|
|
}
|
|
return pixel_buffer;
|
|
}
|
|
|
|
// This is the callback function that VideoToolbox calls when encode is
|
|
// complete. From inspection this happens on its own queue.
|
|
void VTCompressionOutputCallback(void* encoder,
|
|
void* params,
|
|
OSStatus status,
|
|
VTEncodeInfoFlags info_flags,
|
|
CMSampleBufferRef sample_buffer) {
|
|
std::unique_ptr<FrameEncodeParams> encode_params(
|
|
reinterpret_cast<FrameEncodeParams*>(params));
|
|
encode_params->encoder->OnEncodedFrame(
|
|
status, info_flags, sample_buffer, encode_params->codec_specific_info,
|
|
encode_params->width, encode_params->height,
|
|
encode_params->render_time_ms, encode_params->timestamp,
|
|
encode_params->rotation);
|
|
}
|
|
|
|
// Extract VideoToolbox profile out of the cricket::VideoCodec. If there is no
|
|
// specific VideoToolbox profile for the specified level, AutoLevel will be
|
|
// returned. The user must initialize the encoder with a resolution and
|
|
// framerate conforming to the selected H264 level regardless.
|
|
CFStringRef ExtractProfile(const cricket::VideoCodec& codec) {
|
|
const rtc::Optional<webrtc::H264::ProfileLevelId> profile_level_id =
|
|
webrtc::H264::ParseSdpProfileLevelId(codec.params);
|
|
RTC_DCHECK(profile_level_id);
|
|
switch (profile_level_id->profile) {
|
|
case webrtc::H264::kProfileConstrainedBaseline:
|
|
case webrtc::H264::kProfileBaseline:
|
|
switch (profile_level_id->level) {
|
|
case webrtc::H264::kLevel3:
|
|
return kVTProfileLevel_H264_Baseline_3_0;
|
|
case webrtc::H264::kLevel3_1:
|
|
return kVTProfileLevel_H264_Baseline_3_1;
|
|
case webrtc::H264::kLevel3_2:
|
|
return kVTProfileLevel_H264_Baseline_3_2;
|
|
case webrtc::H264::kLevel4:
|
|
return kVTProfileLevel_H264_Baseline_4_0;
|
|
case webrtc::H264::kLevel4_1:
|
|
return kVTProfileLevel_H264_Baseline_4_1;
|
|
case webrtc::H264::kLevel4_2:
|
|
return kVTProfileLevel_H264_Baseline_4_2;
|
|
case webrtc::H264::kLevel5:
|
|
return kVTProfileLevel_H264_Baseline_5_0;
|
|
case webrtc::H264::kLevel5_1:
|
|
return kVTProfileLevel_H264_Baseline_5_1;
|
|
case webrtc::H264::kLevel5_2:
|
|
return kVTProfileLevel_H264_Baseline_5_2;
|
|
case webrtc::H264::kLevel1:
|
|
case webrtc::H264::kLevel1_b:
|
|
case webrtc::H264::kLevel1_1:
|
|
case webrtc::H264::kLevel1_2:
|
|
case webrtc::H264::kLevel1_3:
|
|
case webrtc::H264::kLevel2:
|
|
case webrtc::H264::kLevel2_1:
|
|
case webrtc::H264::kLevel2_2:
|
|
return kVTProfileLevel_H264_Baseline_AutoLevel;
|
|
}
|
|
|
|
case webrtc::H264::kProfileMain:
|
|
switch (profile_level_id->level) {
|
|
case webrtc::H264::kLevel3:
|
|
return kVTProfileLevel_H264_Main_3_0;
|
|
case webrtc::H264::kLevel3_1:
|
|
return kVTProfileLevel_H264_Main_3_1;
|
|
case webrtc::H264::kLevel3_2:
|
|
return kVTProfileLevel_H264_Main_3_2;
|
|
case webrtc::H264::kLevel4:
|
|
return kVTProfileLevel_H264_Main_4_0;
|
|
case webrtc::H264::kLevel4_1:
|
|
return kVTProfileLevel_H264_Main_4_1;
|
|
case webrtc::H264::kLevel4_2:
|
|
return kVTProfileLevel_H264_Main_4_2;
|
|
case webrtc::H264::kLevel5:
|
|
return kVTProfileLevel_H264_Main_5_0;
|
|
case webrtc::H264::kLevel5_1:
|
|
return kVTProfileLevel_H264_Main_5_1;
|
|
case webrtc::H264::kLevel5_2:
|
|
return kVTProfileLevel_H264_Main_5_2;
|
|
case webrtc::H264::kLevel1:
|
|
case webrtc::H264::kLevel1_b:
|
|
case webrtc::H264::kLevel1_1:
|
|
case webrtc::H264::kLevel1_2:
|
|
case webrtc::H264::kLevel1_3:
|
|
case webrtc::H264::kLevel2:
|
|
case webrtc::H264::kLevel2_1:
|
|
case webrtc::H264::kLevel2_2:
|
|
return kVTProfileLevel_H264_Main_AutoLevel;
|
|
}
|
|
|
|
case webrtc::H264::kProfileConstrainedHigh:
|
|
case webrtc::H264::kProfileHigh:
|
|
switch (profile_level_id->level) {
|
|
case webrtc::H264::kLevel3:
|
|
return kVTProfileLevel_H264_High_3_0;
|
|
case webrtc::H264::kLevel3_1:
|
|
return kVTProfileLevel_H264_High_3_1;
|
|
case webrtc::H264::kLevel3_2:
|
|
return kVTProfileLevel_H264_High_3_2;
|
|
case webrtc::H264::kLevel4:
|
|
return kVTProfileLevel_H264_High_4_0;
|
|
case webrtc::H264::kLevel4_1:
|
|
return kVTProfileLevel_H264_High_4_1;
|
|
case webrtc::H264::kLevel4_2:
|
|
return kVTProfileLevel_H264_High_4_2;
|
|
case webrtc::H264::kLevel5:
|
|
return kVTProfileLevel_H264_High_5_0;
|
|
case webrtc::H264::kLevel5_1:
|
|
return kVTProfileLevel_H264_High_5_1;
|
|
case webrtc::H264::kLevel5_2:
|
|
return kVTProfileLevel_H264_High_5_2;
|
|
case webrtc::H264::kLevel1:
|
|
case webrtc::H264::kLevel1_b:
|
|
case webrtc::H264::kLevel1_1:
|
|
case webrtc::H264::kLevel1_2:
|
|
case webrtc::H264::kLevel1_3:
|
|
case webrtc::H264::kLevel2:
|
|
case webrtc::H264::kLevel2_1:
|
|
case webrtc::H264::kLevel2_2:
|
|
return kVTProfileLevel_H264_High_AutoLevel;
|
|
}
|
|
}
|
|
}
|
|
|
|
} // namespace internal
|
|
|
|
namespace webrtc {
|
|
|
|
// .5 is set as a mininum to prevent overcompensating for large temporary
|
|
// overshoots. We don't want to degrade video quality too badly.
|
|
// .95 is set to prevent oscillations. When a lower bitrate is set on the
|
|
// encoder than previously set, its output seems to have a brief period of
|
|
// drastically reduced bitrate, so we want to avoid that. In steady state
|
|
// conditions, 0.95 seems to give us better overall bitrate over long periods
|
|
// of time.
|
|
H264VideoToolboxEncoder::H264VideoToolboxEncoder(const cricket::VideoCodec& codec)
|
|
: callback_(nullptr),
|
|
compression_session_(nullptr),
|
|
bitrate_adjuster_(Clock::GetRealTimeClock(), .5, .95),
|
|
packetization_mode_(H264PacketizationMode::NonInterleaved),
|
|
profile_(internal::ExtractProfile(codec)) {
|
|
LOG(LS_INFO) << "Using profile " << internal::CFStringToString(profile_);
|
|
RTC_CHECK(cricket::CodecNamesEq(codec.name, cricket::kH264CodecName));
|
|
}
|
|
|
|
H264VideoToolboxEncoder::~H264VideoToolboxEncoder() {
|
|
DestroyCompressionSession();
|
|
}
|
|
|
|
int H264VideoToolboxEncoder::InitEncode(const VideoCodec* codec_settings,
|
|
int number_of_cores,
|
|
size_t max_payload_size) {
|
|
RTC_DCHECK(codec_settings);
|
|
RTC_DCHECK_EQ(codec_settings->codecType, kVideoCodecH264);
|
|
|
|
width_ = codec_settings->width;
|
|
height_ = codec_settings->height;
|
|
mode_ = codec_settings->mode;
|
|
// We can only set average bitrate on the HW encoder.
|
|
target_bitrate_bps_ = codec_settings->startBitrate;
|
|
bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_);
|
|
|
|
// TODO(tkchin): Try setting payload size via
|
|
// kVTCompressionPropertyKey_MaxH264SliceBytes.
|
|
|
|
return ResetCompressionSession();
|
|
}
|
|
|
|
int H264VideoToolboxEncoder::Encode(
|
|
const VideoFrame& frame,
|
|
const CodecSpecificInfo* codec_specific_info,
|
|
const std::vector<FrameType>* frame_types) {
|
|
// |input_frame| size should always match codec settings.
|
|
RTC_DCHECK_EQ(frame.width(), width_);
|
|
RTC_DCHECK_EQ(frame.height(), height_);
|
|
if (!callback_ || !compression_session_) {
|
|
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
|
|
}
|
|
#if defined(WEBRTC_IOS)
|
|
if (![[RTCUIApplicationStatusObserver sharedInstance] isApplicationActive]) {
|
|
// Ignore all encode requests when app isn't active. In this state, the
|
|
// hardware encoder has been invalidated by the OS.
|
|
return WEBRTC_VIDEO_CODEC_OK;
|
|
}
|
|
#endif
|
|
bool is_keyframe_required = false;
|
|
|
|
// Get a pixel buffer from the pool and copy frame data over.
|
|
CVPixelBufferPoolRef pixel_buffer_pool =
|
|
VTCompressionSessionGetPixelBufferPool(compression_session_);
|
|
#if defined(WEBRTC_IOS)
|
|
if (!pixel_buffer_pool) {
|
|
// Kind of a hack. On backgrounding, the compression session seems to get
|
|
// invalidated, which causes this pool call to fail when the application
|
|
// is foregrounded and frames are being sent for encoding again.
|
|
// Resetting the session when this happens fixes the issue.
|
|
// In addition we request a keyframe so video can recover quickly.
|
|
ResetCompressionSession();
|
|
pixel_buffer_pool =
|
|
VTCompressionSessionGetPixelBufferPool(compression_session_);
|
|
is_keyframe_required = true;
|
|
LOG(LS_INFO) << "Resetting compression session due to invalid pool.";
|
|
}
|
|
#endif
|
|
|
|
CVPixelBufferRef pixel_buffer = nullptr;
|
|
if (frame.video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative) {
|
|
// Native frame.
|
|
rtc::scoped_refptr<ObjCFrameBuffer> objc_frame_buffer(
|
|
static_cast<ObjCFrameBuffer*>(frame.video_frame_buffer().get()));
|
|
id<RTCVideoFrameBuffer> wrapped_frame_buffer =
|
|
(id<RTCVideoFrameBuffer>)objc_frame_buffer->wrapped_frame_buffer();
|
|
|
|
if ([wrapped_frame_buffer isKindOfClass:[RTCCVPixelBuffer class]]) {
|
|
RTCCVPixelBuffer* rtc_pixel_buffer = (RTCCVPixelBuffer*)wrapped_frame_buffer;
|
|
if (![rtc_pixel_buffer requiresCropping]) {
|
|
// This pixel buffer might have a higher resolution than what the
|
|
// compression session is configured to. The compression session can
|
|
// handle that and will output encoded frames in the configured
|
|
// resolution regardless of the input pixel buffer resolution.
|
|
pixel_buffer = rtc_pixel_buffer.pixelBuffer;
|
|
CVBufferRetain(pixel_buffer);
|
|
} else {
|
|
// Cropping required, we need to crop and scale to a new pixel buffer.
|
|
pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool);
|
|
if (!pixel_buffer) {
|
|
return WEBRTC_VIDEO_CODEC_ERROR;
|
|
}
|
|
int dst_width = CVPixelBufferGetWidth(pixel_buffer);
|
|
int dst_height = CVPixelBufferGetHeight(pixel_buffer);
|
|
if ([rtc_pixel_buffer requiresScalingToWidth:dst_width height:dst_height]) {
|
|
int size =
|
|
[rtc_pixel_buffer bufferSizeForCroppingAndScalingToWidth:dst_width height:dst_height];
|
|
nv12_scale_buffer_.resize(size);
|
|
} else {
|
|
nv12_scale_buffer_.clear();
|
|
}
|
|
nv12_scale_buffer_.shrink_to_fit();
|
|
if (![rtc_pixel_buffer cropAndScaleTo:pixel_buffer
|
|
withTempBuffer:nv12_scale_buffer_.data()]) {
|
|
return WEBRTC_VIDEO_CODEC_ERROR;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!pixel_buffer) {
|
|
// We did not have a native frame, or the ObjCVideoFrame wrapped a non-native frame
|
|
pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool);
|
|
if (!pixel_buffer) {
|
|
return WEBRTC_VIDEO_CODEC_ERROR;
|
|
}
|
|
RTC_DCHECK(pixel_buffer);
|
|
if (!internal::CopyVideoFrameToPixelBuffer(frame.video_frame_buffer()->ToI420(),
|
|
pixel_buffer)) {
|
|
LOG(LS_ERROR) << "Failed to copy frame data.";
|
|
CVBufferRelease(pixel_buffer);
|
|
return WEBRTC_VIDEO_CODEC_ERROR;
|
|
}
|
|
}
|
|
|
|
// Check if we need a keyframe.
|
|
if (!is_keyframe_required && frame_types) {
|
|
for (auto frame_type : *frame_types) {
|
|
if (frame_type == kVideoFrameKey) {
|
|
is_keyframe_required = true;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
CMTime presentation_time_stamp =
|
|
CMTimeMake(frame.render_time_ms(), 1000);
|
|
CFDictionaryRef frame_properties = nullptr;
|
|
if (is_keyframe_required) {
|
|
CFTypeRef keys[] = {kVTEncodeFrameOptionKey_ForceKeyFrame};
|
|
CFTypeRef values[] = {kCFBooleanTrue};
|
|
frame_properties = internal::CreateCFDictionary(keys, values, 1);
|
|
}
|
|
std::unique_ptr<internal::FrameEncodeParams> encode_params;
|
|
encode_params.reset(new internal::FrameEncodeParams(
|
|
this, codec_specific_info, width_, height_, frame.render_time_ms(),
|
|
frame.timestamp(), frame.rotation()));
|
|
|
|
encode_params->codec_specific_info.codecSpecific.H264.packetization_mode =
|
|
packetization_mode_;
|
|
|
|
// Update the bitrate if needed.
|
|
SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps());
|
|
|
|
OSStatus status = VTCompressionSessionEncodeFrame(
|
|
compression_session_, pixel_buffer, presentation_time_stamp,
|
|
kCMTimeInvalid, frame_properties, encode_params.release(), nullptr);
|
|
if (frame_properties) {
|
|
CFRelease(frame_properties);
|
|
}
|
|
if (pixel_buffer) {
|
|
CVBufferRelease(pixel_buffer);
|
|
}
|
|
if (status != noErr) {
|
|
LOG(LS_ERROR) << "Failed to encode frame with code: " << status;
|
|
return WEBRTC_VIDEO_CODEC_ERROR;
|
|
}
|
|
return WEBRTC_VIDEO_CODEC_OK;
|
|
}
|
|
|
|
int H264VideoToolboxEncoder::RegisterEncodeCompleteCallback(
|
|
EncodedImageCallback* callback) {
|
|
callback_ = callback;
|
|
return WEBRTC_VIDEO_CODEC_OK;
|
|
}
|
|
|
|
int H264VideoToolboxEncoder::SetChannelParameters(uint32_t packet_loss,
|
|
int64_t rtt) {
|
|
// Encoder doesn't know anything about packet loss or rtt so just return.
|
|
return WEBRTC_VIDEO_CODEC_OK;
|
|
}
|
|
|
|
int H264VideoToolboxEncoder::SetRates(uint32_t new_bitrate_kbit,
|
|
uint32_t frame_rate) {
|
|
target_bitrate_bps_ = 1000 * new_bitrate_kbit;
|
|
bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_);
|
|
SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps());
|
|
return WEBRTC_VIDEO_CODEC_OK;
|
|
}
|
|
|
|
int H264VideoToolboxEncoder::Release() {
|
|
// Need to reset so that the session is invalidated and won't use the
|
|
// callback anymore. Do not remove callback until the session is invalidated
|
|
// since async encoder callbacks can occur until invalidation.
|
|
int ret = ResetCompressionSession();
|
|
callback_ = nullptr;
|
|
return ret;
|
|
}
|
|
|
|
int H264VideoToolboxEncoder::ResetCompressionSession() {
|
|
DestroyCompressionSession();
|
|
|
|
// Set source image buffer attributes. These attributes will be present on
|
|
// buffers retrieved from the encoder's pixel buffer pool.
|
|
const size_t attributes_size = 3;
|
|
CFTypeRef keys[attributes_size] = {
|
|
#if defined(WEBRTC_IOS)
|
|
kCVPixelBufferOpenGLESCompatibilityKey,
|
|
#elif defined(WEBRTC_MAC)
|
|
kCVPixelBufferOpenGLCompatibilityKey,
|
|
#endif
|
|
kCVPixelBufferIOSurfacePropertiesKey,
|
|
kCVPixelBufferPixelFormatTypeKey
|
|
};
|
|
CFDictionaryRef io_surface_value =
|
|
internal::CreateCFDictionary(nullptr, nullptr, 0);
|
|
int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
|
|
CFNumberRef pixel_format =
|
|
CFNumberCreate(nullptr, kCFNumberLongType, &nv12type);
|
|
CFTypeRef values[attributes_size] = {kCFBooleanTrue, io_surface_value,
|
|
pixel_format};
|
|
CFDictionaryRef source_attributes =
|
|
internal::CreateCFDictionary(keys, values, attributes_size);
|
|
if (io_surface_value) {
|
|
CFRelease(io_surface_value);
|
|
io_surface_value = nullptr;
|
|
}
|
|
if (pixel_format) {
|
|
CFRelease(pixel_format);
|
|
pixel_format = nullptr;
|
|
}
|
|
OSStatus status = VTCompressionSessionCreate(
|
|
nullptr, // use default allocator
|
|
width_, height_, kCMVideoCodecType_H264,
|
|
nullptr, // use default encoder
|
|
source_attributes,
|
|
nullptr, // use default compressed data allocator
|
|
internal::VTCompressionOutputCallback, this, &compression_session_);
|
|
if (source_attributes) {
|
|
CFRelease(source_attributes);
|
|
source_attributes = nullptr;
|
|
}
|
|
if (status != noErr) {
|
|
LOG(LS_ERROR) << "Failed to create compression session: " << status;
|
|
return WEBRTC_VIDEO_CODEC_ERROR;
|
|
}
|
|
ConfigureCompressionSession();
|
|
return WEBRTC_VIDEO_CODEC_OK;
|
|
}
|
|
|
|
void H264VideoToolboxEncoder::ConfigureCompressionSession() {
|
|
RTC_DCHECK(compression_session_);
|
|
internal::SetVTSessionProperty(compression_session_,
|
|
kVTCompressionPropertyKey_RealTime, true);
|
|
internal::SetVTSessionProperty(compression_session_,
|
|
kVTCompressionPropertyKey_ProfileLevel,
|
|
profile_);
|
|
internal::SetVTSessionProperty(compression_session_,
|
|
kVTCompressionPropertyKey_AllowFrameReordering,
|
|
false);
|
|
SetEncoderBitrateBps(target_bitrate_bps_);
|
|
// TODO(tkchin): Look at entropy mode and colorspace matrices.
|
|
// TODO(tkchin): Investigate to see if there's any way to make this work.
|
|
// May need it to interop with Android. Currently this call just fails.
|
|
// On inspecting encoder output on iOS8, this value is set to 6.
|
|
// internal::SetVTSessionProperty(compression_session_,
|
|
// kVTCompressionPropertyKey_MaxFrameDelayCount,
|
|
// 1);
|
|
|
|
// Set a relatively large value for keyframe emission (7200 frames or
|
|
// 4 minutes).
|
|
internal::SetVTSessionProperty(
|
|
compression_session_,
|
|
kVTCompressionPropertyKey_MaxKeyFrameInterval, 7200);
|
|
internal::SetVTSessionProperty(
|
|
compression_session_,
|
|
kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, 240);
|
|
}
|
|
|
|
void H264VideoToolboxEncoder::DestroyCompressionSession() {
|
|
if (compression_session_) {
|
|
VTCompressionSessionInvalidate(compression_session_);
|
|
CFRelease(compression_session_);
|
|
compression_session_ = nullptr;
|
|
}
|
|
}
|
|
|
|
const char* H264VideoToolboxEncoder::ImplementationName() const {
|
|
return "VideoToolbox";
|
|
}
|
|
|
|
bool H264VideoToolboxEncoder::SupportsNativeHandle() const {
|
|
return true;
|
|
}
|
|
|
|
void H264VideoToolboxEncoder::SetBitrateBps(uint32_t bitrate_bps) {
|
|
if (encoder_bitrate_bps_ != bitrate_bps) {
|
|
SetEncoderBitrateBps(bitrate_bps);
|
|
}
|
|
}
|
|
|
|
void H264VideoToolboxEncoder::SetEncoderBitrateBps(uint32_t bitrate_bps) {
|
|
if (compression_session_) {
|
|
internal::SetVTSessionProperty(compression_session_,
|
|
kVTCompressionPropertyKey_AverageBitRate,
|
|
bitrate_bps);
|
|
|
|
// TODO(tkchin): Add a helper method to set array value.
|
|
int64_t data_limit_bytes_per_second_value = static_cast<int64_t>(
|
|
bitrate_bps * internal::kLimitToAverageBitRateFactor / 8);
|
|
CFNumberRef bytes_per_second =
|
|
CFNumberCreate(kCFAllocatorDefault,
|
|
kCFNumberSInt64Type,
|
|
&data_limit_bytes_per_second_value);
|
|
int64_t one_second_value = 1;
|
|
CFNumberRef one_second =
|
|
CFNumberCreate(kCFAllocatorDefault,
|
|
kCFNumberSInt64Type,
|
|
&one_second_value);
|
|
const void* nums[2] = { bytes_per_second, one_second };
|
|
CFArrayRef data_rate_limits =
|
|
CFArrayCreate(nullptr, nums, 2, &kCFTypeArrayCallBacks);
|
|
OSStatus status =
|
|
VTSessionSetProperty(compression_session_,
|
|
kVTCompressionPropertyKey_DataRateLimits,
|
|
data_rate_limits);
|
|
if (bytes_per_second) {
|
|
CFRelease(bytes_per_second);
|
|
}
|
|
if (one_second) {
|
|
CFRelease(one_second);
|
|
}
|
|
if (data_rate_limits) {
|
|
CFRelease(data_rate_limits);
|
|
}
|
|
if (status != noErr) {
|
|
LOG(LS_ERROR) << "Failed to set data rate limit";
|
|
}
|
|
|
|
encoder_bitrate_bps_ = bitrate_bps;
|
|
}
|
|
}
|
|
|
|
void H264VideoToolboxEncoder::OnEncodedFrame(
|
|
OSStatus status,
|
|
VTEncodeInfoFlags info_flags,
|
|
CMSampleBufferRef sample_buffer,
|
|
CodecSpecificInfo codec_specific_info,
|
|
int32_t width,
|
|
int32_t height,
|
|
int64_t render_time_ms,
|
|
uint32_t timestamp,
|
|
VideoRotation rotation) {
|
|
if (status != noErr) {
|
|
LOG(LS_ERROR) << "H264 encode failed.";
|
|
return;
|
|
}
|
|
if (info_flags & kVTEncodeInfo_FrameDropped) {
|
|
LOG(LS_INFO) << "H264 encode dropped frame.";
|
|
return;
|
|
}
|
|
|
|
bool is_keyframe = false;
|
|
CFArrayRef attachments =
|
|
CMSampleBufferGetSampleAttachmentsArray(sample_buffer, 0);
|
|
if (attachments != nullptr && CFArrayGetCount(attachments)) {
|
|
CFDictionaryRef attachment =
|
|
static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0));
|
|
is_keyframe =
|
|
!CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync);
|
|
}
|
|
|
|
if (is_keyframe) {
|
|
LOG(LS_INFO) << "Generated keyframe";
|
|
}
|
|
|
|
// Convert the sample buffer into a buffer suitable for RTP packetization.
|
|
// TODO(tkchin): Allocate buffers through a pool.
|
|
std::unique_ptr<rtc::Buffer> buffer(new rtc::Buffer());
|
|
std::unique_ptr<webrtc::RTPFragmentationHeader> header;
|
|
{
|
|
webrtc::RTPFragmentationHeader* header_raw;
|
|
bool result = H264CMSampleBufferToAnnexBBuffer(sample_buffer, is_keyframe,
|
|
buffer.get(), &header_raw);
|
|
header.reset(header_raw);
|
|
if (!result) {
|
|
return;
|
|
}
|
|
}
|
|
webrtc::EncodedImage frame(buffer->data(), buffer->size(), buffer->size());
|
|
frame._encodedWidth = width;
|
|
frame._encodedHeight = height;
|
|
frame._completeFrame = true;
|
|
frame._frameType =
|
|
is_keyframe ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta;
|
|
frame.capture_time_ms_ = render_time_ms;
|
|
frame._timeStamp = timestamp;
|
|
frame.rotation_ = rotation;
|
|
frame.content_type_ =
|
|
(mode_ == kScreensharing) ? VideoContentType::SCREENSHARE : VideoContentType::UNSPECIFIED;
|
|
frame.timing_.is_timing_frame = false;
|
|
|
|
h264_bitstream_parser_.ParseBitstream(buffer->data(), buffer->size());
|
|
h264_bitstream_parser_.GetLastSliceQp(&frame.qp_);
|
|
|
|
EncodedImageCallback::Result res =
|
|
callback_->OnEncodedImage(frame, &codec_specific_info, header.get());
|
|
if (res.error != EncodedImageCallback::Result::OK) {
|
|
LOG(LS_ERROR) << "Encode callback failed: " << res.error;
|
|
return;
|
|
}
|
|
bitrate_adjuster_.Update(frame._length);
|
|
}
|
|
|
|
VideoEncoder::ScalingSettings H264VideoToolboxEncoder::GetScalingSettings()
|
|
const {
|
|
return VideoEncoder::ScalingSettings(true, internal::kLowH264QpThreshold,
|
|
internal::kHighH264QpThreshold);
|
|
}
|
|
} // namespace webrtc
|