/* * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. * */ #include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.h" #include #include #include #if defined(WEBRTC_IOS) #import "Common/RTCUIApplicationStatusObserver.h" #import "WebRTC/UIDevice+RTCDevice.h" #endif #import "WebRTC/RTCVideoFrameBuffer.h" #include "libyuv/convert_from.h" #include "webrtc/common_video/h264/profile_level_id.h" #include "webrtc/rtc_base/checks.h" #include "webrtc/rtc_base/logging.h" #include "webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h" #include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/nalu_rewriter.h" #include "webrtc/system_wrappers/include/clock.h" namespace internal { // The ratio between kVTCompressionPropertyKey_DataRateLimits and // kVTCompressionPropertyKey_AverageBitRate. The data rate limit is set higher // than the average bit rate to avoid undershooting the target. const float kLimitToAverageBitRateFactor = 1.5f; // These thresholds deviate from the default h264 QP thresholds, as they // have been found to work better on devices that support VideoToolbox const int kLowH264QpThreshold = 28; const int kHighH264QpThreshold = 39; // Convenience function for creating a dictionary. inline CFDictionaryRef CreateCFDictionary(CFTypeRef* keys, CFTypeRef* values, size_t size) { return CFDictionaryCreate(kCFAllocatorDefault, keys, values, size, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); } // Copies characters from a CFStringRef into a std::string. std::string CFStringToString(const CFStringRef cf_string) { RTC_DCHECK(cf_string); std::string std_string; // Get the size needed for UTF8 plus terminating character. size_t buffer_size = CFStringGetMaximumSizeForEncoding(CFStringGetLength(cf_string), kCFStringEncodingUTF8) + 1; std::unique_ptr buffer(new char[buffer_size]); if (CFStringGetCString(cf_string, buffer.get(), buffer_size, kCFStringEncodingUTF8)) { // Copy over the characters. std_string.assign(buffer.get()); } return std_string; } // Convenience function for setting a VT property. void SetVTSessionProperty(VTSessionRef session, CFStringRef key, int32_t value) { CFNumberRef cfNum = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &value); OSStatus status = VTSessionSetProperty(session, key, cfNum); CFRelease(cfNum); if (status != noErr) { std::string key_string = CFStringToString(key); LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string << " to " << value << ": " << status; } } // Convenience function for setting a VT property. void SetVTSessionProperty(VTSessionRef session, CFStringRef key, uint32_t value) { int64_t value_64 = value; CFNumberRef cfNum = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &value_64); OSStatus status = VTSessionSetProperty(session, key, cfNum); CFRelease(cfNum); if (status != noErr) { std::string key_string = CFStringToString(key); LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string << " to " << value << ": " << status; } } // Convenience function for setting a VT property. void SetVTSessionProperty(VTSessionRef session, CFStringRef key, bool value) { CFBooleanRef cf_bool = (value) ? kCFBooleanTrue : kCFBooleanFalse; OSStatus status = VTSessionSetProperty(session, key, cf_bool); if (status != noErr) { std::string key_string = CFStringToString(key); LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string << " to " << value << ": " << status; } } // Convenience function for setting a VT property. void SetVTSessionProperty(VTSessionRef session, CFStringRef key, CFStringRef value) { OSStatus status = VTSessionSetProperty(session, key, value); if (status != noErr) { std::string key_string = CFStringToString(key); std::string val_string = CFStringToString(value); LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string << " to " << val_string << ": " << status; } } // Struct that we pass to the encoder per frame to encode. We receive it again // in the encoder callback. struct FrameEncodeParams { FrameEncodeParams(webrtc::H264VideoToolboxEncoder* e, const webrtc::CodecSpecificInfo* csi, int32_t w, int32_t h, int64_t rtms, uint32_t ts, webrtc::VideoRotation r) : encoder(e), width(w), height(h), render_time_ms(rtms), timestamp(ts), rotation(r) { if (csi) { codec_specific_info = *csi; } else { codec_specific_info.codecType = webrtc::kVideoCodecH264; } } webrtc::H264VideoToolboxEncoder* encoder; webrtc::CodecSpecificInfo codec_specific_info; int32_t width; int32_t height; int64_t render_time_ms; uint32_t timestamp; webrtc::VideoRotation rotation; }; // We receive I420Frames as input, but we need to feed CVPixelBuffers into the // encoder. This performs the copy and format conversion. // TODO(tkchin): See if encoder will accept i420 frames and compare performance. bool CopyVideoFrameToPixelBuffer(const rtc::scoped_refptr& frame, CVPixelBufferRef pixel_buffer) { RTC_DCHECK(pixel_buffer); RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(pixel_buffer), kCVPixelFormatType_420YpCbCr8BiPlanarFullRange); RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixel_buffer, 0), static_cast(frame->height())); RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixel_buffer, 0), static_cast(frame->width())); CVReturn cvRet = CVPixelBufferLockBaseAddress(pixel_buffer, 0); if (cvRet != kCVReturnSuccess) { LOG(LS_ERROR) << "Failed to lock base address: " << cvRet; return false; } uint8_t* dst_y = reinterpret_cast( CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 0)); int dst_stride_y = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 0); uint8_t* dst_uv = reinterpret_cast( CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 1)); int dst_stride_uv = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 1); // Convert I420 to NV12. int ret = libyuv::I420ToNV12( frame->DataY(), frame->StrideY(), frame->DataU(), frame->StrideU(), frame->DataV(), frame->StrideV(), dst_y, dst_stride_y, dst_uv, dst_stride_uv, frame->width(), frame->height()); CVPixelBufferUnlockBaseAddress(pixel_buffer, 0); if (ret) { LOG(LS_ERROR) << "Error converting I420 VideoFrame to NV12 :" << ret; return false; } return true; } CVPixelBufferRef CreatePixelBuffer(CVPixelBufferPoolRef pixel_buffer_pool) { if (!pixel_buffer_pool) { LOG(LS_ERROR) << "Failed to get pixel buffer pool."; return nullptr; } CVPixelBufferRef pixel_buffer; CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(nullptr, pixel_buffer_pool, &pixel_buffer); if (ret != kCVReturnSuccess) { LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret; // We probably want to drop frames here, since failure probably means // that the pool is empty. return nullptr; } return pixel_buffer; } // This is the callback function that VideoToolbox calls when encode is // complete. From inspection this happens on its own queue. void VTCompressionOutputCallback(void* encoder, void* params, OSStatus status, VTEncodeInfoFlags info_flags, CMSampleBufferRef sample_buffer) { std::unique_ptr encode_params( reinterpret_cast(params)); encode_params->encoder->OnEncodedFrame( status, info_flags, sample_buffer, encode_params->codec_specific_info, encode_params->width, encode_params->height, encode_params->render_time_ms, encode_params->timestamp, encode_params->rotation); } // Extract VideoToolbox profile out of the cricket::VideoCodec. If there is no // specific VideoToolbox profile for the specified level, AutoLevel will be // returned. The user must initialize the encoder with a resolution and // framerate conforming to the selected H264 level regardless. CFStringRef ExtractProfile(const cricket::VideoCodec& codec) { const rtc::Optional profile_level_id = webrtc::H264::ParseSdpProfileLevelId(codec.params); RTC_DCHECK(profile_level_id); switch (profile_level_id->profile) { case webrtc::H264::kProfileConstrainedBaseline: case webrtc::H264::kProfileBaseline: switch (profile_level_id->level) { case webrtc::H264::kLevel3: return kVTProfileLevel_H264_Baseline_3_0; case webrtc::H264::kLevel3_1: return kVTProfileLevel_H264_Baseline_3_1; case webrtc::H264::kLevel3_2: return kVTProfileLevel_H264_Baseline_3_2; case webrtc::H264::kLevel4: return kVTProfileLevel_H264_Baseline_4_0; case webrtc::H264::kLevel4_1: return kVTProfileLevel_H264_Baseline_4_1; case webrtc::H264::kLevel4_2: return kVTProfileLevel_H264_Baseline_4_2; case webrtc::H264::kLevel5: return kVTProfileLevel_H264_Baseline_5_0; case webrtc::H264::kLevel5_1: return kVTProfileLevel_H264_Baseline_5_1; case webrtc::H264::kLevel5_2: return kVTProfileLevel_H264_Baseline_5_2; case webrtc::H264::kLevel1: case webrtc::H264::kLevel1_b: case webrtc::H264::kLevel1_1: case webrtc::H264::kLevel1_2: case webrtc::H264::kLevel1_3: case webrtc::H264::kLevel2: case webrtc::H264::kLevel2_1: case webrtc::H264::kLevel2_2: return kVTProfileLevel_H264_Baseline_AutoLevel; } case webrtc::H264::kProfileMain: switch (profile_level_id->level) { case webrtc::H264::kLevel3: return kVTProfileLevel_H264_Main_3_0; case webrtc::H264::kLevel3_1: return kVTProfileLevel_H264_Main_3_1; case webrtc::H264::kLevel3_2: return kVTProfileLevel_H264_Main_3_2; case webrtc::H264::kLevel4: return kVTProfileLevel_H264_Main_4_0; case webrtc::H264::kLevel4_1: return kVTProfileLevel_H264_Main_4_1; case webrtc::H264::kLevel4_2: return kVTProfileLevel_H264_Main_4_2; case webrtc::H264::kLevel5: return kVTProfileLevel_H264_Main_5_0; case webrtc::H264::kLevel5_1: return kVTProfileLevel_H264_Main_5_1; case webrtc::H264::kLevel5_2: return kVTProfileLevel_H264_Main_5_2; case webrtc::H264::kLevel1: case webrtc::H264::kLevel1_b: case webrtc::H264::kLevel1_1: case webrtc::H264::kLevel1_2: case webrtc::H264::kLevel1_3: case webrtc::H264::kLevel2: case webrtc::H264::kLevel2_1: case webrtc::H264::kLevel2_2: return kVTProfileLevel_H264_Main_AutoLevel; } case webrtc::H264::kProfileConstrainedHigh: case webrtc::H264::kProfileHigh: switch (profile_level_id->level) { case webrtc::H264::kLevel3: return kVTProfileLevel_H264_High_3_0; case webrtc::H264::kLevel3_1: return kVTProfileLevel_H264_High_3_1; case webrtc::H264::kLevel3_2: return kVTProfileLevel_H264_High_3_2; case webrtc::H264::kLevel4: return kVTProfileLevel_H264_High_4_0; case webrtc::H264::kLevel4_1: return kVTProfileLevel_H264_High_4_1; case webrtc::H264::kLevel4_2: return kVTProfileLevel_H264_High_4_2; case webrtc::H264::kLevel5: return kVTProfileLevel_H264_High_5_0; case webrtc::H264::kLevel5_1: return kVTProfileLevel_H264_High_5_1; case webrtc::H264::kLevel5_2: return kVTProfileLevel_H264_High_5_2; case webrtc::H264::kLevel1: case webrtc::H264::kLevel1_b: case webrtc::H264::kLevel1_1: case webrtc::H264::kLevel1_2: case webrtc::H264::kLevel1_3: case webrtc::H264::kLevel2: case webrtc::H264::kLevel2_1: case webrtc::H264::kLevel2_2: return kVTProfileLevel_H264_High_AutoLevel; } } } } // namespace internal namespace webrtc { // .5 is set as a mininum to prevent overcompensating for large temporary // overshoots. We don't want to degrade video quality too badly. // .95 is set to prevent oscillations. When a lower bitrate is set on the // encoder than previously set, its output seems to have a brief period of // drastically reduced bitrate, so we want to avoid that. In steady state // conditions, 0.95 seems to give us better overall bitrate over long periods // of time. H264VideoToolboxEncoder::H264VideoToolboxEncoder(const cricket::VideoCodec& codec) : callback_(nullptr), compression_session_(nullptr), bitrate_adjuster_(Clock::GetRealTimeClock(), .5, .95), packetization_mode_(H264PacketizationMode::NonInterleaved), profile_(internal::ExtractProfile(codec)) { LOG(LS_INFO) << "Using profile " << internal::CFStringToString(profile_); RTC_CHECK(cricket::CodecNamesEq(codec.name, cricket::kH264CodecName)); } H264VideoToolboxEncoder::~H264VideoToolboxEncoder() { DestroyCompressionSession(); } int H264VideoToolboxEncoder::InitEncode(const VideoCodec* codec_settings, int number_of_cores, size_t max_payload_size) { RTC_DCHECK(codec_settings); RTC_DCHECK_EQ(codec_settings->codecType, kVideoCodecH264); width_ = codec_settings->width; height_ = codec_settings->height; mode_ = codec_settings->mode; // We can only set average bitrate on the HW encoder. target_bitrate_bps_ = codec_settings->startBitrate; bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_); // TODO(tkchin): Try setting payload size via // kVTCompressionPropertyKey_MaxH264SliceBytes. return ResetCompressionSession(); } int H264VideoToolboxEncoder::Encode( const VideoFrame& frame, const CodecSpecificInfo* codec_specific_info, const std::vector* frame_types) { // |input_frame| size should always match codec settings. RTC_DCHECK_EQ(frame.width(), width_); RTC_DCHECK_EQ(frame.height(), height_); if (!callback_ || !compression_session_) { return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } #if defined(WEBRTC_IOS) if (![[RTCUIApplicationStatusObserver sharedInstance] isApplicationActive]) { // Ignore all encode requests when app isn't active. In this state, the // hardware encoder has been invalidated by the OS. return WEBRTC_VIDEO_CODEC_OK; } #endif bool is_keyframe_required = false; // Get a pixel buffer from the pool and copy frame data over. CVPixelBufferPoolRef pixel_buffer_pool = VTCompressionSessionGetPixelBufferPool(compression_session_); #if defined(WEBRTC_IOS) if (!pixel_buffer_pool) { // Kind of a hack. On backgrounding, the compression session seems to get // invalidated, which causes this pool call to fail when the application // is foregrounded and frames are being sent for encoding again. // Resetting the session when this happens fixes the issue. // In addition we request a keyframe so video can recover quickly. ResetCompressionSession(); pixel_buffer_pool = VTCompressionSessionGetPixelBufferPool(compression_session_); is_keyframe_required = true; LOG(LS_INFO) << "Resetting compression session due to invalid pool."; } #endif CVPixelBufferRef pixel_buffer = nullptr; if (frame.video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative) { // Native frame. rtc::scoped_refptr objc_frame_buffer( static_cast(frame.video_frame_buffer().get())); id wrapped_frame_buffer = (id)objc_frame_buffer->wrapped_frame_buffer(); if ([wrapped_frame_buffer isKindOfClass:[RTCCVPixelBuffer class]]) { RTCCVPixelBuffer* rtc_pixel_buffer = (RTCCVPixelBuffer*)wrapped_frame_buffer; if (![rtc_pixel_buffer requiresCropping]) { // This pixel buffer might have a higher resolution than what the // compression session is configured to. The compression session can // handle that and will output encoded frames in the configured // resolution regardless of the input pixel buffer resolution. pixel_buffer = rtc_pixel_buffer.pixelBuffer; CVBufferRetain(pixel_buffer); } else { // Cropping required, we need to crop and scale to a new pixel buffer. pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool); if (!pixel_buffer) { return WEBRTC_VIDEO_CODEC_ERROR; } int dst_width = CVPixelBufferGetWidth(pixel_buffer); int dst_height = CVPixelBufferGetHeight(pixel_buffer); if ([rtc_pixel_buffer requiresScalingToWidth:dst_width height:dst_height]) { int size = [rtc_pixel_buffer bufferSizeForCroppingAndScalingToWidth:dst_width height:dst_height]; nv12_scale_buffer_.resize(size); } else { nv12_scale_buffer_.clear(); } nv12_scale_buffer_.shrink_to_fit(); if (![rtc_pixel_buffer cropAndScaleTo:pixel_buffer withTempBuffer:nv12_scale_buffer_.data()]) { return WEBRTC_VIDEO_CODEC_ERROR; } } } } if (!pixel_buffer) { // We did not have a native frame, or the ObjCVideoFrame wrapped a non-native frame pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool); if (!pixel_buffer) { return WEBRTC_VIDEO_CODEC_ERROR; } RTC_DCHECK(pixel_buffer); if (!internal::CopyVideoFrameToPixelBuffer(frame.video_frame_buffer()->ToI420(), pixel_buffer)) { LOG(LS_ERROR) << "Failed to copy frame data."; CVBufferRelease(pixel_buffer); return WEBRTC_VIDEO_CODEC_ERROR; } } // Check if we need a keyframe. if (!is_keyframe_required && frame_types) { for (auto frame_type : *frame_types) { if (frame_type == kVideoFrameKey) { is_keyframe_required = true; break; } } } CMTime presentation_time_stamp = CMTimeMake(frame.render_time_ms(), 1000); CFDictionaryRef frame_properties = nullptr; if (is_keyframe_required) { CFTypeRef keys[] = {kVTEncodeFrameOptionKey_ForceKeyFrame}; CFTypeRef values[] = {kCFBooleanTrue}; frame_properties = internal::CreateCFDictionary(keys, values, 1); } std::unique_ptr encode_params; encode_params.reset(new internal::FrameEncodeParams( this, codec_specific_info, width_, height_, frame.render_time_ms(), frame.timestamp(), frame.rotation())); encode_params->codec_specific_info.codecSpecific.H264.packetization_mode = packetization_mode_; // Update the bitrate if needed. SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps()); OSStatus status = VTCompressionSessionEncodeFrame( compression_session_, pixel_buffer, presentation_time_stamp, kCMTimeInvalid, frame_properties, encode_params.release(), nullptr); if (frame_properties) { CFRelease(frame_properties); } if (pixel_buffer) { CVBufferRelease(pixel_buffer); } if (status != noErr) { LOG(LS_ERROR) << "Failed to encode frame with code: " << status; return WEBRTC_VIDEO_CODEC_ERROR; } return WEBRTC_VIDEO_CODEC_OK; } int H264VideoToolboxEncoder::RegisterEncodeCompleteCallback( EncodedImageCallback* callback) { callback_ = callback; return WEBRTC_VIDEO_CODEC_OK; } int H264VideoToolboxEncoder::SetChannelParameters(uint32_t packet_loss, int64_t rtt) { // Encoder doesn't know anything about packet loss or rtt so just return. return WEBRTC_VIDEO_CODEC_OK; } int H264VideoToolboxEncoder::SetRates(uint32_t new_bitrate_kbit, uint32_t frame_rate) { target_bitrate_bps_ = 1000 * new_bitrate_kbit; bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_); SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps()); return WEBRTC_VIDEO_CODEC_OK; } int H264VideoToolboxEncoder::Release() { // Need to reset so that the session is invalidated and won't use the // callback anymore. Do not remove callback until the session is invalidated // since async encoder callbacks can occur until invalidation. int ret = ResetCompressionSession(); callback_ = nullptr; return ret; } int H264VideoToolboxEncoder::ResetCompressionSession() { DestroyCompressionSession(); // Set source image buffer attributes. These attributes will be present on // buffers retrieved from the encoder's pixel buffer pool. const size_t attributes_size = 3; CFTypeRef keys[attributes_size] = { #if defined(WEBRTC_IOS) kCVPixelBufferOpenGLESCompatibilityKey, #elif defined(WEBRTC_MAC) kCVPixelBufferOpenGLCompatibilityKey, #endif kCVPixelBufferIOSurfacePropertiesKey, kCVPixelBufferPixelFormatTypeKey }; CFDictionaryRef io_surface_value = internal::CreateCFDictionary(nullptr, nullptr, 0); int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; CFNumberRef pixel_format = CFNumberCreate(nullptr, kCFNumberLongType, &nv12type); CFTypeRef values[attributes_size] = {kCFBooleanTrue, io_surface_value, pixel_format}; CFDictionaryRef source_attributes = internal::CreateCFDictionary(keys, values, attributes_size); if (io_surface_value) { CFRelease(io_surface_value); io_surface_value = nullptr; } if (pixel_format) { CFRelease(pixel_format); pixel_format = nullptr; } OSStatus status = VTCompressionSessionCreate( nullptr, // use default allocator width_, height_, kCMVideoCodecType_H264, nullptr, // use default encoder source_attributes, nullptr, // use default compressed data allocator internal::VTCompressionOutputCallback, this, &compression_session_); if (source_attributes) { CFRelease(source_attributes); source_attributes = nullptr; } if (status != noErr) { LOG(LS_ERROR) << "Failed to create compression session: " << status; return WEBRTC_VIDEO_CODEC_ERROR; } ConfigureCompressionSession(); return WEBRTC_VIDEO_CODEC_OK; } void H264VideoToolboxEncoder::ConfigureCompressionSession() { RTC_DCHECK(compression_session_); internal::SetVTSessionProperty(compression_session_, kVTCompressionPropertyKey_RealTime, true); internal::SetVTSessionProperty(compression_session_, kVTCompressionPropertyKey_ProfileLevel, profile_); internal::SetVTSessionProperty(compression_session_, kVTCompressionPropertyKey_AllowFrameReordering, false); SetEncoderBitrateBps(target_bitrate_bps_); // TODO(tkchin): Look at entropy mode and colorspace matrices. // TODO(tkchin): Investigate to see if there's any way to make this work. // May need it to interop with Android. Currently this call just fails. // On inspecting encoder output on iOS8, this value is set to 6. // internal::SetVTSessionProperty(compression_session_, // kVTCompressionPropertyKey_MaxFrameDelayCount, // 1); // Set a relatively large value for keyframe emission (7200 frames or // 4 minutes). internal::SetVTSessionProperty( compression_session_, kVTCompressionPropertyKey_MaxKeyFrameInterval, 7200); internal::SetVTSessionProperty( compression_session_, kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, 240); } void H264VideoToolboxEncoder::DestroyCompressionSession() { if (compression_session_) { VTCompressionSessionInvalidate(compression_session_); CFRelease(compression_session_); compression_session_ = nullptr; } } const char* H264VideoToolboxEncoder::ImplementationName() const { return "VideoToolbox"; } bool H264VideoToolboxEncoder::SupportsNativeHandle() const { return true; } void H264VideoToolboxEncoder::SetBitrateBps(uint32_t bitrate_bps) { if (encoder_bitrate_bps_ != bitrate_bps) { SetEncoderBitrateBps(bitrate_bps); } } void H264VideoToolboxEncoder::SetEncoderBitrateBps(uint32_t bitrate_bps) { if (compression_session_) { internal::SetVTSessionProperty(compression_session_, kVTCompressionPropertyKey_AverageBitRate, bitrate_bps); // TODO(tkchin): Add a helper method to set array value. int64_t data_limit_bytes_per_second_value = static_cast( bitrate_bps * internal::kLimitToAverageBitRateFactor / 8); CFNumberRef bytes_per_second = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &data_limit_bytes_per_second_value); int64_t one_second_value = 1; CFNumberRef one_second = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &one_second_value); const void* nums[2] = { bytes_per_second, one_second }; CFArrayRef data_rate_limits = CFArrayCreate(nullptr, nums, 2, &kCFTypeArrayCallBacks); OSStatus status = VTSessionSetProperty(compression_session_, kVTCompressionPropertyKey_DataRateLimits, data_rate_limits); if (bytes_per_second) { CFRelease(bytes_per_second); } if (one_second) { CFRelease(one_second); } if (data_rate_limits) { CFRelease(data_rate_limits); } if (status != noErr) { LOG(LS_ERROR) << "Failed to set data rate limit"; } encoder_bitrate_bps_ = bitrate_bps; } } void H264VideoToolboxEncoder::OnEncodedFrame( OSStatus status, VTEncodeInfoFlags info_flags, CMSampleBufferRef sample_buffer, CodecSpecificInfo codec_specific_info, int32_t width, int32_t height, int64_t render_time_ms, uint32_t timestamp, VideoRotation rotation) { if (status != noErr) { LOG(LS_ERROR) << "H264 encode failed."; return; } if (info_flags & kVTEncodeInfo_FrameDropped) { LOG(LS_INFO) << "H264 encode dropped frame."; return; } bool is_keyframe = false; CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sample_buffer, 0); if (attachments != nullptr && CFArrayGetCount(attachments)) { CFDictionaryRef attachment = static_cast(CFArrayGetValueAtIndex(attachments, 0)); is_keyframe = !CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync); } if (is_keyframe) { LOG(LS_INFO) << "Generated keyframe"; } // Convert the sample buffer into a buffer suitable for RTP packetization. // TODO(tkchin): Allocate buffers through a pool. std::unique_ptr buffer(new rtc::Buffer()); std::unique_ptr header; { webrtc::RTPFragmentationHeader* header_raw; bool result = H264CMSampleBufferToAnnexBBuffer(sample_buffer, is_keyframe, buffer.get(), &header_raw); header.reset(header_raw); if (!result) { return; } } webrtc::EncodedImage frame(buffer->data(), buffer->size(), buffer->size()); frame._encodedWidth = width; frame._encodedHeight = height; frame._completeFrame = true; frame._frameType = is_keyframe ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta; frame.capture_time_ms_ = render_time_ms; frame._timeStamp = timestamp; frame.rotation_ = rotation; frame.content_type_ = (mode_ == kScreensharing) ? VideoContentType::SCREENSHARE : VideoContentType::UNSPECIFIED; frame.timing_.is_timing_frame = false; h264_bitstream_parser_.ParseBitstream(buffer->data(), buffer->size()); h264_bitstream_parser_.GetLastSliceQp(&frame.qp_); EncodedImageCallback::Result res = callback_->OnEncodedImage(frame, &codec_specific_info, header.get()); if (res.error != EncodedImageCallback::Result::OK) { LOG(LS_ERROR) << "Encode callback failed: " << res.error; return; } bitrate_adjuster_.Update(frame._length); } VideoEncoder::ScalingSettings H264VideoToolboxEncoder::GetScalingSettings() const { return VideoEncoder::ScalingSettings(true, internal::kLowH264QpThreshold, internal::kHighH264QpThreshold); } } // namespace webrtc