admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
webrtc_m130/talk/app/webrtc/java/jni/androidmediaencoder_jni.cc
perkj 9576e54836 Reland "Prepare MediaCodecVideoEncoder for surface textures."" 
This reverts commit 12f680214e28dc5f0a13ac8afc0d1445f89e67e6.
Original cl in https://codereview.webrtc.org/1396073003/
Prepare MediaCodecVideoEncoder for surface textures.
This refactors MediaVideoEncoder to prepare for adding support to encode from textures. The C++ layer does not have any functional changes.
- Moves ResetEncoder to always work on the codec thread
- Adds use of ThreadChecker.
- Change Java MediaEncoder.Init to return true or false and introduce method getInputBuffers.
- Add simple unit test for Java MediaCodecVideoEncoder.

The pure revert of the revert is in patchset 1.
Patchset 2, moves getting the input buffer to before storing pending timestamps etc to fix b/24984012.

BUG=webrtc:4993 b/24984012

Review URL: https://codereview.webrtc.org/1406203002

Cr-Commit-Position: refs/heads/master@{#10622}
2015-11-12 14:43:22 +00:00

1018 lines
39 KiB
C++
Raw Blame History

/*
* libjingle
* Copyright 2015 Google Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
* EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/
#include "talk/app/webrtc/java/jni/androidmediaencoder_jni.h"
#include "talk/app/webrtc/java/jni/classreferenceholder.h"
#include "talk/app/webrtc/java/jni/androidmediacodeccommon.h"
#include "webrtc/base/bind.h"
#include "webrtc/base/checks.h"
#include "webrtc/base/logging.h"
#include "webrtc/base/thread.h"
#include "webrtc/base/thread_checker.h"
#include "webrtc/modules/rtp_rtcp/source/h264_bitstream_parser.h"
#include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h"
#include "webrtc/modules/video_coding/utility/include/quality_scaler.h"
#include "webrtc/modules/video_coding/utility/include/vp8_header_parser.h"
#include "webrtc/system_wrappers/include/field_trial.h"
#include "webrtc/system_wrappers/include/logcat_trace_context.h"
#include "third_party/libyuv/include/libyuv/convert.h"
#include "third_party/libyuv/include/libyuv/convert_from.h"
#include "third_party/libyuv/include/libyuv/video_common.h"
using rtc::Bind;
using rtc::Thread;
using rtc::ThreadManager;
using rtc::scoped_ptr;
using webrtc::CodecSpecificInfo;
using webrtc::EncodedImage;
using webrtc::VideoFrame;
using webrtc::RTPFragmentationHeader;
using webrtc::VideoCodec;
using webrtc::VideoCodecType;
using webrtc::kVideoCodecH264;
using webrtc::kVideoCodecVP8;
namespace webrtc_jni {
// H.264 start code length.
#define H264_SC_LENGTH 4
// Maximum allowed NALUs in one output frame.
#define MAX_NALUS_PERFRAME 32
// Maximum supported HW video encoder resolution.
#define MAX_VIDEO_WIDTH 1280
#define MAX_VIDEO_HEIGHT 1280
// Maximum supported HW video encoder fps.
#define MAX_VIDEO_FPS 30
// MediaCodecVideoEncoder is a webrtc::VideoEncoder implementation that uses
// Android's MediaCodec SDK API behind the scenes to implement (hopefully)
// HW-backed video encode. This C++ class is implemented as a very thin shim,
// delegating all of the interesting work to org.webrtc.MediaCodecVideoEncoder.
// MediaCodecVideoEncoder is created, operated, and destroyed on a single
// thread, currently the libjingle Worker thread.
class MediaCodecVideoEncoder : public webrtc::VideoEncoder,
public rtc::MessageHandler {
public:
virtual ~MediaCodecVideoEncoder();
MediaCodecVideoEncoder(JNIEnv* jni,
VideoCodecType codecType);
// webrtc::VideoEncoder implementation. Everything trampolines to
// |codec_thread_| for execution.
int32_t InitEncode(const webrtc::VideoCodec* codec_settings,
int32_t /* number_of_cores */,
size_t /* max_payload_size */) override;
int32_t Encode(const webrtc::VideoFrame& input_image,
const webrtc::CodecSpecificInfo* /* codec_specific_info */,
const std::vector<webrtc::FrameType>* frame_types) override;
int32_t RegisterEncodeCompleteCallback(
webrtc::EncodedImageCallback* callback) override;
int32_t Release() override;
int32_t SetChannelParameters(uint32_t /* packet_loss */,
int64_t /* rtt */) override;
int32_t SetRates(uint32_t new_bit_rate, uint32_t frame_rate) override;
// rtc::MessageHandler implementation.
void OnMessage(rtc::Message* msg) override;
void OnDroppedFrame() override;
int GetTargetFramerate() override;
private:
// ResetCodecOnCodecThread() calls ReleaseOnCodecThread() and
// InitEncodeOnCodecThread() in an attempt to restore the codec to an
// operable state. Necessary after all manner of OMX-layer errors.
bool ResetCodecOnCodecThread();
// Implementation of webrtc::VideoEncoder methods above, all running on the
// codec thread exclusively.
//
// If width==0 then this is assumed to be a re-initialization and the
// previously-current values are reused instead of the passed parameters
// (makes it easier to reason about thread-safety).
int32_t InitEncodeOnCodecThread(int width, int height, int kbps, int fps);
// Reconfigure to match |frame| in width, height. Returns false if
// reconfiguring fails.
bool MaybeReconfigureEncoderOnCodecThread(const webrtc::VideoFrame& frame);
int32_t EncodeOnCodecThread(
const webrtc::VideoFrame& input_image,
const std::vector<webrtc::FrameType>* frame_types);
bool EncodeByteBufferOnCodecThread(JNIEnv* jni,
bool key_frame, const webrtc::VideoFrame& frame, int input_buffer_index);
int32_t RegisterEncodeCompleteCallbackOnCodecThread(
webrtc::EncodedImageCallback* callback);
int32_t ReleaseOnCodecThread();
int32_t SetRatesOnCodecThread(uint32_t new_bit_rate, uint32_t frame_rate);
// Helper accessors for MediaCodecVideoEncoder$OutputBufferInfo members.
int GetOutputBufferInfoIndex(JNIEnv* jni, jobject j_output_buffer_info);
jobject GetOutputBufferInfoBuffer(JNIEnv* jni, jobject j_output_buffer_info);
bool GetOutputBufferInfoIsKeyFrame(JNIEnv* jni, jobject j_output_buffer_info);
jlong GetOutputBufferInfoPresentationTimestampUs(
JNIEnv* jni, jobject j_output_buffer_info);
// Deliver any outputs pending in the MediaCodec to our |callback_| and return
// true on success.
bool DeliverPendingOutputs(JNIEnv* jni);
// Search for H.264 start codes.
int32_t NextNaluPosition(uint8_t *buffer, size_t buffer_size);
// Type of video codec.
VideoCodecType codecType_;
// Valid all the time since RegisterEncodeCompleteCallback() Invoke()s to
// |codec_thread_| synchronously.
webrtc::EncodedImageCallback* callback_;
// State that is constant for the lifetime of this object once the ctor
// returns.
scoped_ptr<Thread> codec_thread_; // Thread on which to operate MediaCodec.
rtc::ThreadChecker codec_thread_checker_;
ScopedGlobalRef<jclass> j_media_codec_video_encoder_class_;
ScopedGlobalRef<jobject> j_media_codec_video_encoder_;
jmethodID j_init_encode_method_;
jmethodID j_get_input_buffers_method_;
jmethodID j_dequeue_input_buffer_method_;
jmethodID j_encode_buffer_method_;
jmethodID j_release_method_;
jmethodID j_set_rates_method_;
jmethodID j_dequeue_output_buffer_method_;
jmethodID j_release_output_buffer_method_;
jfieldID j_color_format_field_;
jfieldID j_info_index_field_;
jfieldID j_info_buffer_field_;
jfieldID j_info_is_key_frame_field_;
jfieldID j_info_presentation_timestamp_us_field_;
// State that is valid only between InitEncode() and the next Release().
// Touched only on codec_thread_ so no explicit synchronization necessary.
int width_; // Frame width in pixels.
int height_; // Frame height in pixels.
bool inited_;
uint16_t picture_id_;
enum libyuv::FourCC encoder_fourcc_; // Encoder color space format.
int last_set_bitrate_kbps_; // Last-requested bitrate in kbps.
int last_set_fps_; // Last-requested frame rate.
int64_t current_timestamp_us_; // Current frame timestamps in us.
int frames_received_; // Number of frames received by encoder.
int frames_encoded_; // Number of frames encoded by encoder.
int frames_dropped_; // Number of frames dropped by encoder.
int frames_in_queue_; // Number of frames in encoder queue.
int64_t start_time_ms_; // Start time for statistics.
int current_frames_; // Number of frames in the current statistics interval.
int current_bytes_; // Encoded bytes in the current statistics interval.
int current_encoding_time_ms_; // Overall encoding time in the current second
int64_t last_input_timestamp_ms_; // Timestamp of last received yuv frame.
int64_t last_output_timestamp_ms_; // Timestamp of last encoded frame.
std::vector<int32_t> timestamps_; // Video frames timestamp queue.
std::vector<int64_t> render_times_ms_; // Video frames render time queue.
std::vector<int64_t> frame_rtc_times_ms_; // Time when video frame is sent to
// encoder input.
int32_t output_timestamp_; // Last output frame timestamp from timestamps_ Q.
int64_t output_render_time_ms_; // Last output frame render time from
// render_times_ms_ queue.
// Frame size in bytes fed to MediaCodec.
int yuv_size_;
// True only when between a callback_->Encoded() call return a positive value
// and the next Encode() call being ignored.
bool drop_next_input_frame_;
// Global references; must be deleted in Release().
std::vector<jobject> input_buffers_;
webrtc::QualityScaler quality_scaler_;
// Dynamic resolution change, off by default.
bool scale_;
// H264 bitstream parser, used to extract QP from encoded bitstreams.
webrtc::H264BitstreamParser h264_bitstream_parser_;
};
MediaCodecVideoEncoder::~MediaCodecVideoEncoder() {
// Call Release() to ensure no more callbacks to us after we are deleted.
Release();
}
MediaCodecVideoEncoder::MediaCodecVideoEncoder(
JNIEnv* jni, VideoCodecType codecType) :
codecType_(codecType),
callback_(NULL),
inited_(false),
picture_id_(0),
codec_thread_(new Thread()),
j_media_codec_video_encoder_class_(
jni,
FindClass(jni, "org/webrtc/MediaCodecVideoEncoder")),
j_media_codec_video_encoder_(
jni,
jni->NewObject(*j_media_codec_video_encoder_class_,
GetMethodID(jni,
*j_media_codec_video_encoder_class_,
"<init>",
"()V"))) {
ScopedLocalRefFrame local_ref_frame(jni);
// It would be nice to avoid spinning up a new thread per MediaCodec, and
// instead re-use e.g. the PeerConnectionFactory's |worker_thread_|, but bug
// 2732 means that deadlocks abound. This class synchronously trampolines
// to |codec_thread_|, so if anything else can be coming to _us_ from
// |codec_thread_|, or from any thread holding the |_sendCritSect| described
// in the bug, we have a problem. For now work around that with a dedicated
// thread.
codec_thread_->SetName("MediaCodecVideoEncoder", NULL);
RTC_CHECK(codec_thread_->Start()) << "Failed to start MediaCodecVideoEncoder";
codec_thread_checker_.DetachFromThread();
jclass j_output_buffer_info_class =
FindClass(jni, "org/webrtc/MediaCodecVideoEncoder$OutputBufferInfo");
j_init_encode_method_ = GetMethodID(
jni,
*j_media_codec_video_encoder_class_,
"initEncode",
"(Lorg/webrtc/MediaCodecVideoEncoder$VideoCodecType;IIII)Z");
j_get_input_buffers_method_ = GetMethodID(
jni,
*j_media_codec_video_encoder_class_,
"getInputBuffers",
"()[Ljava/nio/ByteBuffer;");
j_dequeue_input_buffer_method_ = GetMethodID(
jni, *j_media_codec_video_encoder_class_, "dequeueInputBuffer", "()I");
j_encode_buffer_method_ = GetMethodID(
jni, *j_media_codec_video_encoder_class_, "encodeBuffer", "(ZIIJ)Z");
j_release_method_ =
GetMethodID(jni, *j_media_codec_video_encoder_class_, "release", "()V");
j_set_rates_method_ = GetMethodID(
jni, *j_media_codec_video_encoder_class_, "setRates", "(II)Z");
j_dequeue_output_buffer_method_ = GetMethodID(
jni,
*j_media_codec_video_encoder_class_,
"dequeueOutputBuffer",
"()Lorg/webrtc/MediaCodecVideoEncoder$OutputBufferInfo;");
j_release_output_buffer_method_ = GetMethodID(
jni, *j_media_codec_video_encoder_class_, "releaseOutputBuffer", "(I)Z");
j_color_format_field_ =
GetFieldID(jni, *j_media_codec_video_encoder_class_, "colorFormat", "I");
j_info_index_field_ =
GetFieldID(jni, j_output_buffer_info_class, "index", "I");
j_info_buffer_field_ = GetFieldID(
jni, j_output_buffer_info_class, "buffer", "Ljava/nio/ByteBuffer;");
j_info_is_key_frame_field_ =
GetFieldID(jni, j_output_buffer_info_class, "isKeyFrame", "Z");
j_info_presentation_timestamp_us_field_ = GetFieldID(
jni, j_output_buffer_info_class, "presentationTimestampUs", "J");
CHECK_EXCEPTION(jni) << "MediaCodecVideoEncoder ctor failed";
AllowBlockingCalls();
}
int32_t MediaCodecVideoEncoder::InitEncode(
const webrtc::VideoCodec* codec_settings,
int32_t /* number_of_cores */,
size_t /* max_payload_size */) {
const int kMinWidth = 320;
const int kMinHeight = 180;
const int kLowQpThresholdDenominator = 3;
if (codec_settings == NULL) {
ALOGE << "NULL VideoCodec instance";
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
// Factory should guard against other codecs being used with us.
RTC_CHECK(codec_settings->codecType == codecType_)
<< "Unsupported codec " << codec_settings->codecType << " for "
<< codecType_;
ALOGD << "InitEncode request";
scale_ = webrtc::field_trial::FindFullName(
"WebRTC-MediaCodecVideoEncoder-AutomaticResize") == "Enabled";
ALOGD << "Encoder automatic resize " << (scale_ ? "enabled" : "disabled");
if (scale_) {
if (codecType_ == kVideoCodecVP8) {
// QP is obtained from VP8-bitstream for HW, so the QP corresponds to the
// (internal) range: [0, 127]. And we cannot change QP_max in HW, so it is
// always = 127. Note that in SW, QP is that of the user-level range [0,
// 63].
const int kMaxQp = 127;
// TODO(pbos): Investigate whether high-QP thresholds make sense for VP8.
// This effectively disables high QP as VP8 QP can't go above this
// threshold.
const int kDisabledBadQpThreshold = kMaxQp + 1;
quality_scaler_.Init(kMaxQp / kLowQpThresholdDenominator,
kDisabledBadQpThreshold, true);
} else if (codecType_ == kVideoCodecH264) {
// H264 QP is in the range [0, 51].
const int kMaxQp = 51;
const int kBadQpThreshold = 40;
quality_scaler_.Init(kMaxQp / kLowQpThresholdDenominator, kBadQpThreshold,
false);
} else {
// When adding codec support to additional hardware codecs, also configure
// their QP thresholds for scaling.
RTC_NOTREACHED() << "Unsupported codec without configured QP thresholds.";
}
quality_scaler_.SetMinResolution(kMinWidth, kMinHeight);
quality_scaler_.ReportFramerate(codec_settings->maxFramerate);
}
return codec_thread_->Invoke<int32_t>(
Bind(&MediaCodecVideoEncoder::InitEncodeOnCodecThread,
this,
codec_settings->width,
codec_settings->height,
codec_settings->startBitrate,
codec_settings->maxFramerate));
}
int32_t MediaCodecVideoEncoder::Encode(
const webrtc::VideoFrame& frame,
const webrtc::CodecSpecificInfo* /* codec_specific_info */,
const std::vector<webrtc::FrameType>* frame_types) {
return codec_thread_->Invoke<int32_t>(Bind(
&MediaCodecVideoEncoder::EncodeOnCodecThread, this, frame, frame_types));
}
int32_t MediaCodecVideoEncoder::RegisterEncodeCompleteCallback(
webrtc::EncodedImageCallback* callback) {
return codec_thread_->Invoke<int32_t>(
Bind(&MediaCodecVideoEncoder::RegisterEncodeCompleteCallbackOnCodecThread,
this,
callback));
}
int32_t MediaCodecVideoEncoder::Release() {
ALOGD << "EncoderRelease request";
return codec_thread_->Invoke<int32_t>(
Bind(&MediaCodecVideoEncoder::ReleaseOnCodecThread, this));
}
int32_t MediaCodecVideoEncoder::SetChannelParameters(uint32_t /* packet_loss */,
int64_t /* rtt */) {
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t MediaCodecVideoEncoder::SetRates(uint32_t new_bit_rate,
uint32_t frame_rate) {
if (scale_)
quality_scaler_.ReportFramerate(frame_rate);
return codec_thread_->Invoke<int32_t>(
Bind(&MediaCodecVideoEncoder::SetRatesOnCodecThread,
this,
new_bit_rate,
frame_rate));
}
void MediaCodecVideoEncoder::OnMessage(rtc::Message* msg) {
RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedLocalRefFrame local_ref_frame(jni);
// We only ever send one message to |this| directly (not through a Bind()'d
// functor), so expect no ID/data.
RTC_CHECK(!msg->message_id) << "Unexpected message!";
RTC_CHECK(!msg->pdata) << "Unexpected message!";
if (!inited_) {
return;
}
// It would be nice to recover from a failure here if one happened, but it's
// unclear how to signal such a failure to the app, so instead we stay silent
// about it and let the next app-called API method reveal the borkedness.
DeliverPendingOutputs(jni);
codec_thread_->PostDelayed(kMediaCodecPollMs, this);
}
bool MediaCodecVideoEncoder::ResetCodecOnCodecThread() {
RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
ALOGE << "ResetOnCodecThread";
if (ReleaseOnCodecThread() != WEBRTC_VIDEO_CODEC_OK ||
InitEncodeOnCodecThread(width_, height_, 0, 0)
!= WEBRTC_VIDEO_CODEC_OK) {
// TODO(fischman): wouldn't it be nice if there was a way to gracefully
// degrade to a SW encoder at this point? There isn't one AFAICT :(
// https://code.google.com/p/webrtc/issues/detail?id=2920
return false;
}
return true;
}
int32_t MediaCodecVideoEncoder::InitEncodeOnCodecThread(
int width, int height, int kbps, int fps) {
RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedLocalRefFrame local_ref_frame(jni);
ALOGD << "InitEncodeOnCodecThread Type: " << (int)codecType_ << ", " <<
width << " x " << height << ". Bitrate: " << kbps <<
" kbps. Fps: " << fps;
if (kbps == 0) {
kbps = last_set_bitrate_kbps_;
}
if (fps == 0) {
fps = last_set_fps_;
}
width_ = width;
height_ = height;
last_set_bitrate_kbps_ = kbps;
last_set_fps_ = fps;
yuv_size_ = width_ * height_ * 3 / 2;
frames_received_ = 0;
frames_encoded_ = 0;
frames_dropped_ = 0;
frames_in_queue_ = 0;
current_timestamp_us_ = 0;
start_time_ms_ = GetCurrentTimeMs();
current_frames_ = 0;
current_bytes_ = 0;
current_encoding_time_ms_ = 0;
last_input_timestamp_ms_ = -1;
last_output_timestamp_ms_ = -1;
output_timestamp_ = 0;
output_render_time_ms_ = 0;
timestamps_.clear();
render_times_ms_.clear();
frame_rtc_times_ms_.clear();
drop_next_input_frame_ = false;
picture_id_ = static_cast<uint16_t>(rand()) & 0x7FFF;
// We enforce no extra stride/padding in the format creation step.
jobject j_video_codec_enum = JavaEnumFromIndex(
jni, "MediaCodecVideoEncoder$VideoCodecType", codecType_);
const bool encode_status = jni->CallBooleanMethod(
*j_media_codec_video_encoder_, j_init_encode_method_,
j_video_codec_enum, width, height, kbps, fps);
if (!encode_status) {
ALOGE << "Failed to configure encoder.";
return WEBRTC_VIDEO_CODEC_ERROR;
}
CHECK_EXCEPTION(jni);
jobjectArray input_buffers = reinterpret_cast<jobjectArray>(
jni->CallObjectMethod(*j_media_codec_video_encoder_,
j_get_input_buffers_method_));
CHECK_EXCEPTION(jni);
if (IsNull(jni, input_buffers)) {
return WEBRTC_VIDEO_CODEC_ERROR;
}
switch (GetIntField(jni, *j_media_codec_video_encoder_,
j_color_format_field_)) {
case COLOR_FormatYUV420Planar:
encoder_fourcc_ = libyuv::FOURCC_YU12;
break;
case COLOR_FormatYUV420SemiPlanar:
case COLOR_QCOM_FormatYUV420SemiPlanar:
case COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m:
encoder_fourcc_ = libyuv::FOURCC_NV12;
break;
default:
LOG(LS_ERROR) << "Wrong color format.";
return WEBRTC_VIDEO_CODEC_ERROR;
}
size_t num_input_buffers = jni->GetArrayLength(input_buffers);
RTC_CHECK(input_buffers_.empty())
<< "Unexpected double InitEncode without Release";
input_buffers_.resize(num_input_buffers);
for (size_t i = 0; i < num_input_buffers; ++i) {
input_buffers_[i] =
jni->NewGlobalRef(jni->GetObjectArrayElement(input_buffers, i));
int64_t yuv_buffer_capacity =
jni->GetDirectBufferCapacity(input_buffers_[i]);
CHECK_EXCEPTION(jni);
RTC_CHECK(yuv_buffer_capacity >= yuv_size_) << "Insufficient capacity";
}
CHECK_EXCEPTION(jni);
inited_ = true;
codec_thread_->PostDelayed(kMediaCodecPollMs, this);
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t MediaCodecVideoEncoder::EncodeOnCodecThread(
const webrtc::VideoFrame& frame,
const std::vector<webrtc::FrameType>* frame_types) {
RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedLocalRefFrame local_ref_frame(jni);
if (!inited_) {
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
frames_received_++;
if (!DeliverPendingOutputs(jni)) {
if (!ResetCodecOnCodecThread())
return WEBRTC_VIDEO_CODEC_ERROR;
}
if (drop_next_input_frame_) {
ALOGW << "Encoder drop frame - failed callback.";
drop_next_input_frame_ = false;
return WEBRTC_VIDEO_CODEC_OK;
}
RTC_CHECK(frame_types->size() == 1) << "Unexpected stream count";
// Check framerate before spatial resolution change.
if (scale_)
quality_scaler_.OnEncodeFrame(frame);
const VideoFrame& input_frame =
scale_ ? quality_scaler_.GetScaledFrame(frame) : frame;
if (!MaybeReconfigureEncoderOnCodecThread(input_frame)) {
ALOGE << "Failed to reconfigure encoder.";
return WEBRTC_VIDEO_CODEC_ERROR;
}
// Check if we accumulated too many frames in encoder input buffers
// or the encoder latency exceeds 70 ms and drop frame if so.
if (frames_in_queue_ > 0 && last_input_timestamp_ms_ >= 0) {
int encoder_latency_ms = last_input_timestamp_ms_ -
last_output_timestamp_ms_;
if (frames_in_queue_ > 2 || encoder_latency_ms > 70) {
ALOGD << "Drop frame - encoder is behind by " << encoder_latency_ms <<
" ms. Q size: " << frames_in_queue_;
frames_dropped_++;
// Report dropped frame to quality_scaler_.
OnDroppedFrame();
return WEBRTC_VIDEO_CODEC_OK;
}
}
int j_input_buffer_index = jni->CallIntMethod(*j_media_codec_video_encoder_,
j_dequeue_input_buffer_method_);
CHECK_EXCEPTION(jni);
if (j_input_buffer_index == -1) {
// Video codec falls behind - no input buffer available.
ALOGW << "Encoder drop frame - no input buffers available";
frames_dropped_++;
// Report dropped frame to quality_scaler_.
OnDroppedFrame();
return WEBRTC_VIDEO_CODEC_OK; // TODO(fischman): see webrtc bug 2887.
}
if (j_input_buffer_index == -2) {
ResetCodecOnCodecThread();
return WEBRTC_VIDEO_CODEC_ERROR;
}
last_input_timestamp_ms_ =
current_timestamp_us_ / rtc::kNumMicrosecsPerMillisec;
frames_in_queue_++;
// Save input image timestamps for later output
timestamps_.push_back(input_frame.timestamp());
render_times_ms_.push_back(input_frame.render_time_ms());
frame_rtc_times_ms_.push_back(GetCurrentTimeMs());
const bool key_frame = frame_types->front() != webrtc::kVideoFrameDelta;
const bool encode_status =
EncodeByteBufferOnCodecThread(jni, key_frame, input_frame,
j_input_buffer_index);
current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
if (!encode_status || !DeliverPendingOutputs(jni)) {
ALOGE << "Failed deliver pending outputs.";
ResetCodecOnCodecThread();
return WEBRTC_VIDEO_CODEC_ERROR;
}
return WEBRTC_VIDEO_CODEC_OK;
}
bool MediaCodecVideoEncoder::MaybeReconfigureEncoderOnCodecThread(
const webrtc::VideoFrame& frame) {
RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
const bool reconfigure_due_to_size =
frame.width() != width_ || frame.height() != height_;
if (reconfigure_due_to_size) {
ALOGD << "Reconfigure encoder due to frame resolution change from "
<< width_ << " x " << height_ << " to " << frame.width() << " x "
<< frame.height();
width_ = frame.width();
height_ = frame.height();
}
if (!reconfigure_due_to_size)
return true;
ReleaseOnCodecThread();
return InitEncodeOnCodecThread(width_, height_, 0, 0) ==
WEBRTC_VIDEO_CODEC_OK;
}
bool MediaCodecVideoEncoder::EncodeByteBufferOnCodecThread(JNIEnv* jni,
bool key_frame, const webrtc::VideoFrame& frame, int input_buffer_index) {
RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
ALOGV("Encoder frame in # %d. TS: %lld. Q: %d",
frames_received_ - 1, current_timestamp_us_ / 1000, frames_in_queue_);
jobject j_input_buffer = input_buffers_[input_buffer_index];
uint8_t* yuv_buffer =
reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress(j_input_buffer));
CHECK_EXCEPTION(jni);
RTC_CHECK(yuv_buffer) << "Indirect buffer??";
RTC_CHECK(!libyuv::ConvertFromI420(
frame.buffer(webrtc::kYPlane), frame.stride(webrtc::kYPlane),
frame.buffer(webrtc::kUPlane), frame.stride(webrtc::kUPlane),
frame.buffer(webrtc::kVPlane), frame.stride(webrtc::kVPlane),
yuv_buffer, width_, width_, height_, encoder_fourcc_))
<< "ConvertFromI420 failed";
bool encode_status = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
j_encode_buffer_method_,
key_frame,
input_buffer_index,
yuv_size_,
current_timestamp_us_);
CHECK_EXCEPTION(jni);
return encode_status;
}
int32_t MediaCodecVideoEncoder::RegisterEncodeCompleteCallbackOnCodecThread(
webrtc::EncodedImageCallback* callback) {
RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedLocalRefFrame local_ref_frame(jni);
callback_ = callback;
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t MediaCodecVideoEncoder::ReleaseOnCodecThread() {
RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
if (!inited_) {
return WEBRTC_VIDEO_CODEC_OK;
}
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ALOGD << "EncoderReleaseOnCodecThread: Frames received: " <<
frames_received_ << ". Encoded: " << frames_encoded_ <<
". Dropped: " << frames_dropped_;
ScopedLocalRefFrame local_ref_frame(jni);
for (size_t i = 0; i < input_buffers_.size(); ++i)
jni->DeleteGlobalRef(input_buffers_[i]);
input_buffers_.clear();
jni->CallVoidMethod(*j_media_codec_video_encoder_, j_release_method_);
CHECK_EXCEPTION(jni);
rtc::MessageQueueManager::Clear(this);
inited_ = false;
ALOGD << "EncoderReleaseOnCodecThread done.";
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t MediaCodecVideoEncoder::SetRatesOnCodecThread(uint32_t new_bit_rate,
uint32_t frame_rate) {
RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
if (last_set_bitrate_kbps_ == new_bit_rate &&
last_set_fps_ == frame_rate) {
return WEBRTC_VIDEO_CODEC_OK;
}
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedLocalRefFrame local_ref_frame(jni);
if (new_bit_rate > 0) {
last_set_bitrate_kbps_ = new_bit_rate;
}
if (frame_rate > 0) {
last_set_fps_ = frame_rate;
}
bool ret = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
j_set_rates_method_,
last_set_bitrate_kbps_,
last_set_fps_);
CHECK_EXCEPTION(jni);
if (!ret) {
ResetCodecOnCodecThread();
return WEBRTC_VIDEO_CODEC_ERROR;
}
return WEBRTC_VIDEO_CODEC_OK;
}
int MediaCodecVideoEncoder::GetOutputBufferInfoIndex(
JNIEnv* jni,
jobject j_output_buffer_info) {
return GetIntField(jni, j_output_buffer_info, j_info_index_field_);
}
jobject MediaCodecVideoEncoder::GetOutputBufferInfoBuffer(
JNIEnv* jni,
jobject j_output_buffer_info) {
return GetObjectField(jni, j_output_buffer_info, j_info_buffer_field_);
}
bool MediaCodecVideoEncoder::GetOutputBufferInfoIsKeyFrame(
JNIEnv* jni,
jobject j_output_buffer_info) {
return GetBooleanField(jni, j_output_buffer_info, j_info_is_key_frame_field_);
}
jlong MediaCodecVideoEncoder::GetOutputBufferInfoPresentationTimestampUs(
JNIEnv* jni,
jobject j_output_buffer_info) {
return GetLongField(
jni, j_output_buffer_info, j_info_presentation_timestamp_us_field_);
}
bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) {
RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
while (true) {
jobject j_output_buffer_info = jni->CallObjectMethod(
*j_media_codec_video_encoder_, j_dequeue_output_buffer_method_);
CHECK_EXCEPTION(jni);
if (IsNull(jni, j_output_buffer_info)) {
break;
}
int output_buffer_index =
GetOutputBufferInfoIndex(jni, j_output_buffer_info);
if (output_buffer_index == -1) {
ResetCodecOnCodecThread();
return false;
}
// Get key and config frame flags.
jobject j_output_buffer =
GetOutputBufferInfoBuffer(jni, j_output_buffer_info);
bool key_frame = GetOutputBufferInfoIsKeyFrame(jni, j_output_buffer_info);
// Get frame timestamps from a queue - for non config frames only.
int64_t frame_encoding_time_ms = 0;
last_output_timestamp_ms_ =
GetOutputBufferInfoPresentationTimestampUs(jni, j_output_buffer_info) /
1000;
if (frames_in_queue_ > 0) {
output_timestamp_ = timestamps_.front();
timestamps_.erase(timestamps_.begin());
output_render_time_ms_ = render_times_ms_.front();
render_times_ms_.erase(render_times_ms_.begin());
frame_encoding_time_ms = GetCurrentTimeMs() - frame_rtc_times_ms_.front();
frame_rtc_times_ms_.erase(frame_rtc_times_ms_.begin());
frames_in_queue_--;
}
// Extract payload.
size_t payload_size = jni->GetDirectBufferCapacity(j_output_buffer);
uint8_t* payload = reinterpret_cast<uint8_t*>(
jni->GetDirectBufferAddress(j_output_buffer));
CHECK_EXCEPTION(jni);
ALOGV("Encoder frame out # %d. Key: %d. Size: %d. TS: %lld."
" Latency: %lld. EncTime: %lld",
frames_encoded_, key_frame, payload_size,
last_output_timestamp_ms_,
last_input_timestamp_ms_ - last_output_timestamp_ms_,
frame_encoding_time_ms);
// Calculate and print encoding statistics - every 3 seconds.
frames_encoded_++;
current_frames_++;
current_bytes_ += payload_size;
current_encoding_time_ms_ += frame_encoding_time_ms;
int statistic_time_ms = GetCurrentTimeMs() - start_time_ms_;
if (statistic_time_ms >= kMediaCodecStatisticsIntervalMs &&
current_frames_ > 0) {
ALOGD << "Encoded frames: " << frames_encoded_ << ". Bitrate: " <<
(current_bytes_ * 8 / statistic_time_ms) <<
", target: " << last_set_bitrate_kbps_ << " kbps, fps: " <<
((current_frames_ * 1000 + statistic_time_ms / 2) / statistic_time_ms)
<< ", encTime: " <<
(current_encoding_time_ms_ / current_frames_) << " for last " <<
statistic_time_ms << " ms.";
start_time_ms_ = GetCurrentTimeMs();
current_frames_ = 0;
current_bytes_ = 0;
current_encoding_time_ms_ = 0;
}
// Callback - return encoded frame.
int32_t callback_status = 0;
if (callback_) {
scoped_ptr<webrtc::EncodedImage> image(
new webrtc::EncodedImage(payload, payload_size, payload_size));
image->_encodedWidth = width_;
image->_encodedHeight = height_;
image->_timeStamp = output_timestamp_;
image->capture_time_ms_ = output_render_time_ms_;
image->_frameType =
(key_frame ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta);
image->_completeFrame = true;
image->adapt_reason_.quality_resolution_downscales =
scale_ ? quality_scaler_.downscale_shift() : -1;
webrtc::CodecSpecificInfo info;
memset(&info, 0, sizeof(info));
info.codecType = codecType_;
if (codecType_ == kVideoCodecVP8) {
info.codecSpecific.VP8.pictureId = picture_id_;
info.codecSpecific.VP8.nonReference = false;
info.codecSpecific.VP8.simulcastIdx = 0;
info.codecSpecific.VP8.temporalIdx = webrtc::kNoTemporalIdx;
info.codecSpecific.VP8.layerSync = false;
info.codecSpecific.VP8.tl0PicIdx = webrtc::kNoTl0PicIdx;
info.codecSpecific.VP8.keyIdx = webrtc::kNoKeyIdx;
picture_id_ = (picture_id_ + 1) & 0x7FFF;
}
// Generate a header describing a single fragment.
webrtc::RTPFragmentationHeader header;
memset(&header, 0, sizeof(header));
if (codecType_ == kVideoCodecVP8) {
header.VerifyAndAllocateFragmentationHeader(1);
header.fragmentationOffset[0] = 0;
header.fragmentationLength[0] = image->_length;
header.fragmentationPlType[0] = 0;
header.fragmentationTimeDiff[0] = 0;
if (scale_) {
int qp;
if (webrtc::vp8::GetQp(payload, payload_size, &qp))
quality_scaler_.ReportQP(qp);
}
} else if (codecType_ == kVideoCodecH264) {
if (scale_) {
h264_bitstream_parser_.ParseBitstream(payload, payload_size);
int qp;
if (h264_bitstream_parser_.GetLastSliceQp(&qp))
quality_scaler_.ReportQP(qp);
}
// For H.264 search for start codes.
int32_t scPositions[MAX_NALUS_PERFRAME + 1] = {};
int32_t scPositionsLength = 0;
int32_t scPosition = 0;
while (scPositionsLength < MAX_NALUS_PERFRAME) {
int32_t naluPosition = NextNaluPosition(
payload + scPosition, payload_size - scPosition);
if (naluPosition < 0) {
break;
}
scPosition += naluPosition;
scPositions[scPositionsLength++] = scPosition;
scPosition += H264_SC_LENGTH;
}
if (scPositionsLength == 0) {
ALOGE << "Start code is not found!";
ALOGE << "Data:" << image->_buffer[0] << " " << image->_buffer[1]
<< " " << image->_buffer[2] << " " << image->_buffer[3]
<< " " << image->_buffer[4] << " " << image->_buffer[5];
ResetCodecOnCodecThread();
return false;
}
scPositions[scPositionsLength] = payload_size;
header.VerifyAndAllocateFragmentationHeader(scPositionsLength);
for (size_t i = 0; i < scPositionsLength; i++) {
header.fragmentationOffset[i] = scPositions[i] + H264_SC_LENGTH;
header.fragmentationLength[i] =
scPositions[i + 1] - header.fragmentationOffset[i];
header.fragmentationPlType[i] = 0;
header.fragmentationTimeDiff[i] = 0;
}
}
callback_status = callback_->Encoded(*image, &info, &header);
}
// Return output buffer back to the encoder.
bool success = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
j_release_output_buffer_method_,
output_buffer_index);
CHECK_EXCEPTION(jni);
if (!success) {
ResetCodecOnCodecThread();
return false;
}
if (callback_status > 0) {
drop_next_input_frame_ = true;
// Theoretically could handle callback_status<0 here, but unclear what
// that would mean for us.
}
}
return true;
}
int32_t MediaCodecVideoEncoder::NextNaluPosition(
uint8_t *buffer, size_t buffer_size) {
if (buffer_size < H264_SC_LENGTH) {
return -1;
}
uint8_t *head = buffer;
// Set end buffer pointer to 4 bytes before actual buffer end so we can
// access head[1], head[2] and head[3] in a loop without buffer overrun.
uint8_t *end = buffer + buffer_size - H264_SC_LENGTH;
while (head < end) {
if (head[0]) {
head++;
continue;
}
if (head[1]) { // got 00xx
head += 2;
continue;
}
if (head[2]) { // got 0000xx
head += 3;
continue;
}
if (head[3] != 0x01) { // got 000000xx
head++; // xx != 1, continue searching.
continue;
}
return (int32_t)(head - buffer);
}
return -1;
}
void MediaCodecVideoEncoder::OnDroppedFrame() {
if (scale_)
quality_scaler_.ReportDroppedFrame();
}
int MediaCodecVideoEncoder::GetTargetFramerate() {
return scale_ ? quality_scaler_.GetTargetFramerate() : -1;
}
MediaCodecVideoEncoderFactory::MediaCodecVideoEncoderFactory() {
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedLocalRefFrame local_ref_frame(jni);
jclass j_encoder_class = FindClass(jni, "org/webrtc/MediaCodecVideoEncoder");
supported_codecs_.clear();
bool is_vp8_hw_supported = jni->CallStaticBooleanMethod(
j_encoder_class,
GetStaticMethodID(jni, j_encoder_class, "isVp8HwSupported", "()Z"));
CHECK_EXCEPTION(jni);
if (is_vp8_hw_supported) {
ALOGD << "VP8 HW Encoder supported.";
supported_codecs_.push_back(VideoCodec(kVideoCodecVP8, "VP8",
MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS));
}
bool is_h264_hw_supported = jni->CallStaticBooleanMethod(
j_encoder_class,
GetStaticMethodID(jni, j_encoder_class, "isH264HwSupported", "()Z"));
CHECK_EXCEPTION(jni);
if (is_h264_hw_supported) {
ALOGD << "H.264 HW Encoder supported.";
supported_codecs_.push_back(VideoCodec(kVideoCodecH264, "H264",
MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS));
}
}
MediaCodecVideoEncoderFactory::~MediaCodecVideoEncoderFactory() {}
webrtc::VideoEncoder* MediaCodecVideoEncoderFactory::CreateVideoEncoder(
VideoCodecType type) {
if (supported_codecs_.empty()) {
return NULL;
}
for (std::vector<VideoCodec>::const_iterator it = supported_codecs_.begin();
it != supported_codecs_.end(); ++it) {
if (it->type == type) {
ALOGD << "Create HW video encoder for type " << (int)type <<
" (" << it->name << ").";
return new MediaCodecVideoEncoder(AttachCurrentThreadIfNeeded(), type);
}
}
return NULL;
}
const std::vector<MediaCodecVideoEncoderFactory::VideoCodec>&
MediaCodecVideoEncoderFactory::codecs() const {
return supported_codecs_;
}
void MediaCodecVideoEncoderFactory::DestroyVideoEncoder(
webrtc::VideoEncoder* encoder) {
ALOGD << "Destroy video encoder.";
delete encoder;
}
} // namespace webrtc_jni
Reference in New Issue View Git Blame Copy Permalink