admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Hooking up audio network adaptor to VoE.

Browse Source 
BUG=webrtc:6303

Review-Url: https://codereview.webrtc.org/2390883004
Cr-Commit-Position: refs/heads/master@{#14611}
This commit is contained in:
minyue 2016-10-12 05:00:55 -07:00 committed by Commit bot
parent 917d4e1e71
commit 7e30432b36
12 changed files with 139 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
webrtc/modules/audio_coding/codecs/audio_encoder.cc
View File

@ -77,9 +77,13 @@ void AudioEncoder::DisableAudioNetworkAdaptor() {}
void AudioEncoder::OnReceivedUplinkBandwidth(int uplink_bandwidth_bps) {}
void AudioEncoder::OnReceivedUplinkPacketLossFraction(
float uplink_packet_loss_fraction) {}
float uplink_packet_loss_fraction) {
SetProjectedPacketLossRate(uplink_packet_loss_fraction);
}
void AudioEncoder::OnReceivedTargetAudioBitrate(int target_audio_bitrate_bps) {}
void AudioEncoder::OnReceivedTargetAudioBitrate(int target_audio_bitrate_bps) {
SetTargetBitrate(target_audio_bitrate_bps);
}
void AudioEncoder::OnReceivedRtt(int rtt_ms) {}

53
webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc
View File

@ -13,6 +13,7 @@
#include <algorithm>
#include "webrtc/base/checks.h"
#include "webrtc/base/exp_filter.h"
#include "webrtc/base/safe_conversions.h"
#include "webrtc/common_types.h"
#include "webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h"
@ -24,11 +25,15 @@ namespace webrtc {
namespace {
const int kSampleRateHz = 48000;
const int kMinBitrateBps = 500;
const int kMaxBitrateBps = 512000;
constexpr int kSampleRateHz = 48000;
constexpr int kMinBitrateBps = 500;
constexpr int kMaxBitrateBps = 512000;
constexpr int kSupportedFrameLengths[] = {20, 60};
// PacketLossFractionSmoother uses an exponential filter with a time constant
// of -1.0 / ln(0.9999) = 10000 ms.
constexpr float kAlphaForPacketLossFractionSmoother = 0.9999f;
AudioEncoderOpus::Config CreateConfig(const CodecInst& codec_inst) {
AudioEncoderOpus::Config config;
config.frame_size_ms = rtc::CheckedDivExact(codec_inst.pacsize, 48);
@ -82,6 +87,35 @@ double OptimizePacketLossRate(double new_loss_rate, double old_loss_rate) {
} // namespace
class AudioEncoderOpus::PacketLossFractionSmoother {
public:
explicit PacketLossFractionSmoother(const Clock* clock)
: clock_(clock),
last_sample_time_ms_(clock_->TimeInMilliseconds()),
smoother_(kAlphaForPacketLossFractionSmoother) {}
// Gets the smoothed packet loss fraction.
float GetAverage() const {
float value = smoother_.filtered();
return (value == rtc::ExpFilter::kValueUndefined) ? 0.0f : value;
}
// Add new observation to the packet loss fraction smoother.
void AddSample(float packet_loss_fraction) {
int64_t now_ms = clock_->TimeInMilliseconds();
smoother_.Apply(static_cast<float>(now_ms - last_sample_time_ms_),
packet_loss_fraction);
last_sample_time_ms_ = now_ms;
}
private:
const Clock* const clock_;
int64_t last_sample_time_ms_;
// An exponential filter is used to smooth the packet loss fraction.
rtc::ExpFilter smoother_;
};
AudioEncoderOpus::Config::Config() = default;
AudioEncoderOpus::Config::Config(const Config&) = default;
AudioEncoderOpus::Config::~Config() = default;
@ -113,9 +147,11 @@ AudioEncoderOpus::AudioEncoderOpus(
AudioNetworkAdaptorCreator&& audio_network_adaptor_creator)
: packet_loss_rate_(0.0),
inst_(nullptr),
packet_loss_fraction_smoother_(new PacketLossFractionSmoother(
config.clock ? config.clock : Clock::GetRealTimeClock())),
audio_network_adaptor_creator_(
audio_network_adaptor_creator
? audio_network_adaptor_creator
? std::move(audio_network_adaptor_creator)
: [this](const std::string& config_string, const Clock* clock) {
return DefaultAudioNetworkAdaptorCreator(config_string,
clock);
@ -234,8 +270,11 @@ void AudioEncoderOpus::OnReceivedUplinkBandwidth(int uplink_bandwidth_bps) {
void AudioEncoderOpus::OnReceivedUplinkPacketLossFraction(
float uplink_packet_loss_fraction) {
if (!audio_network_adaptor_)
return;
if (!audio_network_adaptor_) {
packet_loss_fraction_smoother_->AddSample(uplink_packet_loss_fraction);
float average_fraction_loss = packet_loss_fraction_smoother_->GetAverage();
return SetProjectedPacketLossRate(average_fraction_loss);
}
audio_network_adaptor_->SetUplinkPacketLossFraction(
uplink_packet_loss_fraction);
ApplyAudioNetworkAdaptor();
@ -244,7 +283,7 @@ void AudioEncoderOpus::OnReceivedUplinkPacketLossFraction(
void AudioEncoderOpus::OnReceivedTargetAudioBitrate(
int target_audio_bitrate_bps) {
if (!audio_network_adaptor_)
return;
return SetTargetBitrate(target_audio_bitrate_bps);
audio_network_adaptor_->SetTargetAudioBitrate(target_audio_bitrate_bps);
ApplyAudioNetworkAdaptor();
}

4
webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h
View File

@ -49,6 +49,7 @@ class AudioEncoderOpus final : public AudioEncoder {
int max_playback_rate_hz = 48000;
int complexity = kDefaultComplexity;
bool dtx_enabled = false;
const Clock* clock = nullptr;
private:
#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) || defined(WEBRTC_ARCH_ARM)
@ -115,6 +116,8 @@ class AudioEncoderOpus final : public AudioEncoder {
rtc::Buffer* encoded) override;
private:
class PacketLossFractionSmoother;
size_t Num10msFramesPerPacket() const;
size_t SamplesPer10msFrame() const;
size_t SufficientOutputBufferSize() const;
@ -133,6 +136,7 @@ class AudioEncoderOpus final : public AudioEncoder {
uint32_t first_timestamp_in_buffer_;
size_t num_channels_to_encode_;
int next_frame_length_ms_;
std::unique_ptr<PacketLossFractionSmoother> packet_loss_fraction_smoother_;
AudioNetworkAdaptorCreator audio_network_adaptor_creator_;
std::unique_ptr<AudioNetworkAdaptor> audio_network_adaptor_;

32
webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc
View File

@ -15,6 +15,7 @@
#include "webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h"
#include "webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h"
#include "webrtc/test/gtest.h"
#include "webrtc/system_wrappers/include/clock.h"
namespace webrtc {
using ::testing::NiceMock;
@ -23,6 +24,7 @@ using ::testing::Return;
namespace {
const CodecInst kDefaultOpusSettings = {105, "opus", 48000, 960, 1, 32000};
constexpr int64_t kInitialTimeUs = 12345678;
AudioEncoderOpus::Config CreateConfig(const CodecInst& codec_inst) {
AudioEncoderOpus::Config config;
@ -38,6 +40,7 @@ AudioEncoderOpus::Config CreateConfig(const CodecInst& codec_inst) {
struct AudioEncoderOpusStates {
std::shared_ptr<MockAudioNetworkAdaptor*> mock_audio_network_adaptor;
std::unique_ptr<AudioEncoderOpus> encoder;
std::unique_ptr<SimulatedClock> simulated_clock;
};
AudioEncoderOpusStates CreateCodec(size_t num_channels) {
@ -63,6 +66,9 @@ AudioEncoderOpusStates CreateCodec(size_t num_channels) {
CodecInst codec_inst = kDefaultOpusSettings;
codec_inst.channels = num_channels;
auto config = CreateConfig(codec_inst);
states.simulated_clock.reset(new SimulatedClock(kInitialTimeUs));
config.clock = states.simulated_clock.get();
states.encoder.reset(new AudioEncoderOpus(config, std::move(creator)));
return states;
}
@ -303,4 +309,30 @@ TEST(AudioEncoderOpusTest,
CheckEncoderRuntimeConfig(states.encoder.get(), config);
}
TEST(AudioEncoderOpusTest,
PacketLossFractionSmoothedOnSetUplinkPacketLossFraction) {
auto states = CreateCodec(2);
// The values are carefully chosen so that if no smoothing is made, the test
// will fail.
constexpr float kPacketLossFraction_1 = 0.02f;
constexpr float kPacketLossFraction_2 = 0.198f;
// |kSecondSampleTimeMs| is chose to ease the calculation since
// 0.9999 ^ 6931 = 0.5.
constexpr float kSecondSampleTimeMs = 6931;
// First time, no filtering.
states.encoder->OnReceivedUplinkPacketLossFraction(kPacketLossFraction_1);
EXPECT_DOUBLE_EQ(0.01, states.encoder->packet_loss_rate());
states.simulated_clock->AdvanceTimeMilliseconds(kSecondSampleTimeMs);
states.encoder->OnReceivedUplinkPacketLossFraction(kPacketLossFraction_2);
// Now the output of packet loss fraction smoother should be
// (0.02 + 0.198) / 2 = 0.109, which reach the threshold for the optimized
// packet loss rate to increase to 0.05. If no smoothing has been made, the
// optimized packet loss rate should have been increase to 0.1.
EXPECT_DOUBLE_EQ(0.05, states.encoder->packet_loss_rate());
}
} // namespace webrtc

5
webrtc/modules/audio_coding/include/audio_coding_module.h
View File

@ -252,6 +252,9 @@ class AudioCodingModule {
///////////////////////////////////////////////////////////////////////////
// Sets the bitrate to the specified value in bits/sec. If the value is not
// supported by the codec, it will choose another appropriate value.
//
// This is only used in test code that rely on old ACM APIs.
// TODO(minyue): Remove it when possible.
virtual void SetBitRate(int bitrate_bps) = 0;
// int32_t RegisterTransportCallback()
@ -371,6 +374,8 @@ class AudioCodingModule {
// -1 if failed to set packet loss rate,
// 0 if succeeded.
//
// This is only used in test code that rely on old ACM APIs.
// TODO(minyue): Remove it when possible.
virtual int SetPacketLossRate(int packet_loss_rate) = 0;
///////////////////////////////////////////////////////////////////////////

3
webrtc/voice_engine/BUILD.gn
View File

@ -30,8 +30,6 @@ rtc_static_library("voice_engine") {
"include/voe_volume_control.h",
"monitor_module.cc",
"monitor_module.h",
"network_predictor.cc",
"network_predictor.h",
"output_mixer.cc",
"output_mixer.h",
"shared_data.cc",
@ -206,7 +204,6 @@ if (rtc_include_tests) {
sources = [
"channel_unittest.cc",
"network_predictor_unittest.cc",
"test/channel_transport/udp_socket_manager_unittest.cc",
"test/channel_transport/udp_socket_wrapper_unittest.cc",
"test/channel_transport/udp_transport_unittest.cc",

52
webrtc/voice_engine/channel.cc
View File

@ -899,7 +899,6 @@ Channel::Channel(int32_t channelId,
_outputSpeechType(AudioFrame::kNormalSpeech),
restored_packet_in_use_(false),
rtcp_observer_(new VoERtcpObserver(this)),
network_predictor_(new NetworkPredictor(Clock::GetRealTimeClock())),
associate_send_channel_(ChannelOwner(nullptr)),
pacing_enabled_(config.enable_voice_pacing),
feedback_observer_proxy_(new TransportFeedbackProxy()),
@ -1331,19 +1330,18 @@ int32_t Channel::SetSendCodec(const CodecInst& codec) {
void Channel::SetBitRate(int bitrate_bps) {
WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, _channelId),
"Channel::SetBitRate(bitrate_bps=%d)", bitrate_bps);
audio_coding_->SetBitRate(bitrate_bps);
audio_coding_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder) {
if (*encoder)
(*encoder)->OnReceivedTargetAudioBitrate(bitrate_bps);
});
retransmission_rate_limiter_->SetMaxRate(bitrate_bps);
}
void Channel::OnIncomingFractionLoss(int fraction_lost) {
network_predictor_->UpdatePacketLossRate(fraction_lost);
uint8_t average_fraction_loss = network_predictor_->GetLossRate();
// Normalizes rate to 0 - 100.
if (audio_coding_->SetPacketLossRate(100 * average_fraction_loss / 255) !=
0) {
assert(false); // This should not happen.
}
audio_coding_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder) {
if (*encoder)
(*encoder)->OnReceivedUplinkPacketLossFraction(fraction_lost / 255.0f);
});
}
int32_t Channel::SetVADStatus(bool enableVAD,
@ -1540,6 +1538,34 @@ int Channel::GetOpusDtx(bool* enabled) {
return success;
}
bool Channel::EnableAudioNetworkAdaptor(const std::string& config_string) {
bool success = false;
audio_coding_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder) {
if (*encoder) {
success = (*encoder)->EnableAudioNetworkAdaptor(
config_string, Clock::GetRealTimeClock());
}
});
return success;
}
void Channel::DisableAudioNetworkAdaptor() {
audio_coding_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder) {
if (*encoder)
(*encoder)->DisableAudioNetworkAdaptor();
});
}
void Channel::SetReceiverFrameLengthRange(int min_frame_length_ms,
int max_frame_length_ms) {
audio_coding_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder) {
if (*encoder) {
(*encoder)->SetReceiverFrameLengthRange(min_frame_length_ms,
max_frame_length_ms);
}
});
}
int32_t Channel::RegisterExternalTransport(Transport* transport) {
WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, _channelId),
"Channel::RegisterExternalTransport()");
@ -1700,6 +1726,12 @@ int32_t Channel::ReceivedRTCPPacket(const uint8_t* data, size_t length) {
}
retransmission_rate_limiter_->SetWindowSize(nack_window_ms);
// Invoke audio encoders OnReceivedRtt().
audio_coding_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder) {
if (*encoder)
(*encoder)->OnReceivedRtt(rtt);
});
uint32_t ntp_secs = 0;
uint32_t ntp_frac = 0;
uint32_t rtp_timestamp = 0;

6
webrtc/voice_engine/channel.h
View File

@ -32,7 +32,6 @@
#include "webrtc/voice_engine/include/voe_base.h"
#include "webrtc/voice_engine/include/voe_network.h"
#include "webrtc/voice_engine/level_indicator.h"
#include "webrtc/voice_engine/network_predictor.h"
#include "webrtc/voice_engine/shared_data.h"
#include "webrtc/voice_engine/voice_engine_defines.h"
@ -209,6 +208,10 @@ class Channel
int SetOpusMaxPlaybackRate(int frequency_hz);
int SetOpusDtx(bool enable_dtx);
int GetOpusDtx(bool* enabled);
bool EnableAudioNetworkAdaptor(const std::string& config_string);
void DisableAudioNetworkAdaptor();
void SetReceiverFrameLengthRange(int min_frame_length_ms,
int max_frame_length_ms);
// VoENetwork
int32_t RegisterExternalTransport(Transport* transport);
@ -537,7 +540,6 @@ class Channel
bool restored_packet_in_use_;
// RtcpBandwidthObserver
std::unique_ptr<VoERtcpObserver> rtcp_observer_;
std::unique_ptr<NetworkPredictor> network_predictor_;
// An associated send channel.
rtc::CriticalSection assoc_send_channel_lock_;
ChannelOwner associate_send_channel_ GUARDED_BY(assoc_send_channel_lock_);

38
webrtc/voice_engine/network_predictor.cc
View File

@ -1,38 +0,0 @@
/*
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "webrtc/voice_engine/network_predictor.h"
namespace webrtc {
namespace voe {
NetworkPredictor::NetworkPredictor(Clock* clock)
: clock_(clock),
last_loss_rate_update_time_ms_(clock_->TimeInMilliseconds()),
loss_rate_filter_(new rtc::ExpFilter(0.9999f)) {
}
uint8_t NetworkPredictor::GetLossRate() {
float value = loss_rate_filter_->filtered();
return (value == rtc::ExpFilter::kValueUndefined) ? 0 :
static_cast<uint8_t>(value + 0.5);
}
void NetworkPredictor::UpdatePacketLossRate(uint8_t loss_rate) {
int64_t now_ms = clock_->TimeInMilliseconds();
// Update the recursive average filter.
loss_rate_filter_->Apply(
static_cast<float>(now_ms - last_loss_rate_update_time_ms_),
static_cast<float>(loss_rate));
last_loss_rate_update_time_ms_ = now_ms;
}
} // namespace voe
} // namespace webrtc

48
webrtc/voice_engine/network_predictor.h
View File

@ -1,48 +0,0 @@
/*
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef WEBRTC_VOICE_ENGINE_NETWORK_PREDICTOR_H_
#define WEBRTC_VOICE_ENGINE_NETWORK_PREDICTOR_H_
#include <memory>
#include "webrtc/base/exp_filter.h"
#include "webrtc/system_wrappers/include/clock.h"
namespace webrtc {
namespace voe {
// NetworkPredictor is to predict network conditions e.g., packet loss rate, for
// sender and/or receiver to cope with changes in the network condition.
class NetworkPredictor {
public:
explicit NetworkPredictor(Clock* clock);
~NetworkPredictor() {}
// Gets the predicted packet loss rate.
uint8_t GetLossRate();
// Updates the packet loss rate predictor, on receiving a new observation of
// packet loss rate from past. Input packet loss rate should be in the
// interval [0, 255].
void UpdatePacketLossRate(uint8_t loss_rate);
private:
Clock* clock_;
int64_t last_loss_rate_update_time_ms_;
// An exponential filter is used to predict packet loss rate.
std::unique_ptr<rtc::ExpFilter> loss_rate_filter_;
};
} // namespace voe
} // namespace webrtc
#endif // WEBRTC_VOICE_ENGINE_NETWORK_PREDICTOR_H_

45
webrtc/voice_engine/network_predictor_unittest.cc
View File

@ -1,45 +0,0 @@
/*
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <math.h>
#include <memory>
#include "webrtc/system_wrappers/include/clock.h"
#include "webrtc/test/gtest.h"
#include "webrtc/voice_engine/network_predictor.h"
namespace webrtc {
namespace voe {
class TestNetworkPredictor : public ::testing::Test {
protected:
TestNetworkPredictor()
: clock_(0),
network_predictor_(new NetworkPredictor(&clock_)) {}
SimulatedClock clock_;
std::unique_ptr<NetworkPredictor> network_predictor_;
};
TEST_F(TestNetworkPredictor, TestPacketLossRateFilter) {
// Test initial packet loss rate estimate is 0.
EXPECT_EQ(0, network_predictor_->GetLossRate());
network_predictor_->UpdatePacketLossRate(32);
// First time, no filtering.
EXPECT_EQ(32, network_predictor_->GetLossRate());
clock_.AdvanceTimeMilliseconds(1000);
network_predictor_->UpdatePacketLossRate(40);
float exp = pow(0.9999f, 1000);
float value = 32.0f * exp + (1 - exp) * 40.0f;
EXPECT_EQ(static_cast<uint8_t>(value + 0.5f),
network_predictor_->GetLossRate());
}
} // namespace voe
} // namespace webrtc

2
webrtc/voice_engine/voice_engine.gyp
View File

@ -56,8 +56,6 @@
'channel_proxy.h',
'monitor_module.cc',
'monitor_module.h',
'network_predictor.cc',
'network_predictor.h',
'output_mixer.cc',
'output_mixer.h',
'shared_data.cc',