Hooking up audio network adaptor to VoE.

BUG=webrtc:6303

Review-Url: https://codereview.webrtc.org/2390883004
Cr-Commit-Position: refs/heads/master@{#14611}

webrtc/modules/audio_coding/codecs/audio_encoder.cc

@@ -77,9 +77,13 @@ void AudioEncoder::DisableAudioNetworkAdaptor() {}
 void AudioEncoder::OnReceivedUplinkBandwidth(int uplink_bandwidth_bps) {}
 
 void AudioEncoder::OnReceivedUplinkPacketLossFraction(
-    float uplink_packet_loss_fraction) {}
+    float uplink_packet_loss_fraction) {
+  SetProjectedPacketLossRate(uplink_packet_loss_fraction);
+}
 
-void AudioEncoder::OnReceivedTargetAudioBitrate(int target_audio_bitrate_bps) {}
+void AudioEncoder::OnReceivedTargetAudioBitrate(int target_audio_bitrate_bps) {
+  SetTargetBitrate(target_audio_bitrate_bps);
+}
 
 void AudioEncoder::OnReceivedRtt(int rtt_ms) {}
 

webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc

@@ -13,6 +13,7 @@
 #include <algorithm>
 
 #include "webrtc/base/checks.h"
+#include "webrtc/base/exp_filter.h"
 #include "webrtc/base/safe_conversions.h"
 #include "webrtc/common_types.h"
 #include "webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h"
@@ -24,11 +25,15 @@ namespace webrtc {
 
 namespace {
 
-const int kSampleRateHz = 48000;
+constexpr int kSampleRateHz = 48000;
-const int kMinBitrateBps = 500;
+constexpr int kMinBitrateBps = 500;
-const int kMaxBitrateBps = 512000;
+constexpr int kMaxBitrateBps = 512000;
 constexpr int kSupportedFrameLengths[] = {20, 60};
 
+// PacketLossFractionSmoother uses an exponential filter with a time constant
+// of -1.0 / ln(0.9999) = 10000 ms.
+constexpr float kAlphaForPacketLossFractionSmoother = 0.9999f;
+
 AudioEncoderOpus::Config CreateConfig(const CodecInst& codec_inst) {
   AudioEncoderOpus::Config config;
   config.frame_size_ms = rtc::CheckedDivExact(codec_inst.pacsize, 48);
@@ -82,6 +87,35 @@ double OptimizePacketLossRate(double new_loss_rate, double old_loss_rate) {
 
 }  // namespace
 
+class AudioEncoderOpus::PacketLossFractionSmoother {
+ public:
+  explicit PacketLossFractionSmoother(const Clock* clock)
+      : clock_(clock),
+        last_sample_time_ms_(clock_->TimeInMilliseconds()),
+        smoother_(kAlphaForPacketLossFractionSmoother) {}
+
+  // Gets the smoothed packet loss fraction.
+  float GetAverage() const {
+    float value = smoother_.filtered();
+    return (value == rtc::ExpFilter::kValueUndefined) ? 0.0f : value;
+  }
+
+  // Add new observation to the packet loss fraction smoother.
+  void AddSample(float packet_loss_fraction) {
+    int64_t now_ms = clock_->TimeInMilliseconds();
+    smoother_.Apply(static_cast<float>(now_ms - last_sample_time_ms_),
+                    packet_loss_fraction);
+    last_sample_time_ms_ = now_ms;
+  }
+
+ private:
+  const Clock* const clock_;
+  int64_t last_sample_time_ms_;
+
+  // An exponential filter is used to smooth the packet loss fraction.
+  rtc::ExpFilter smoother_;
+};
+
 AudioEncoderOpus::Config::Config() = default;
 AudioEncoderOpus::Config::Config(const Config&) = default;
 AudioEncoderOpus::Config::~Config() = default;
@@ -113,9 +147,11 @@ AudioEncoderOpus::AudioEncoderOpus(
     AudioNetworkAdaptorCreator&& audio_network_adaptor_creator)
     : packet_loss_rate_(0.0),
       inst_(nullptr),
+      packet_loss_fraction_smoother_(new PacketLossFractionSmoother(
+          config.clock ? config.clock : Clock::GetRealTimeClock())),
       audio_network_adaptor_creator_(
           audio_network_adaptor_creator
-              ? audio_network_adaptor_creator
+              ? std::move(audio_network_adaptor_creator)
               : [this](const std::string& config_string, const Clock* clock) {
                   return DefaultAudioNetworkAdaptorCreator(config_string,
                                                            clock);
@@ -234,8 +270,11 @@ void AudioEncoderOpus::OnReceivedUplinkBandwidth(int uplink_bandwidth_bps) {
 
 void AudioEncoderOpus::OnReceivedUplinkPacketLossFraction(
     float uplink_packet_loss_fraction) {
-  if (!audio_network_adaptor_)
+  if (!audio_network_adaptor_) {
-    return;
+    packet_loss_fraction_smoother_->AddSample(uplink_packet_loss_fraction);
+    float average_fraction_loss = packet_loss_fraction_smoother_->GetAverage();
+    return SetProjectedPacketLossRate(average_fraction_loss);
+  }
   audio_network_adaptor_->SetUplinkPacketLossFraction(
       uplink_packet_loss_fraction);
   ApplyAudioNetworkAdaptor();
@@ -244,7 +283,7 @@ void AudioEncoderOpus::OnReceivedUplinkPacketLossFraction(
 void AudioEncoderOpus::OnReceivedTargetAudioBitrate(
     int target_audio_bitrate_bps) {
   if (!audio_network_adaptor_)
-    return;
+    return SetTargetBitrate(target_audio_bitrate_bps);
   audio_network_adaptor_->SetTargetAudioBitrate(target_audio_bitrate_bps);
   ApplyAudioNetworkAdaptor();
 }

webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h

@@ -49,6 +49,7 @@ class AudioEncoderOpus final : public AudioEncoder {
     int max_playback_rate_hz = 48000;
     int complexity = kDefaultComplexity;
     bool dtx_enabled = false;
+    const Clock* clock = nullptr;
 
    private:
 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) || defined(WEBRTC_ARCH_ARM)
@@ -115,6 +116,8 @@ class AudioEncoderOpus final : public AudioEncoder {
                          rtc::Buffer* encoded) override;
 
  private:
+  class PacketLossFractionSmoother;
+
   size_t Num10msFramesPerPacket() const;
   size_t SamplesPer10msFrame() const;
   size_t SufficientOutputBufferSize() const;
@@ -133,6 +136,7 @@ class AudioEncoderOpus final : public AudioEncoder {
   uint32_t first_timestamp_in_buffer_;
   size_t num_channels_to_encode_;
   int next_frame_length_ms_;
+  std::unique_ptr<PacketLossFractionSmoother> packet_loss_fraction_smoother_;
   AudioNetworkAdaptorCreator audio_network_adaptor_creator_;
   std::unique_ptr<AudioNetworkAdaptor> audio_network_adaptor_;
 

webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc

@@ -15,6 +15,7 @@
 #include "webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h"
 #include "webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h"
 #include "webrtc/test/gtest.h"
+#include "webrtc/system_wrappers/include/clock.h"
 
 namespace webrtc {
 using ::testing::NiceMock;
@@ -23,6 +24,7 @@ using ::testing::Return;
 namespace {
 
 const CodecInst kDefaultOpusSettings = {105, "opus", 48000, 960, 1, 32000};
+constexpr int64_t kInitialTimeUs = 12345678;
 
 AudioEncoderOpus::Config CreateConfig(const CodecInst& codec_inst) {
   AudioEncoderOpus::Config config;
@@ -38,6 +40,7 @@ AudioEncoderOpus::Config CreateConfig(const CodecInst& codec_inst) {
 struct AudioEncoderOpusStates {
   std::shared_ptr<MockAudioNetworkAdaptor*> mock_audio_network_adaptor;
   std::unique_ptr<AudioEncoderOpus> encoder;
+  std::unique_ptr<SimulatedClock> simulated_clock;
 };
 
 AudioEncoderOpusStates CreateCodec(size_t num_channels) {
@@ -63,6 +66,9 @@ AudioEncoderOpusStates CreateCodec(size_t num_channels) {
   CodecInst codec_inst = kDefaultOpusSettings;
   codec_inst.channels = num_channels;
   auto config = CreateConfig(codec_inst);
+  states.simulated_clock.reset(new SimulatedClock(kInitialTimeUs));
+  config.clock = states.simulated_clock.get();
+
   states.encoder.reset(new AudioEncoderOpus(config, std::move(creator)));
   return states;
 }
@@ -303,4 +309,30 @@ TEST(AudioEncoderOpusTest,
   CheckEncoderRuntimeConfig(states.encoder.get(), config);
 }
 
+TEST(AudioEncoderOpusTest,
+     PacketLossFractionSmoothedOnSetUplinkPacketLossFraction) {
+  auto states = CreateCodec(2);
+
+  // The values are carefully chosen so that if no smoothing is made, the test
+  // will fail.
+  constexpr float kPacketLossFraction_1 = 0.02f;
+  constexpr float kPacketLossFraction_2 = 0.198f;
+  // |kSecondSampleTimeMs| is chose to ease the calculation since
+  // 0.9999 ^ 6931 = 0.5.
+  constexpr float kSecondSampleTimeMs = 6931;
+
+  // First time, no filtering.
+  states.encoder->OnReceivedUplinkPacketLossFraction(kPacketLossFraction_1);
+  EXPECT_DOUBLE_EQ(0.01, states.encoder->packet_loss_rate());
+
+  states.simulated_clock->AdvanceTimeMilliseconds(kSecondSampleTimeMs);
+  states.encoder->OnReceivedUplinkPacketLossFraction(kPacketLossFraction_2);
+
+  // Now the output of packet loss fraction smoother should be
+  // (0.02 + 0.198) / 2 = 0.109, which reach the threshold for the optimized
+  // packet loss rate to increase to 0.05. If no smoothing has been made, the
+  // optimized packet loss rate should have been increase to 0.1.
+  EXPECT_DOUBLE_EQ(0.05, states.encoder->packet_loss_rate());
+}
+
 }  // namespace webrtc

webrtc/modules/audio_coding/include/audio_coding_module.h

@@ -252,6 +252,9 @@ class AudioCodingModule {
   ///////////////////////////////////////////////////////////////////////////
   // Sets the bitrate to the specified value in bits/sec. If the value is not
   // supported by the codec, it will choose another appropriate value.
+  //
+  // This is only used in test code that rely on old ACM APIs.
+  // TODO(minyue): Remove it when possible.
   virtual void SetBitRate(int bitrate_bps) = 0;
 
   // int32_t RegisterTransportCallback()
@@ -371,6 +374,8 @@ class AudioCodingModule {
   //   -1 if failed to set packet loss rate,
   //   0 if succeeded.
   //
+  // This is only used in test code that rely on old ACM APIs.
+  // TODO(minyue): Remove it when possible.
   virtual int SetPacketLossRate(int packet_loss_rate) = 0;
 
   ///////////////////////////////////////////////////////////////////////////

webrtc/voice_engine/BUILD.gn

@@ -30,8 +30,6 @@ rtc_static_library("voice_engine") {
     "include/voe_volume_control.h",
     "monitor_module.cc",
     "monitor_module.h",
-    "network_predictor.cc",
-    "network_predictor.h",
     "output_mixer.cc",
     "output_mixer.h",
     "shared_data.cc",
@@ -206,7 +204,6 @@ if (rtc_include_tests) {
 
     sources = [
       "channel_unittest.cc",
-      "network_predictor_unittest.cc",
       "test/channel_transport/udp_socket_manager_unittest.cc",
       "test/channel_transport/udp_socket_wrapper_unittest.cc",
       "test/channel_transport/udp_transport_unittest.cc",

webrtc/voice_engine/channel.cc

@@ -899,7 +899,6 @@ Channel::Channel(int32_t channelId,
       _outputSpeechType(AudioFrame::kNormalSpeech),
       restored_packet_in_use_(false),
       rtcp_observer_(new VoERtcpObserver(this)),
-      network_predictor_(new NetworkPredictor(Clock::GetRealTimeClock())),
       associate_send_channel_(ChannelOwner(nullptr)),
       pacing_enabled_(config.enable_voice_pacing),
       feedback_observer_proxy_(new TransportFeedbackProxy()),
@@ -1331,19 +1330,18 @@ int32_t Channel::SetSendCodec(const CodecInst& codec) {
 void Channel::SetBitRate(int bitrate_bps) {
   WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, _channelId),
                "Channel::SetBitRate(bitrate_bps=%d)", bitrate_bps);
-  audio_coding_->SetBitRate(bitrate_bps);
+  audio_coding_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder) {
+    if (*encoder)
+      (*encoder)->OnReceivedTargetAudioBitrate(bitrate_bps);
+  });
   retransmission_rate_limiter_->SetMaxRate(bitrate_bps);
 }
 
 void Channel::OnIncomingFractionLoss(int fraction_lost) {
-  network_predictor_->UpdatePacketLossRate(fraction_lost);
+  audio_coding_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder) {
-  uint8_t average_fraction_loss = network_predictor_->GetLossRate();
+    if (*encoder)
-
+      (*encoder)->OnReceivedUplinkPacketLossFraction(fraction_lost / 255.0f);
-  // Normalizes rate to 0 - 100.
+  });
-  if (audio_coding_->SetPacketLossRate(100 * average_fraction_loss / 255) !=
-      0) {
-    assert(false);  // This should not happen.
-  }
 }
 
 int32_t Channel::SetVADStatus(bool enableVAD,
@@ -1540,6 +1538,34 @@ int Channel::GetOpusDtx(bool* enabled) {
   return success;
 }
 
+bool Channel::EnableAudioNetworkAdaptor(const std::string& config_string) {
+  bool success = false;
+  audio_coding_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder) {
+    if (*encoder) {
+      success = (*encoder)->EnableAudioNetworkAdaptor(
+          config_string, Clock::GetRealTimeClock());
+    }
+  });
+  return success;
+}
+
+void Channel::DisableAudioNetworkAdaptor() {
+  audio_coding_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder) {
+    if (*encoder)
+      (*encoder)->DisableAudioNetworkAdaptor();
+  });
+}
+
+void Channel::SetReceiverFrameLengthRange(int min_frame_length_ms,
+                                          int max_frame_length_ms) {
+  audio_coding_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder) {
+    if (*encoder) {
+      (*encoder)->SetReceiverFrameLengthRange(min_frame_length_ms,
+                                              max_frame_length_ms);
+    }
+  });
+}
+
 int32_t Channel::RegisterExternalTransport(Transport* transport) {
   WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, _channelId),
                "Channel::RegisterExternalTransport()");
@@ -1700,6 +1726,12 @@ int32_t Channel::ReceivedRTCPPacket(const uint8_t* data, size_t length) {
   }
   retransmission_rate_limiter_->SetWindowSize(nack_window_ms);
 
+  // Invoke audio encoders OnReceivedRtt().
+  audio_coding_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder) {
+    if (*encoder)
+      (*encoder)->OnReceivedRtt(rtt);
+  });
+
   uint32_t ntp_secs = 0;
   uint32_t ntp_frac = 0;
   uint32_t rtp_timestamp = 0;

webrtc/voice_engine/channel.h

@@ -32,7 +32,6 @@
 #include "webrtc/voice_engine/include/voe_base.h"
 #include "webrtc/voice_engine/include/voe_network.h"
 #include "webrtc/voice_engine/level_indicator.h"
-#include "webrtc/voice_engine/network_predictor.h"
 #include "webrtc/voice_engine/shared_data.h"
 #include "webrtc/voice_engine/voice_engine_defines.h"
 
@@ -209,6 +208,10 @@ class Channel
   int SetOpusMaxPlaybackRate(int frequency_hz);
   int SetOpusDtx(bool enable_dtx);
   int GetOpusDtx(bool* enabled);
+  bool EnableAudioNetworkAdaptor(const std::string& config_string);
+  void DisableAudioNetworkAdaptor();
+  void SetReceiverFrameLengthRange(int min_frame_length_ms,
+                                   int max_frame_length_ms);
 
   // VoENetwork
   int32_t RegisterExternalTransport(Transport* transport);
@@ -537,7 +540,6 @@ class Channel
   bool restored_packet_in_use_;
   // RtcpBandwidthObserver
   std::unique_ptr<VoERtcpObserver> rtcp_observer_;
-  std::unique_ptr<NetworkPredictor> network_predictor_;
   // An associated send channel.
   rtc::CriticalSection assoc_send_channel_lock_;
   ChannelOwner associate_send_channel_ GUARDED_BY(assoc_send_channel_lock_);

webrtc/voice_engine/network_predictor.cc

@@ -1,38 +0,0 @@
-/*
- *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
- *
- *  Use of this source code is governed by a BSD-style license
- *  that can be found in the LICENSE file in the root of the source
- *  tree. An additional intellectual property rights grant can be found
- *  in the file PATENTS.  All contributing project authors may
- *  be found in the AUTHORS file in the root of the source tree.
- */
-
-#include "webrtc/voice_engine/network_predictor.h"
-
-namespace webrtc {
-namespace voe {
-
-NetworkPredictor::NetworkPredictor(Clock* clock)
-    : clock_(clock),
-      last_loss_rate_update_time_ms_(clock_->TimeInMilliseconds()),
-      loss_rate_filter_(new rtc::ExpFilter(0.9999f)) {
-}
-
-uint8_t NetworkPredictor::GetLossRate() {
-  float value = loss_rate_filter_->filtered();
-  return (value == rtc::ExpFilter::kValueUndefined) ? 0 :
-      static_cast<uint8_t>(value + 0.5);
-}
-
-void NetworkPredictor::UpdatePacketLossRate(uint8_t loss_rate) {
-  int64_t now_ms = clock_->TimeInMilliseconds();
-  // Update the recursive average filter.
-  loss_rate_filter_->Apply(
-      static_cast<float>(now_ms - last_loss_rate_update_time_ms_),
-      static_cast<float>(loss_rate));
-  last_loss_rate_update_time_ms_ = now_ms;
-}
-
-}  // namespace voe
-}  // namespace webrtc

webrtc/voice_engine/network_predictor.h

@@ -1,48 +0,0 @@
-/*
- *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
- *
- *  Use of this source code is governed by a BSD-style license
- *  that can be found in the LICENSE file in the root of the source
- *  tree. An additional intellectual property rights grant can be found
- *  in the file PATENTS.  All contributing project authors may
- *  be found in the AUTHORS file in the root of the source tree.
- */
-
-#ifndef WEBRTC_VOICE_ENGINE_NETWORK_PREDICTOR_H_
-#define WEBRTC_VOICE_ENGINE_NETWORK_PREDICTOR_H_
-
-#include <memory>
-
-#include "webrtc/base/exp_filter.h"
-#include "webrtc/system_wrappers/include/clock.h"
-
-namespace webrtc {
-
-namespace voe {
-
-// NetworkPredictor is to predict network conditions e.g., packet loss rate, for
-// sender and/or receiver to cope with changes in the network condition.
-class NetworkPredictor {
- public:
-  explicit NetworkPredictor(Clock* clock);
-  ~NetworkPredictor() {}
-
-  // Gets the predicted packet loss rate.
-  uint8_t GetLossRate();
-
-  // Updates the packet loss rate predictor, on receiving a new observation of
-  // packet loss rate from past. Input packet loss rate should be in the
-  // interval [0, 255].
-  void UpdatePacketLossRate(uint8_t loss_rate);
-
- private:
-  Clock* clock_;
-  int64_t last_loss_rate_update_time_ms_;
-
-  // An exponential filter is used to predict packet loss rate.
-  std::unique_ptr<rtc::ExpFilter> loss_rate_filter_;
-};
-
-}  // namespace voe
-}  // namespace webrtc
-#endif  // WEBRTC_VOICE_ENGINE_NETWORK_PREDICTOR_H_

webrtc/voice_engine/network_predictor_unittest.cc

@@ -1,45 +0,0 @@
-/*
- *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
- *
- *  Use of this source code is governed by a BSD-style license
- *  that can be found in the LICENSE file in the root of the source
- *  tree. An additional intellectual property rights grant can be found
- *  in the file PATENTS.  All contributing project authors may
- *  be found in the AUTHORS file in the root of the source tree.
- */
-
-#include <math.h>
-
-#include <memory>
-
-#include "webrtc/system_wrappers/include/clock.h"
-#include "webrtc/test/gtest.h"
-#include "webrtc/voice_engine/network_predictor.h"
-
-namespace webrtc {
-namespace voe {
-
-class TestNetworkPredictor : public ::testing::Test {
- protected:
-  TestNetworkPredictor()
-      : clock_(0),
-        network_predictor_(new NetworkPredictor(&clock_)) {}
-  SimulatedClock clock_;
-  std::unique_ptr<NetworkPredictor> network_predictor_;
-};
-
-TEST_F(TestNetworkPredictor, TestPacketLossRateFilter) {
-  // Test initial packet loss rate estimate is 0.
-  EXPECT_EQ(0, network_predictor_->GetLossRate());
-  network_predictor_->UpdatePacketLossRate(32);
-  // First time, no filtering.
-  EXPECT_EQ(32, network_predictor_->GetLossRate());
-  clock_.AdvanceTimeMilliseconds(1000);
-  network_predictor_->UpdatePacketLossRate(40);
-  float exp = pow(0.9999f, 1000);
-  float value = 32.0f * exp + (1 - exp) * 40.0f;
-  EXPECT_EQ(static_cast<uint8_t>(value + 0.5f),
-            network_predictor_->GetLossRate());
-}
-}  // namespace voe
-}  // namespace webrtc

webrtc/voice_engine/voice_engine.gyp

@@ -56,8 +56,6 @@
         'channel_proxy.h',
         'monitor_module.cc',
         'monitor_module.h',
-        'network_predictor.cc',
-        'network_predictor.h',
         'output_mixer.cc',
         'output_mixer.h',
         'shared_data.cc',