2017-03-07 01:58:55 -08:00
|
|
|
/*
|
|
|
|
|
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
|
|
|
|
|
*
|
|
|
|
|
* Use of this source code is governed by a BSD-style license
|
|
|
|
|
* that can be found in the LICENSE file in the root of the source
|
|
|
|
|
* tree. An additional intellectual property rights grant can be found
|
|
|
|
|
* in the file PATENTS. All contributing project authors may
|
|
|
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
|
|
|
*/
|
|
|
|
|
|
2018-01-17 11:18:31 +01:00
|
|
|
#include "audio/audio_level.h"
|
2017-03-07 01:58:55 -08:00
|
|
|
|
2018-04-12 22:44:09 +02:00
|
|
|
#include "api/audio/audio_frame.h"
|
2017-09-15 06:47:31 +02:00
|
|
|
#include "common_audio/signal_processing/include/signal_processing_library.h"
|
2017-03-07 01:58:55 -08:00
|
|
|
|
|
|
|
|
namespace webrtc {
|
|
|
|
|
namespace voe {
|
|
|
|
|
|
|
|
|
|
AudioLevel::AudioLevel()
|
2018-01-30 09:33:03 +01:00
|
|
|
: abs_max_(0), count_(0), current_level_full_range_(0) {}
|
2017-03-07 01:58:55 -08:00
|
|
|
|
|
|
|
|
AudioLevel::~AudioLevel() {}
|
|
|
|
|
|
[getStats] Implement "media-source" audio levels, fixing Chrome bug.
Implements RTCAudioSourceStats members:
- audioLevel
- totalAudioEnergy
- totalSamplesDuration
In this CL description these are collectively referred to as the audio
levels.
The audio levels are removed from sending "track" stats (in Chrome,
these are now reported as undefined instead of 0).
Background:
For sending tracks, audio levels were always reported as 0 in Chrome
(https://crbug.com/736403), while audio levels were correctly reported
for receiving tracks. This problem affected the standard getStats() but
not the legacy getStats(), blocking some people from migrating. This
was likely not a problem in native third_party/webrtc code because the
delivery of audio frames from device to send-stream uses a different
code path outside of chromium.
A recent PR (https://github.com/w3c/webrtc-stats/pull/451) moved the
send-side audio levels to the RTCAudioSourceStats, while keeping the
receive-side audio levels on the "track" stats. This allows an
implementation to report the audio levels even if samples are not sent
onto the network (such as if an ICE connection has not been established
yet), reflecting some of the current implementation.
Changes:
1. Audio levels are added to RTCAudioSourceStats. Send-side audio
"track" stats are left undefined. Receive-side audio "track" stats
are not changed in this CL and continue to work.
2. Audio level computation is moved from the AudioState and
AudioTransportImpl to the AudioSendStream. This is because a) the
AudioTransportImpl::RecordedDataIsAvailable() code path is not
exercised in chromium, and b) audio levels should, per-spec, not be
calculated on a per-call basis, for which the AudioState is defined.
3. The audio level computation is now performed in
AudioSendStream::SendAudioData(), a code path used by both native
and chromium code.
4. Comments are added to document behavior of existing code, such as
AudioLevel and AudioSendStream::SendAudioData().
Note:
In this CL, just like before this CL, audio level is only calculated
after an AudioSendStream has been created. This means that before an
O/A negotiation, audio levels are unavailable.
According to spec, if we have an audio source, we should have audio
levels. An immediate solution to this would have been to calculate the
audio level at pc/rtp_sender.cc. The problem is that the
LocalAudioSinkAdapter::OnData() code path, while exercised in chromium,
is not exercised in native code. The issue of calculating audio levels
on a per-source bases rather than on a per-send stream basis is left to
https://crbug.com/webrtc/10771, an existing "media-source" bug.
This CL can be verified manually in Chrome at:
https://codepen.io/anon/pen/vqRGyq
Bug: chromium:736403, webrtc:10771
Change-Id: I8036cd9984f3b187c3177470a8c0d6670a201a5a
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/143789
Reviewed-by: Oskar Sundbom <ossu@webrtc.org>
Reviewed-by: Stefan Holmer <stefan@webrtc.org>
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#28480}
2019-07-03 17:11:10 +02:00
|
|
|
void AudioLevel::Reset() {
|
2020-07-06 15:15:07 +02:00
|
|
|
MutexLock lock(&mutex_);
|
[getStats] Implement "media-source" audio levels, fixing Chrome bug.
Implements RTCAudioSourceStats members:
- audioLevel
- totalAudioEnergy
- totalSamplesDuration
In this CL description these are collectively referred to as the audio
levels.
The audio levels are removed from sending "track" stats (in Chrome,
these are now reported as undefined instead of 0).
Background:
For sending tracks, audio levels were always reported as 0 in Chrome
(https://crbug.com/736403), while audio levels were correctly reported
for receiving tracks. This problem affected the standard getStats() but
not the legacy getStats(), blocking some people from migrating. This
was likely not a problem in native third_party/webrtc code because the
delivery of audio frames from device to send-stream uses a different
code path outside of chromium.
A recent PR (https://github.com/w3c/webrtc-stats/pull/451) moved the
send-side audio levels to the RTCAudioSourceStats, while keeping the
receive-side audio levels on the "track" stats. This allows an
implementation to report the audio levels even if samples are not sent
onto the network (such as if an ICE connection has not been established
yet), reflecting some of the current implementation.
Changes:
1. Audio levels are added to RTCAudioSourceStats. Send-side audio
"track" stats are left undefined. Receive-side audio "track" stats
are not changed in this CL and continue to work.
2. Audio level computation is moved from the AudioState and
AudioTransportImpl to the AudioSendStream. This is because a) the
AudioTransportImpl::RecordedDataIsAvailable() code path is not
exercised in chromium, and b) audio levels should, per-spec, not be
calculated on a per-call basis, for which the AudioState is defined.
3. The audio level computation is now performed in
AudioSendStream::SendAudioData(), a code path used by both native
and chromium code.
4. Comments are added to document behavior of existing code, such as
AudioLevel and AudioSendStream::SendAudioData().
Note:
In this CL, just like before this CL, audio level is only calculated
after an AudioSendStream has been created. This means that before an
O/A negotiation, audio levels are unavailable.
According to spec, if we have an audio source, we should have audio
levels. An immediate solution to this would have been to calculate the
audio level at pc/rtp_sender.cc. The problem is that the
LocalAudioSinkAdapter::OnData() code path, while exercised in chromium,
is not exercised in native code. The issue of calculating audio levels
on a per-source bases rather than on a per-send stream basis is left to
https://crbug.com/webrtc/10771, an existing "media-source" bug.
This CL can be verified manually in Chrome at:
https://codepen.io/anon/pen/vqRGyq
Bug: chromium:736403, webrtc:10771
Change-Id: I8036cd9984f3b187c3177470a8c0d6670a201a5a
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/143789
Reviewed-by: Oskar Sundbom <ossu@webrtc.org>
Reviewed-by: Stefan Holmer <stefan@webrtc.org>
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#28480}
2019-07-03 17:11:10 +02:00
|
|
|
abs_max_ = 0;
|
|
|
|
|
count_ = 0;
|
|
|
|
|
current_level_full_range_ = 0;
|
|
|
|
|
total_energy_ = 0.0;
|
|
|
|
|
total_duration_ = 0.0;
|
|
|
|
|
}
|
|
|
|
|
|
2017-03-07 01:58:55 -08:00
|
|
|
int16_t AudioLevel::LevelFullRange() const {
|
2020-07-06 15:15:07 +02:00
|
|
|
MutexLock lock(&mutex_);
|
2017-03-07 01:58:55 -08:00
|
|
|
return current_level_full_range_;
|
|
|
|
|
}
|
|
|
|
|
|
[getStats] Implement "media-source" audio levels, fixing Chrome bug.
Implements RTCAudioSourceStats members:
- audioLevel
- totalAudioEnergy
- totalSamplesDuration
In this CL description these are collectively referred to as the audio
levels.
The audio levels are removed from sending "track" stats (in Chrome,
these are now reported as undefined instead of 0).
Background:
For sending tracks, audio levels were always reported as 0 in Chrome
(https://crbug.com/736403), while audio levels were correctly reported
for receiving tracks. This problem affected the standard getStats() but
not the legacy getStats(), blocking some people from migrating. This
was likely not a problem in native third_party/webrtc code because the
delivery of audio frames from device to send-stream uses a different
code path outside of chromium.
A recent PR (https://github.com/w3c/webrtc-stats/pull/451) moved the
send-side audio levels to the RTCAudioSourceStats, while keeping the
receive-side audio levels on the "track" stats. This allows an
implementation to report the audio levels even if samples are not sent
onto the network (such as if an ICE connection has not been established
yet), reflecting some of the current implementation.
Changes:
1. Audio levels are added to RTCAudioSourceStats. Send-side audio
"track" stats are left undefined. Receive-side audio "track" stats
are not changed in this CL and continue to work.
2. Audio level computation is moved from the AudioState and
AudioTransportImpl to the AudioSendStream. This is because a) the
AudioTransportImpl::RecordedDataIsAvailable() code path is not
exercised in chromium, and b) audio levels should, per-spec, not be
calculated on a per-call basis, for which the AudioState is defined.
3. The audio level computation is now performed in
AudioSendStream::SendAudioData(), a code path used by both native
and chromium code.
4. Comments are added to document behavior of existing code, such as
AudioLevel and AudioSendStream::SendAudioData().
Note:
In this CL, just like before this CL, audio level is only calculated
after an AudioSendStream has been created. This means that before an
O/A negotiation, audio levels are unavailable.
According to spec, if we have an audio source, we should have audio
levels. An immediate solution to this would have been to calculate the
audio level at pc/rtp_sender.cc. The problem is that the
LocalAudioSinkAdapter::OnData() code path, while exercised in chromium,
is not exercised in native code. The issue of calculating audio levels
on a per-source bases rather than on a per-send stream basis is left to
https://crbug.com/webrtc/10771, an existing "media-source" bug.
This CL can be verified manually in Chrome at:
https://codepen.io/anon/pen/vqRGyq
Bug: chromium:736403, webrtc:10771
Change-Id: I8036cd9984f3b187c3177470a8c0d6670a201a5a
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/143789
Reviewed-by: Oskar Sundbom <ossu@webrtc.org>
Reviewed-by: Stefan Holmer <stefan@webrtc.org>
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#28480}
2019-07-03 17:11:10 +02:00
|
|
|
void AudioLevel::ResetLevelFullRange() {
|
2020-07-06 15:15:07 +02:00
|
|
|
MutexLock lock(&mutex_);
|
2017-03-07 01:58:55 -08:00
|
|
|
abs_max_ = 0;
|
|
|
|
|
count_ = 0;
|
|
|
|
|
current_level_full_range_ = 0;
|
|
|
|
|
}
|
|
|
|
|
|
2017-07-20 09:57:42 -07:00
|
|
|
double AudioLevel::TotalEnergy() const {
|
2020-07-06 15:15:07 +02:00
|
|
|
MutexLock lock(&mutex_);
|
2017-07-20 09:57:42 -07:00
|
|
|
return total_energy_;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
double AudioLevel::TotalDuration() const {
|
2020-07-06 15:15:07 +02:00
|
|
|
MutexLock lock(&mutex_);
|
2017-07-20 09:57:42 -07:00
|
|
|
return total_duration_;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void AudioLevel::ComputeLevel(const AudioFrame& audioFrame, double duration) {
|
2017-03-07 01:58:55 -08:00
|
|
|
// Check speech level (works for 2 channels as well)
|
2017-06-12 12:45:32 -07:00
|
|
|
int16_t abs_value =
|
|
|
|
|
audioFrame.muted()
|
|
|
|
|
? 0
|
|
|
|
|
: WebRtcSpl_MaxAbsValueW16(
|
|
|
|
|
audioFrame.data(),
|
|
|
|
|
audioFrame.samples_per_channel_ * audioFrame.num_channels_);
|
2017-03-07 01:58:55 -08:00
|
|
|
|
|
|
|
|
// Protect member access using a lock since this method is called on a
|
|
|
|
|
// dedicated audio thread in the RecordedDataIsAvailable() callback.
|
2020-07-06 15:15:07 +02:00
|
|
|
MutexLock lock(&mutex_);
|
2017-03-07 01:58:55 -08:00
|
|
|
|
|
|
|
|
if (abs_value > abs_max_)
|
|
|
|
|
abs_max_ = abs_value;
|
|
|
|
|
|
[getStats] Implement "media-source" audio levels, fixing Chrome bug.
Implements RTCAudioSourceStats members:
- audioLevel
- totalAudioEnergy
- totalSamplesDuration
In this CL description these are collectively referred to as the audio
levels.
The audio levels are removed from sending "track" stats (in Chrome,
these are now reported as undefined instead of 0).
Background:
For sending tracks, audio levels were always reported as 0 in Chrome
(https://crbug.com/736403), while audio levels were correctly reported
for receiving tracks. This problem affected the standard getStats() but
not the legacy getStats(), blocking some people from migrating. This
was likely not a problem in native third_party/webrtc code because the
delivery of audio frames from device to send-stream uses a different
code path outside of chromium.
A recent PR (https://github.com/w3c/webrtc-stats/pull/451) moved the
send-side audio levels to the RTCAudioSourceStats, while keeping the
receive-side audio levels on the "track" stats. This allows an
implementation to report the audio levels even if samples are not sent
onto the network (such as if an ICE connection has not been established
yet), reflecting some of the current implementation.
Changes:
1. Audio levels are added to RTCAudioSourceStats. Send-side audio
"track" stats are left undefined. Receive-side audio "track" stats
are not changed in this CL and continue to work.
2. Audio level computation is moved from the AudioState and
AudioTransportImpl to the AudioSendStream. This is because a) the
AudioTransportImpl::RecordedDataIsAvailable() code path is not
exercised in chromium, and b) audio levels should, per-spec, not be
calculated on a per-call basis, for which the AudioState is defined.
3. The audio level computation is now performed in
AudioSendStream::SendAudioData(), a code path used by both native
and chromium code.
4. Comments are added to document behavior of existing code, such as
AudioLevel and AudioSendStream::SendAudioData().
Note:
In this CL, just like before this CL, audio level is only calculated
after an AudioSendStream has been created. This means that before an
O/A negotiation, audio levels are unavailable.
According to spec, if we have an audio source, we should have audio
levels. An immediate solution to this would have been to calculate the
audio level at pc/rtp_sender.cc. The problem is that the
LocalAudioSinkAdapter::OnData() code path, while exercised in chromium,
is not exercised in native code. The issue of calculating audio levels
on a per-source bases rather than on a per-send stream basis is left to
https://crbug.com/webrtc/10771, an existing "media-source" bug.
This CL can be verified manually in Chrome at:
https://codepen.io/anon/pen/vqRGyq
Bug: chromium:736403, webrtc:10771
Change-Id: I8036cd9984f3b187c3177470a8c0d6670a201a5a
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/143789
Reviewed-by: Oskar Sundbom <ossu@webrtc.org>
Reviewed-by: Stefan Holmer <stefan@webrtc.org>
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#28480}
2019-07-03 17:11:10 +02:00
|
|
|
// Update level approximately 9 times per second, assuming audio frame
|
|
|
|
|
// duration is approximately 10 ms. (The update frequency is every
|
|
|
|
|
// 11th (= |kUpdateFrequency+1|) call: 1000/(11*10)=9.09..., we should
|
|
|
|
|
// probably change this behavior, see https://crbug.com/webrtc/10784).
|
2017-03-07 01:58:55 -08:00
|
|
|
if (count_++ == kUpdateFrequency) {
|
|
|
|
|
current_level_full_range_ = abs_max_;
|
|
|
|
|
|
|
|
|
|
count_ = 0;
|
|
|
|
|
|
|
|
|
|
// Decay the absolute maximum (divide by 4)
|
|
|
|
|
abs_max_ >>= 2;
|
|
|
|
|
}
|
2017-07-20 09:57:42 -07:00
|
|
|
|
|
|
|
|
// See the description for "totalAudioEnergy" in the WebRTC stats spec
|
|
|
|
|
// (https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-totalaudioenergy)
|
|
|
|
|
// for an explanation of these formulas. In short, we need a value that can
|
|
|
|
|
// be used to compute RMS audio levels over different time intervals, by
|
|
|
|
|
// taking the difference between the results from two getStats calls. To do
|
|
|
|
|
// this, the value needs to be of units "squared sample value * time".
|
|
|
|
|
double additional_energy =
|
|
|
|
|
static_cast<double>(current_level_full_range_) / INT16_MAX;
|
|
|
|
|
additional_energy *= additional_energy;
|
|
|
|
|
total_energy_ += additional_energy * duration;
|
|
|
|
|
total_duration_ += duration;
|
2017-03-07 01:58:55 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
} // namespace voe
|
|
|
|
|
} // namespace webrtc
|