mirror of
https://github.com/danog/libtgvoip.git
synced 2024-12-04 10:37:49 +01:00
216 lines
8.7 KiB
C++
216 lines
8.7 KiB
C++
/*
|
|
* Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*/
|
|
|
|
#include "modules/audio_processing/residual_echo_detector.h"
|
|
|
|
#include <algorithm>
|
|
#include <numeric>
|
|
|
|
#include "absl/types/optional.h"
|
|
#include "modules/audio_processing/audio_buffer.h"
|
|
#include "modules/audio_processing/logging/apm_data_dumper.h"
|
|
#include "rtc_base/atomicops.h"
|
|
#include "rtc_base/checks.h"
|
|
#include "rtc_base/logging.h"
|
|
#include "system_wrappers/include/metrics.h"
|
|
|
|
namespace {
|
|
|
|
float Power(rtc::ArrayView<const float> input) {
|
|
if (input.empty()) {
|
|
return 0.f;
|
|
}
|
|
return std::inner_product(input.begin(), input.end(), input.begin(), 0.f) /
|
|
input.size();
|
|
}
|
|
|
|
constexpr size_t kLookbackFrames = 650;
|
|
// TODO(ivoc): Verify the size of this buffer.
|
|
constexpr size_t kRenderBufferSize = 30;
|
|
constexpr float kAlpha = 0.001f;
|
|
// 10 seconds of data, updated every 10 ms.
|
|
constexpr size_t kAggregationBufferSize = 10 * 100;
|
|
|
|
} // namespace
|
|
|
|
namespace webrtc {
|
|
|
|
int ResidualEchoDetector::instance_count_ = 0;
|
|
|
|
ResidualEchoDetector::ResidualEchoDetector()
|
|
: data_dumper_(
|
|
new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
|
|
render_buffer_(kRenderBufferSize),
|
|
render_power_(kLookbackFrames),
|
|
render_power_mean_(kLookbackFrames),
|
|
render_power_std_dev_(kLookbackFrames),
|
|
covariances_(kLookbackFrames),
|
|
recent_likelihood_max_(kAggregationBufferSize) {}
|
|
|
|
ResidualEchoDetector::~ResidualEchoDetector() = default;
|
|
|
|
void ResidualEchoDetector::AnalyzeRenderAudio(
|
|
rtc::ArrayView<const float> render_audio) {
|
|
// Dump debug data assuming 48 kHz sample rate (if this assumption is not
|
|
// valid the dumped audio will need to be converted offline accordingly).
|
|
data_dumper_->DumpWav("ed_render", render_audio.size(), render_audio.data(),
|
|
48000, 1);
|
|
|
|
if (render_buffer_.Size() == 0) {
|
|
frames_since_zero_buffer_size_ = 0;
|
|
} else if (frames_since_zero_buffer_size_ >= kRenderBufferSize) {
|
|
// This can happen in a few cases: at the start of a call, due to a glitch
|
|
// or due to clock drift. The excess capture value will be ignored.
|
|
// TODO(ivoc): Include how often this happens in APM stats.
|
|
render_buffer_.Pop();
|
|
frames_since_zero_buffer_size_ = 0;
|
|
}
|
|
++frames_since_zero_buffer_size_;
|
|
float power = Power(render_audio);
|
|
render_buffer_.Push(power);
|
|
}
|
|
|
|
void ResidualEchoDetector::AnalyzeCaptureAudio(
|
|
rtc::ArrayView<const float> capture_audio) {
|
|
// Dump debug data assuming 48 kHz sample rate (if this assumption is not
|
|
// valid the dumped audio will need to be converted offline accordingly).
|
|
data_dumper_->DumpWav("ed_capture", capture_audio.size(),
|
|
capture_audio.data(), 48000, 1);
|
|
|
|
if (first_process_call_) {
|
|
// On the first process call (so the start of a call), we must flush the
|
|
// render buffer, otherwise the render data will be delayed.
|
|
render_buffer_.Clear();
|
|
first_process_call_ = false;
|
|
}
|
|
|
|
// Get the next render value.
|
|
const absl::optional<float> buffered_render_power = render_buffer_.Pop();
|
|
if (!buffered_render_power) {
|
|
// This can happen in a few cases: at the start of a call, due to a glitch
|
|
// or due to clock drift. The excess capture value will be ignored.
|
|
// TODO(ivoc): Include how often this happens in APM stats.
|
|
return;
|
|
}
|
|
// Update the render statistics, and store the statistics in circular buffers.
|
|
render_statistics_.Update(*buffered_render_power);
|
|
RTC_DCHECK_LT(next_insertion_index_, kLookbackFrames);
|
|
render_power_[next_insertion_index_] = *buffered_render_power;
|
|
render_power_mean_[next_insertion_index_] = render_statistics_.mean();
|
|
render_power_std_dev_[next_insertion_index_] =
|
|
render_statistics_.std_deviation();
|
|
|
|
// Get the next capture value, update capture statistics and add the relevant
|
|
// values to the buffers.
|
|
const float capture_power = Power(capture_audio);
|
|
capture_statistics_.Update(capture_power);
|
|
const float capture_mean = capture_statistics_.mean();
|
|
const float capture_std_deviation = capture_statistics_.std_deviation();
|
|
|
|
// Update the covariance values and determine the new echo likelihood.
|
|
echo_likelihood_ = 0.f;
|
|
size_t read_index = next_insertion_index_;
|
|
|
|
int best_delay = -1;
|
|
for (size_t delay = 0; delay < covariances_.size(); ++delay) {
|
|
RTC_DCHECK_LT(read_index, render_power_.size());
|
|
covariances_[delay].Update(capture_power, capture_mean,
|
|
capture_std_deviation, render_power_[read_index],
|
|
render_power_mean_[read_index],
|
|
render_power_std_dev_[read_index]);
|
|
read_index = read_index > 0 ? read_index - 1 : kLookbackFrames - 1;
|
|
|
|
if (covariances_[delay].normalized_cross_correlation() > echo_likelihood_) {
|
|
echo_likelihood_ = covariances_[delay].normalized_cross_correlation();
|
|
best_delay = static_cast<int>(delay);
|
|
}
|
|
}
|
|
// This is a temporary log message to help find the underlying cause for echo
|
|
// likelihoods > 1.0.
|
|
// TODO(ivoc): Remove once the issue is resolved.
|
|
if (echo_likelihood_ > 1.1f) {
|
|
// Make sure we don't spam the log.
|
|
if (log_counter_ < 5 && best_delay != -1) {
|
|
size_t read_index = kLookbackFrames + next_insertion_index_ - best_delay;
|
|
if (read_index >= kLookbackFrames) {
|
|
read_index -= kLookbackFrames;
|
|
}
|
|
RTC_DCHECK_LT(read_index, render_power_.size());
|
|
RTC_LOG_F(LS_ERROR) << "Echo detector internal state: {"
|
|
"Echo likelihood: "
|
|
<< echo_likelihood_ << ", Best Delay: " << best_delay
|
|
<< ", Covariance: "
|
|
<< covariances_[best_delay].covariance()
|
|
<< ", Last capture power: " << capture_power
|
|
<< ", Capture mean: " << capture_mean
|
|
<< ", Capture_standard deviation: "
|
|
<< capture_std_deviation << ", Last render power: "
|
|
<< render_power_[read_index]
|
|
<< ", Render mean: " << render_power_mean_[read_index]
|
|
<< ", Render standard deviation: "
|
|
<< render_power_std_dev_[read_index]
|
|
<< ", Reliability: " << reliability_ << "}";
|
|
log_counter_++;
|
|
}
|
|
}
|
|
RTC_DCHECK_LT(echo_likelihood_, 1.1f);
|
|
|
|
reliability_ = (1.0f - kAlpha) * reliability_ + kAlpha * 1.0f;
|
|
echo_likelihood_ *= reliability_;
|
|
// This is a temporary fix to prevent echo likelihood values > 1.0.
|
|
// TODO(ivoc): Find the root cause of this issue and fix it.
|
|
echo_likelihood_ = std::min(echo_likelihood_, 1.0f);
|
|
int echo_percentage = static_cast<int>(echo_likelihood_ * 100);
|
|
RTC_HISTOGRAM_COUNTS("WebRTC.Audio.ResidualEchoDetector.EchoLikelihood",
|
|
echo_percentage, 0, 100, 100 /* number of bins */);
|
|
|
|
// Update the buffer of recent likelihood values.
|
|
recent_likelihood_max_.Update(echo_likelihood_);
|
|
|
|
// Update the next insertion index.
|
|
next_insertion_index_ = next_insertion_index_ < (kLookbackFrames - 1)
|
|
? next_insertion_index_ + 1
|
|
: 0;
|
|
}
|
|
|
|
void ResidualEchoDetector::Initialize(int /*capture_sample_rate_hz*/,
|
|
int /*num_capture_channels*/,
|
|
int /*render_sample_rate_hz*/,
|
|
int /*num_render_channels*/) {
|
|
render_buffer_.Clear();
|
|
std::fill(render_power_.begin(), render_power_.end(), 0.f);
|
|
std::fill(render_power_mean_.begin(), render_power_mean_.end(), 0.f);
|
|
std::fill(render_power_std_dev_.begin(), render_power_std_dev_.end(), 0.f);
|
|
render_statistics_.Clear();
|
|
capture_statistics_.Clear();
|
|
recent_likelihood_max_.Clear();
|
|
for (auto& cov : covariances_) {
|
|
cov.Clear();
|
|
}
|
|
echo_likelihood_ = 0.f;
|
|
next_insertion_index_ = 0;
|
|
reliability_ = 0.f;
|
|
}
|
|
|
|
void EchoDetector::PackRenderAudioBuffer(AudioBuffer* audio,
|
|
std::vector<float>* packed_buffer) {
|
|
packed_buffer->clear();
|
|
packed_buffer->insert(packed_buffer->end(), audio->channels_f()[0],
|
|
audio->channels_f()[0] + audio->num_frames());
|
|
}
|
|
|
|
EchoDetector::Metrics ResidualEchoDetector::GetMetrics() const {
|
|
EchoDetector::Metrics metrics;
|
|
metrics.echo_likelihood = echo_likelihood_;
|
|
metrics.echo_likelihood_recent_max = recent_likelihood_max_.max();
|
|
return metrics;
|
|
}
|
|
} // namespace webrtc
|