/* * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include "modules/audio_processing/aec3/block_processor.h" #include #include "absl/types/optional.h" #include "modules/audio_processing/aec3/aec3_common.h" #include "modules/audio_processing/aec3/block_processor_metrics.h" #include "modules/audio_processing/aec3/delay_estimate.h" #include "modules/audio_processing/aec3/echo_path_variability.h" #include "modules/audio_processing/logging/apm_data_dumper.h" #include "rtc_base/atomicops.h" #include "rtc_base/checks.h" #include "rtc_base/constructormagic.h" #include "rtc_base/logging.h" namespace webrtc { namespace { enum class BlockProcessorApiCall { kCapture, kRender }; class BlockProcessorImpl final : public BlockProcessor { public: BlockProcessorImpl(const EchoCanceller3Config& config, int sample_rate_hz, std::unique_ptr render_buffer, std::unique_ptr delay_controller, std::unique_ptr echo_remover); ~BlockProcessorImpl() override; void ProcessCapture(bool echo_path_gain_change, bool capture_signal_saturation, std::vector>* capture_block) override; void BufferRender(const std::vector>& block) override; void UpdateEchoLeakageStatus(bool leakage_detected) override; void GetMetrics(EchoControl::Metrics* metrics) const override; void SetAudioBufferDelay(size_t delay_ms) override; private: static int instance_count_; std::unique_ptr data_dumper_; const EchoCanceller3Config config_; bool capture_properly_started_ = false; bool render_properly_started_ = false; const size_t sample_rate_hz_; std::unique_ptr render_buffer_; std::unique_ptr delay_controller_; std::unique_ptr echo_remover_; BlockProcessorMetrics metrics_; RenderDelayBuffer::BufferingEvent render_event_; size_t capture_call_counter_ = 0; absl::optional estimated_delay_; absl::optional echo_remover_delay_; RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(BlockProcessorImpl); }; int BlockProcessorImpl::instance_count_ = 0; BlockProcessorImpl::BlockProcessorImpl( const EchoCanceller3Config& config, int sample_rate_hz, std::unique_ptr render_buffer, std::unique_ptr delay_controller, std::unique_ptr echo_remover) : data_dumper_( new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), config_(config), sample_rate_hz_(sample_rate_hz), render_buffer_(std::move(render_buffer)), delay_controller_(std::move(delay_controller)), echo_remover_(std::move(echo_remover)), render_event_(RenderDelayBuffer::BufferingEvent::kNone) { RTC_DCHECK(ValidFullBandRate(sample_rate_hz_)); } BlockProcessorImpl::~BlockProcessorImpl() = default; void BlockProcessorImpl::ProcessCapture( bool echo_path_gain_change, bool capture_signal_saturation, std::vector>* capture_block) { RTC_DCHECK(capture_block); RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), capture_block->size()); RTC_DCHECK_EQ(kBlockSize, (*capture_block)[0].size()); capture_call_counter_++; data_dumper_->DumpRaw("aec3_processblock_call_order", static_cast(BlockProcessorApiCall::kCapture)); data_dumper_->DumpWav("aec3_processblock_capture_input", kBlockSize, &(*capture_block)[0][0], LowestBandRate(sample_rate_hz_), 1); if (render_properly_started_) { if (!capture_properly_started_) { capture_properly_started_ = true; render_buffer_->Reset(); delay_controller_->Reset(true); } } else { // If no render data has yet arrived, do not process the capture signal. return; } EchoPathVariability echo_path_variability( echo_path_gain_change, EchoPathVariability::DelayAdjustment::kNone, false); if (render_event_ == RenderDelayBuffer::BufferingEvent::kRenderOverrun && render_properly_started_) { echo_path_variability.delay_change = EchoPathVariability::DelayAdjustment::kBufferFlush; delay_controller_->Reset(true); RTC_LOG(LS_WARNING) << "Reset due to render buffer overrun at block " << capture_call_counter_; } // Update the render buffers with any newly arrived render blocks and prepare // the render buffers for reading the render data corresponding to the current // capture block. render_event_ = render_buffer_->PrepareCaptureProcessing(); RTC_DCHECK(RenderDelayBuffer::BufferingEvent::kRenderOverrun != render_event_); if (render_event_ == RenderDelayBuffer::BufferingEvent::kRenderUnderrun) { if (estimated_delay_ && estimated_delay_->quality == DelayEstimate::Quality::kRefined) { echo_path_variability.delay_change = EchoPathVariability::DelayAdjustment::kDelayReset; delay_controller_->Reset(true); capture_properly_started_ = false; render_properly_started_ = false; RTC_LOG(LS_WARNING) << "Reset due to render buffer underrun at block " << capture_call_counter_; } } else if (render_event_ == RenderDelayBuffer::BufferingEvent::kApiCallSkew) { // There have been too many render calls in a row. Reset to avoid noncausal // echo. echo_path_variability.delay_change = EchoPathVariability::DelayAdjustment::kDelayReset; delay_controller_->Reset(true); capture_properly_started_ = false; render_properly_started_ = false; RTC_LOG(LS_WARNING) << "Reset due to render buffer api skew at block " << capture_call_counter_; } data_dumper_->DumpWav("aec3_processblock_capture_input2", kBlockSize, &(*capture_block)[0][0], LowestBandRate(sample_rate_hz_), 1); // Compute and and apply the render delay required to achieve proper signal // alignment. estimated_delay_ = delay_controller_->GetDelay( render_buffer_->GetDownsampledRenderBuffer(), render_buffer_->Delay(), echo_remover_delay_, (*capture_block)[0]); if (estimated_delay_) { if (render_buffer_->CausalDelay(estimated_delay_->delay)) { bool delay_change = render_buffer_->SetDelay(estimated_delay_->delay); if (delay_change) { RTC_LOG(LS_WARNING) << "Delay changed to " << estimated_delay_->delay << " at block " << capture_call_counter_; echo_path_variability.delay_change = EchoPathVariability::DelayAdjustment::kNewDetectedDelay; } } else { // A noncausal delay has been detected. This can only happen if there is // clockdrift, an audio pipeline issue has occurred, an unreliable delay // estimate is used or the specified minimum delay is too short. if (estimated_delay_->quality == DelayEstimate::Quality::kRefined) { echo_path_variability.delay_change = EchoPathVariability::DelayAdjustment::kDelayReset; delay_controller_->Reset(true); render_buffer_->Reset(); capture_properly_started_ = false; render_properly_started_ = false; RTC_LOG(LS_WARNING) << "Reset due to noncausal delay at block " << capture_call_counter_; } } } // Remove the echo from the capture signal. echo_remover_->ProcessCapture( echo_path_variability, capture_signal_saturation, estimated_delay_, render_buffer_->GetRenderBuffer(), capture_block); // Check to see if a refined delay estimate has been obtained from the echo // remover. echo_remover_delay_ = echo_remover_->Delay(); // Update the metrics. metrics_.UpdateCapture(false); render_event_ = RenderDelayBuffer::BufferingEvent::kNone; } void BlockProcessorImpl::BufferRender( const std::vector>& block) { RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), block.size()); RTC_DCHECK_EQ(kBlockSize, block[0].size()); data_dumper_->DumpRaw("aec3_processblock_call_order", static_cast(BlockProcessorApiCall::kRender)); data_dumper_->DumpWav("aec3_processblock_render_input", kBlockSize, &block[0][0], LowestBandRate(sample_rate_hz_), 1); data_dumper_->DumpWav("aec3_processblock_render_input2", kBlockSize, &block[0][0], LowestBandRate(sample_rate_hz_), 1); render_event_ = render_buffer_->Insert(block); metrics_.UpdateRender(render_event_ != RenderDelayBuffer::BufferingEvent::kNone); render_properly_started_ = true; delay_controller_->LogRenderCall(); } void BlockProcessorImpl::UpdateEchoLeakageStatus(bool leakage_detected) { echo_remover_->UpdateEchoLeakageStatus(leakage_detected); } void BlockProcessorImpl::GetMetrics(EchoControl::Metrics* metrics) const { echo_remover_->GetMetrics(metrics); const int block_size_ms = sample_rate_hz_ == 8000 ? 8 : 4; absl::optional delay = render_buffer_->Delay(); metrics->delay_ms = delay ? static_cast(*delay) * block_size_ms : 0; } void BlockProcessorImpl::SetAudioBufferDelay(size_t delay_ms) { render_buffer_->SetAudioBufferDelay(delay_ms); } } // namespace BlockProcessor* BlockProcessor::Create(const EchoCanceller3Config& config, int sample_rate_hz) { std::unique_ptr render_buffer( RenderDelayBuffer::Create(config, NumBandsForRate(sample_rate_hz))); std::unique_ptr delay_controller( RenderDelayController::Create( config, RenderDelayBuffer::DelayEstimatorOffset(config), sample_rate_hz)); std::unique_ptr echo_remover( EchoRemover::Create(config, sample_rate_hz)); return Create(config, sample_rate_hz, std::move(render_buffer), std::move(delay_controller), std::move(echo_remover)); } BlockProcessor* BlockProcessor::Create( const EchoCanceller3Config& config, int sample_rate_hz, std::unique_ptr render_buffer) { std::unique_ptr delay_controller( RenderDelayController::Create( config, RenderDelayBuffer::DelayEstimatorOffset(config), sample_rate_hz)); std::unique_ptr echo_remover( EchoRemover::Create(config, sample_rate_hz)); return Create(config, sample_rate_hz, std::move(render_buffer), std::move(delay_controller), std::move(echo_remover)); } BlockProcessor* BlockProcessor::Create( const EchoCanceller3Config& config, int sample_rate_hz, std::unique_ptr render_buffer, std::unique_ptr delay_controller, std::unique_ptr echo_remover) { return new BlockProcessorImpl( config, sample_rate_hz, std::move(render_buffer), std::move(delay_controller), std::move(echo_remover)); } } // namespace webrtc