Bug 1766646 - Vendor libwebrtc from d7fdb95346

Upstream commit: https://webrtc.googlesource.com/src/+/d7fdb953468789b93c37b1a1be512f99a34639d3
    Remove typing detection

    Motivation: never used.
    Follow-up CL of https://webrtc-review.googlesource.com/c/src/+/250680.

    Tested on a custom chromium build by making a video call and with
    keyboard activity. The expected logs from the APM transient suppressor
    sub-module were shown.

    Bug: webrtc:11226
    Change-Id: I4186994412dd8ba2e71ed8f9dcc9cf8f8e40fbd3
    Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/250667
    Reviewed-by: Niels Moller <nisse@webrtc.org>
    Reviewed-by: Per Kjellander <perkj@webrtc.org>
    Reviewed-by: Björn Terelius <terelius@webrtc.org>
    Reviewed-by: Gustaf Ullberg <gustaf@webrtc.org>
    Commit-Queue: Alessio Bazzica <alessiob@webrtc.org>
    Cr-Commit-Position: refs/heads/main@{#36300}
This commit is contained in:
Michael Froman 2022-07-12 09:53:52 -05:00 коммит произвёл Connor Sheehan
Родитель 35ee31b9fa
Коммит 0f8c511876
21 изменённых файлов: 7 добавлений и 247 удалений

3
third_party/libwebrtc/README.moz-ff-commit поставляемый
Просмотреть файл

@ -12438,3 +12438,6 @@ a76ae851ad
# MOZ_LIBWEBRTC_SRC=/home/mfroman/git-checkouts/trial-webrtc-builds/moz-libwebrtc-checkout/src MOZ_LIBWEBRTC_COMMIT=mjfdev bash dom/media/webrtc/third_party_build/fast-forward-libwebrtc.sh
# base of lastest vendoring
2756ace4ce
# MOZ_LIBWEBRTC_SRC=/home/mfroman/git-checkouts/trial-webrtc-builds/moz-libwebrtc-checkout/src MOZ_LIBWEBRTC_COMMIT=mjfdev bash dom/media/webrtc/third_party_build/fast-forward-libwebrtc.sh
# base of lastest vendoring
d7fdb95346

2
third_party/libwebrtc/README.mozilla поставляемый
Просмотреть файл

@ -8302,3 +8302,5 @@ libwebrtc updated from /home/mfroman/git-checkouts/trial-webrtc-builds/moz-libwe
libwebrtc updated from /home/mfroman/git-checkouts/trial-webrtc-builds/moz-libwebrtc-checkout/src commit mjfdev on 2022-07-12T14:52:29.484898.
# python3 vendor-libwebrtc.py --from-local /home/mfroman/git-checkouts/trial-webrtc-builds/moz-libwebrtc-checkout/src --commit mjfdev libwebrtc
libwebrtc updated from /home/mfroman/git-checkouts/trial-webrtc-builds/moz-libwebrtc-checkout/src commit mjfdev on 2022-07-12T14:53:08.428513.
# python3 vendor-libwebrtc.py --from-local /home/mfroman/git-checkouts/trial-webrtc-builds/moz-libwebrtc-checkout/src --commit mjfdev libwebrtc
libwebrtc updated from /home/mfroman/git-checkouts/trial-webrtc-builds/moz-libwebrtc-checkout/src commit mjfdev on 2022-07-12T14:53:47.136341.

3
third_party/libwebrtc/api/audio_options.cc поставляемый
Просмотреть файл

@ -54,7 +54,6 @@ void AudioOptions::SetAll(const AudioOptions& change) {
change.audio_jitter_buffer_min_delay_ms);
SetFrom(&audio_jitter_buffer_enable_rtx_handling,
change.audio_jitter_buffer_enable_rtx_handling);
SetFrom(&typing_detection, change.typing_detection);
SetFrom(&residual_echo_detector, change.residual_echo_detector);
SetFrom(&combined_audio_video_bwe, change.combined_audio_video_bwe);
SetFrom(&audio_network_adaptor, change.audio_network_adaptor);
@ -78,7 +77,6 @@ bool AudioOptions::operator==(const AudioOptions& o) const {
o.audio_jitter_buffer_min_delay_ms &&
audio_jitter_buffer_enable_rtx_handling ==
o.audio_jitter_buffer_enable_rtx_handling &&
typing_detection == o.typing_detection &&
residual_echo_detector == o.residual_echo_detector &&
combined_audio_video_bwe == o.combined_audio_video_bwe &&
audio_network_adaptor == o.audio_network_adaptor &&
@ -107,7 +105,6 @@ std::string AudioOptions::ToString() const {
audio_jitter_buffer_min_delay_ms);
ToStringIfSet(&result, "audio_jitter_buffer_enable_rtx_handling",
audio_jitter_buffer_enable_rtx_handling);
ToStringIfSet(&result, "typing", typing_detection);
ToStringIfSet(&result, "residual_echo_detector", residual_echo_detector);
ToStringIfSet(&result, "combined_audio_video_bwe", combined_audio_video_bwe);
ToStringIfSet(&result, "audio_network_adaptor", audio_network_adaptor);

4
third_party/libwebrtc/api/audio_options.h поставляемый
Просмотреть файл

@ -60,10 +60,6 @@ struct RTC_EXPORT AudioOptions {
absl::optional<int> audio_jitter_buffer_min_delay_ms;
// Audio receiver jitter buffer (NetEq) should handle retransmitted packets.
absl::optional<bool> audio_jitter_buffer_enable_rtx_handling;
// Deprecated.
// TODO(bugs.webrtc.org/11226): Remove.
// Audio processing to detect typing.
absl::optional<bool> typing_detection;
// TODO(bugs.webrtc.org/11539): Deprecated, replaced by
// webrtc::CreateEchoDetector() and injection when creating the audio
// processing module.

3
third_party/libwebrtc/api/stats_types.cc поставляемый
Просмотреть файл

@ -648,9 +648,6 @@ const char* StatsReport::Value::display_name() const {
return "googTrackId";
case kStatsValueNameTimingFrameInfo:
return "googTimingFrameInfo";
// TODO(bugs.webrtc.org/11226): Remove.
case kStatsValueNameTypingNoiseState:
return "googTypingNoiseState";
case kStatsValueNameWritable:
return "googWritable";
case kStatsValueNameAudioDeviceUnderrunCounter:

2
third_party/libwebrtc/api/stats_types.h поставляемый
Просмотреть файл

@ -235,8 +235,6 @@ class RTC_EXPORT StatsReport {
kStatsValueNameTrackId,
kStatsValueNameTransmitBitrate,
kStatsValueNameTransportType,
// TODO(bugs.webrtc.org/11226): Remove.
kStatsValueNameTypingNoiseState,
kStatsValueNameWritable,
kStatsValueNameAudioDeviceUnderrunCounter,
kStatsValueNameLocalCandidateRelayProtocol,

Просмотреть файл

@ -476,7 +476,6 @@ webrtc::AudioSendStream::Stats AudioSendStream::GetStats(
stats.total_input_duration = audio_level_.TotalDuration();
}
stats.typing_noise_detected = audio_state()->typing_noise_detected();
stats.ana_statistics = channel_send_->GetANAStatistics();
AudioProcessing* ap = audio_state_->audio_processing();

Просмотреть файл

@ -472,7 +472,6 @@ TEST(AudioSendStreamTest, GetStats) {
stats.apm_statistics.residual_echo_likelihood);
EXPECT_EQ(kResidualEchoLikelihoodMax,
stats.apm_statistics.residual_echo_likelihood_recent_max);
EXPECT_FALSE(stats.typing_noise_detected);
}
}
}

5
third_party/libwebrtc/audio/audio_state.cc поставляемый
Просмотреть файл

@ -50,11 +50,6 @@ AudioTransport* AudioState::audio_transport() {
return &audio_transport_;
}
bool AudioState::typing_noise_detected() const {
RTC_DCHECK(thread_checker_.IsCurrent());
return audio_transport_.typing_noise_detected();
}
void AudioState::AddReceivingStream(webrtc::AudioReceiveStream* stream) {
RTC_DCHECK(thread_checker_.IsCurrent());
RTC_DCHECK_EQ(0, receiving_streams_.count(stream));

2
third_party/libwebrtc/audio/audio_state.h поставляемый
Просмотреть файл

@ -51,8 +51,6 @@ class AudioState : public webrtc::AudioState {
return config_.audio_device_module.get();
}
bool typing_noise_detected() const;
void AddReceivingStream(webrtc::AudioReceiveStream* stream);
void RemoveReceivingStream(webrtc::AudioReceiveStream* stream);

Просмотреть файл

@ -20,7 +20,6 @@
#include "modules/async_audio_processing/async_audio_processing.h"
#include "modules/audio_device/include/audio_device.h"
#include "modules/audio_processing/include/audio_processing.h"
#include "modules/audio_processing/typing_detection.h"
#include "rtc_base/synchronization/mutex.h"
#include "rtc_base/thread_annotations.h"
@ -86,9 +85,6 @@ class AudioTransportImpl : public AudioTransport {
int send_sample_rate_hz,
size_t send_num_channels);
void SetStereoChannelSwapping(bool enable);
// Deprecated.
// TODO(bugs.webrtc.org/11226): Remove.
bool typing_noise_detected() const { return false; }
private:
void SendProcessedData(std::unique_ptr<AudioFrame> audio_frame);
@ -107,7 +103,6 @@ class AudioTransportImpl : public AudioTransport {
size_t send_num_channels_ RTC_GUARDED_BY(capture_lock_) = 1;
bool swap_stereo_channels_ RTC_GUARDED_BY(capture_lock_) = false;
PushResampler<int16_t> capture_resampler_;
TypingDetection typing_detection_;
// Render side.

Просмотреть файл

@ -63,7 +63,6 @@ class NoLossTest : public AudioEndToEndTest {
EXPECT_FALSE(send_stats.apm_statistics.echo_return_loss_enhancement);
EXPECT_FALSE(send_stats.apm_statistics.residual_echo_likelihood);
EXPECT_FALSE(send_stats.apm_statistics.residual_echo_likelihood_recent_max);
EXPECT_EQ(false, send_stats.typing_noise_detected);
AudioReceiveStream::Stats recv_stats =
receive_stream()->GetStats(/*get_and_clear_legacy_stats=*/true);

Просмотреть файл

@ -60,7 +60,6 @@ class AudioSendStream : public AudioSender {
// https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-totalaudioenergy
double total_input_energy = 0.0;
double total_input_duration = 0.0;
bool typing_noise_detected = false;
ANAStats ana_statistics;
AudioProcessingStats apm_statistics;

Просмотреть файл

@ -465,7 +465,6 @@ struct VoiceSenderInfo : public MediaSenderInfo {
// https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-totalaudioenergy
double total_input_energy = 0.0;
double total_input_duration = 0.0;
bool typing_noise_detected = false;
webrtc::ANAStats ana_statistics;
webrtc::AudioProcessingStats apm_statistics;
};

Просмотреть файл

@ -393,10 +393,8 @@ void WebRtcVoiceEngine::Init() {
#if defined(WEBRTC_IOS)
// On iOS, VPIO provides built-in NS.
options.noise_suppression = false;
options.typing_detection = false;
#else
options.noise_suppression = true;
options.typing_detection = true;
#endif
options.highpass_filter = true;
options.stereo_swapping = false;
@ -453,11 +451,6 @@ bool WebRtcVoiceEngine::ApplyOptions(const AudioOptions& options_in) {
use_mobile_software_aec = true;
#endif
// Override noise suppression options for Android.
#if defined(WEBRTC_ANDROID)
options.typing_detection = false;
#endif
// Set and adjust gain control options.
#if defined(WEBRTC_IOS)
// On iOS, VPIO provides built-in AGC.
@ -602,10 +595,6 @@ bool WebRtcVoiceEngine::ApplyOptions(const AudioOptions& options_in) {
RTC_LOG(LS_INFO) << "NS set to " << enabled;
}
if (options.typing_detection) {
RTC_LOG(LS_WARNING) << "Typing detection is requested, but unsupported.";
}
ap->ApplyConfig(apm_config);
return true;
}
@ -2339,7 +2328,6 @@ bool WebRtcVoiceMediaChannel::GetStats(VoiceMediaInfo* info,
sinfo.audio_level = stats.audio_level;
sinfo.total_input_energy = stats.total_input_energy;
sinfo.total_input_duration = stats.total_input_duration;
sinfo.typing_noise_detected = (send_ ? stats.typing_noise_detected : false);
sinfo.ana_statistics = stats.ana_statistics;
sinfo.apm_statistics = stats.apm_statistics;
sinfo.report_block_datas = std::move(stats.report_block_datas);

Просмотреть файл

@ -606,7 +606,6 @@ class WebRtcVoiceEngineTestFake : public ::testing::TestWithParam<bool> {
stats.ana_statistics.frame_length_increase_counter = 765;
stats.ana_statistics.frame_length_decrease_counter = 876;
stats.ana_statistics.uplink_packet_loss_fraction = 987.0;
stats.typing_noise_detected = true;
return stats;
}
void SetAudioSendStreamStats() {
@ -655,8 +654,6 @@ class WebRtcVoiceEngineTestFake : public ::testing::TestWithParam<bool> {
stats.ana_statistics.frame_length_decrease_counter);
EXPECT_EQ(info.ana_statistics.uplink_packet_loss_fraction,
stats.ana_statistics.uplink_packet_loss_fraction);
EXPECT_EQ(info.typing_noise_detected,
stats.typing_noise_detected && is_sending);
}
webrtc::AudioReceiveStream::Stats GetAudioReceiveStreamStats() const {

Просмотреть файл

@ -151,8 +151,6 @@ rtc_library("audio_processing") {
"gain_control_impl.cc",
"gain_control_impl.h",
"render_queue_item_verifier.h",
"typing_detection.cc",
"typing_detection.h",
]
defines = []

Просмотреть файл

@ -1,93 +0,0 @@
/*
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/audio_processing/typing_detection.h"
namespace webrtc {
TypingDetection::TypingDetection()
: time_active_(0),
time_since_last_typing_(0),
penalty_counter_(0),
counter_since_last_detection_update_(0),
detection_to_report_(false),
new_detection_to_report_(false),
time_window_(10),
cost_per_typing_(100),
reporting_threshold_(300),
penalty_decay_(1),
type_event_delay_(2),
report_detection_update_period_(1) {}
TypingDetection::~TypingDetection() {}
bool TypingDetection::Process(bool key_pressed, bool vad_activity) {
if (vad_activity)
time_active_++;
else
time_active_ = 0;
// Keep track if time since last typing event
if (key_pressed)
time_since_last_typing_ = 0;
else
++time_since_last_typing_;
if (time_since_last_typing_ < type_event_delay_ && vad_activity &&
time_active_ < time_window_) {
penalty_counter_ += cost_per_typing_;
if (penalty_counter_ > reporting_threshold_)
new_detection_to_report_ = true;
}
if (penalty_counter_ > 0)
penalty_counter_ -= penalty_decay_;
if (++counter_since_last_detection_update_ ==
report_detection_update_period_) {
detection_to_report_ = new_detection_to_report_;
new_detection_to_report_ = false;
counter_since_last_detection_update_ = 0;
}
return detection_to_report_;
}
int TypingDetection::TimeSinceLastDetectionInSeconds() {
// Round to whole seconds.
return (time_since_last_typing_ + 50) / 100;
}
void TypingDetection::SetParameters(int time_window,
int cost_per_typing,
int reporting_threshold,
int penalty_decay,
int type_event_delay,
int report_detection_update_period) {
if (time_window)
time_window_ = time_window;
if (cost_per_typing)
cost_per_typing_ = cost_per_typing;
if (reporting_threshold)
reporting_threshold_ = reporting_threshold;
if (penalty_decay)
penalty_decay_ = penalty_decay;
if (type_event_delay)
type_event_delay_ = type_event_delay;
if (report_detection_update_period)
report_detection_update_period_ = report_detection_update_period;
}
} // namespace webrtc

Просмотреть файл

@ -1,92 +0,0 @@
/*
* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_AUDIO_PROCESSING_TYPING_DETECTION_H_
#define MODULES_AUDIO_PROCESSING_TYPING_DETECTION_H_
#include "rtc_base/system/rtc_export.h"
namespace webrtc {
class RTC_EXPORT TypingDetection {
public:
TypingDetection();
virtual ~TypingDetection();
// Run the detection algortihm. Shall be called every 10 ms. Returns true if
// typing is detected, or false if not, based on the update period as set with
// SetParameters(). See `report_detection_update_period_` description below.
bool Process(bool key_pressed, bool vad_activity);
// Gets the time in seconds since the last detection.
int TimeSinceLastDetectionInSeconds();
// Sets the algorithm parameters. A parameter value of 0 leaves it unchanged.
// See the correspondning member variables below for descriptions.
void SetParameters(int time_window,
int cost_per_typing,
int reporting_threshold,
int penalty_decay,
int type_event_delay,
int report_detection_update_period);
private:
int time_active_;
int time_since_last_typing_;
int penalty_counter_;
// Counter since last time the detection status reported by Process() was
// updated. See also `report_detection_update_period_`.
int counter_since_last_detection_update_;
// The detection status to report. Updated every
// `report_detection_update_period_` call to Process().
bool detection_to_report_;
// What `detection_to_report_` should be set to next time it is updated.
bool new_detection_to_report_;
// Settable threshold values.
// Number of 10 ms slots accepted to count as a hit.
int time_window_;
// Penalty added for a typing + activity coincide.
int cost_per_typing_;
// Threshold for `penalty_counter_`.
int reporting_threshold_;
// How much we reduce `penalty_counter_` every 10 ms.
int penalty_decay_;
// How old typing events we allow.
int type_event_delay_;
// Settable update period.
// Number of 10 ms slots between each update of the detection status returned
// by Process(). This inertia added to the algorithm is usually desirable and
// provided so that consumers of the class don't have to implement that
// themselves if they don't wish.
// If set to 1, each call to Process() will return the detection status for
// that 10 ms slot.
// If set to N (where N > 1), the detection status returned from Process()
// will remain the same until Process() has been called N times. Then, if none
// of the last N calls to Process() has detected typing for each respective
// 10 ms slot, Process() will return false. If at least one of the last N
// calls has detected typing, Process() will return true. And that returned
// status will then remain the same until the next N calls have been done.
int report_detection_update_period_;
};
} // namespace webrtc
#endif // #ifndef MODULES_AUDIO_PROCESSING_TYPING_DETECTION_H_

9
third_party/libwebrtc/pc/stats_collector.cc поставляемый
Просмотреть файл

@ -143,10 +143,7 @@ void ExtractCommonReceiveProperties(const cricket::MediaReceiverInfo& info,
}
void SetAudioProcessingStats(StatsReport* report,
bool typing_noise_detected,
const AudioProcessingStats& apm_stats) {
report->AddBoolean(StatsReport::kStatsValueNameTypingNoiseState,
typing_noise_detected);
if (apm_stats.delay_median_ms) {
report->AddInt(StatsReport::kStatsValueNameEchoDelayMedian,
*apm_stats.delay_median_ms);
@ -245,8 +242,7 @@ void ExtractStats(const cricket::VoiceSenderInfo& info,
bool use_standard_bytes_stats) {
ExtractCommonSendProperties(info, report, use_standard_bytes_stats);
SetAudioProcessingStats(report, info.typing_noise_detected,
info.apm_statistics);
SetAudioProcessingStats(report, info.apm_statistics);
const FloatForAdd floats[] = {
{StatsReport::kStatsValueNameTotalAudioEnergy, info.total_input_energy},
@ -1354,8 +1350,7 @@ void StatsCollector::UpdateReportFromAudioTrack(AudioTrackInterface* track,
AudioProcessorInterface::AudioProcessorStatistics stats =
audio_processor->GetStats(has_remote_tracks);
SetAudioProcessingStats(report, stats.typing_noise_detected,
stats.apm_statistics);
SetAudioProcessingStats(report, stats.apm_statistics);
}
}

Просмотреть файл

@ -89,7 +89,6 @@ class FakeAudioProcessor : public AudioProcessorInterface {
AudioProcessorInterface::AudioProcessorStatistics GetStats(
bool has_recv_streams) override {
AudioProcessorStatistics stats;
stats.typing_noise_detected = true;
if (has_recv_streams) {
stats.apm_statistics.echo_return_loss = 2.0;
stats.apm_statistics.echo_return_loss_enhancement = 3.0;
@ -132,7 +131,6 @@ class FakeAudioProcessorWithInitValue : public AudioProcessorInterface {
AudioProcessorInterface::AudioProcessorStatistics GetStats(
bool /*has_recv_streams*/) override {
AudioProcessorStatistics stats;
stats.typing_noise_detected = false;
return stats;
}
};
@ -488,10 +486,6 @@ void VerifyVoiceSenderInfoReport(const StatsReport* report,
EXPECT_TRUE(GetValue(report, StatsReport::kStatsValueNameAudioInputLevel,
&value_in_report));
EXPECT_EQ(rtc::ToString(sinfo.audio_level), value_in_report);
EXPECT_TRUE(GetValue(report, StatsReport::kStatsValueNameTypingNoiseState,
&value_in_report));
std::string typing_detected = sinfo.typing_noise_detected ? "true" : "false";
EXPECT_EQ(typing_detected, value_in_report);
EXPECT_TRUE(GetValue(report,
StatsReport::kStatsValueNameAnaBitrateActionCounter,
&value_in_report));
@ -551,7 +545,6 @@ void InitVoiceSenderInfo(cricket::VoiceSenderInfo* voice_sender_info,
voice_sender_info->apm_statistics.echo_return_loss_enhancement = 109;
voice_sender_info->apm_statistics.delay_median_ms = 110;
voice_sender_info->apm_statistics.delay_standard_deviation_ms = 111;
voice_sender_info->typing_noise_detected = false;
voice_sender_info->ana_statistics.bitrate_action_counter = 112;
voice_sender_info->ana_statistics.channel_action_counter = 113;
voice_sender_info->ana_statistics.dtx_action_counter = 114;
@ -568,8 +561,6 @@ void UpdateVoiceSenderInfoFromAudioTrack(
audio_track->GetSignalLevel(&voice_sender_info->audio_level);
AudioProcessorInterface::AudioProcessorStatistics audio_processor_stats =
audio_track->GetAudioProcessor()->GetStats(has_remote_tracks);
voice_sender_info->typing_noise_detected =
audio_processor_stats.typing_noise_detected;
voice_sender_info->apm_statistics = audio_processor_stats.apm_statistics;
}