Bug 1059573 - Add an option to use hardware AEC for WebRTC. r=jesup,ted

build/gyp.mozbuild

@@ -45,9 +45,13 @@ gyp_vars = {
 
     # (for vp8) chromium sets to 0 also
     'use_temporal_layers': 0,
+
     # Creates AEC internal sample dump files in current directory
     'aec_debug_dump': 1,
 
+    # Enable and force use of hardware AEC
+    'hardware_aec_ns': 1 if CONFIG['MOZ_WEBRTC_HARDWARE_AEC_NS'] else 0,
+
     # codec enable/disables:
     'include_g711': 1,
     'include_opus': 1,

config/system-headers

@@ -200,6 +200,8 @@ atlwin.cpp
 ATSTypes.h
 ATSUnicode.h
 #ifdef ANDROID
+audio_effects/effect_aec.h
+audio_effects/effect_ns.h
 AudioParameter.h
 AudioSystem.h
 AudioTrack.h
@@ -729,6 +731,8 @@ Math64.h
 math.h
 mbstring.h
 #ifdef ANDROID
+media/AudioEffect.h
+media/AudioSystem.h
 media/ICrypto.h
 media/IOMX.h
 media/MediaProfiles.h

configure.in

@@ -3793,6 +3793,7 @@ MOZ_PEERCONNECTION=
 MOZ_SRTP=
 MOZ_WEBRTC_SIGNALING=
 MOZ_WEBRTC_ASSERT_ALWAYS=1
+MOZ_WEBRTC_HARDWARE_AEC_NS=
 MOZ_SCTP=
 MOZ_ANDROID_OMX=
 MOZ_MEDIA_NAVIGATOR=
@@ -5006,11 +5007,24 @@ else
     MOZ_SYNTH_PICO=
 fi
 
+dnl ========================================================
+dnl = Force hardware AEC, disable webrtc.org AEC
+dnl ========================================================
+MOZ_ARG_ENABLE_BOOL(hardware-aec-ns,
+[  --enable-hardware-aec-ns   Enable support for hardware AEC and noise suppression],
+    MOZ_WEBRTC_HARDWARE_AEC_NS=1,
+    MOZ_WEBRTC_HARDWARE_AEC_NS=)
+
+if test -n "$MOZ_WEBRTC_HARDWARE_AEC_NS"; then
+    AC_DEFINE(MOZ_WEBRTC_HARDWARE_AEC_NS)
+fi
+
 AC_SUBST(MOZ_WEBRTC)
 AC_SUBST(MOZ_WEBRTC_LEAKING_TESTS)
 AC_SUBST(MOZ_WEBRTC_SIGNALING)
 AC_SUBST(MOZ_PEERCONNECTION)
 AC_SUBST(MOZ_WEBRTC_ASSERT_ALWAYS)
+AC_SUBST(MOZ_WEBRTC_HARDWARE_AEC_NS)
 AC_SUBST(MOZ_SCTP)
 AC_SUBST(MOZ_SRTP)
 AC_SUBST_LIST(MOZ_WEBRTC_X11_LIBS)

media/libcubeb/src/android/sles_definitions.h

@@ -33,14 +33,20 @@
 /** Audio recording preset values */
 /**   preset "none" cannot be set, it is used to indicate the current settings
  *     do not match any of the presets. */
-#define SL_ANDROID_RECORDING_PRESET_NONE              ((SLuint32) 0x00000000)
+#define SL_ANDROID_RECORDING_PRESET_NONE                ((SLuint32) 0x00000000)
 /**   generic recording configuration on the platform */
-#define SL_ANDROID_RECORDING_PRESET_GENERIC           ((SLuint32) 0x00000001)
+#define SL_ANDROID_RECORDING_PRESET_GENERIC             ((SLuint32) 0x00000001)
 /**   uses the microphone audio source with the same orientation as the camera
  *     if available, the main device microphone otherwise */
-#define SL_ANDROID_RECORDING_PRESET_CAMCORDER         ((SLuint32) 0x00000002)
+#define SL_ANDROID_RECORDING_PRESET_CAMCORDER           ((SLuint32) 0x00000002)
 /**   uses the main microphone tuned for voice recognition */
-#define SL_ANDROID_RECORDING_PRESET_VOICE_RECOGNITION ((SLuint32) 0x00000003)
+#define SL_ANDROID_RECORDING_PRESET_VOICE_RECOGNITION   ((SLuint32) 0x00000003)
+/**   uses the main microphone tuned for audio communications */
+#define SL_ANDROID_RECORDING_PRESET_VOICE_COMMUNICATION ((SLuint32) 0x00000004)
+
+/** Audio recording get session ID (read only) */
+/** Audio recording get session ID key */
+#define SL_ANDROID_KEY_RECORDING_SESSION_ID ((const SLchar*) "androidRecordingSessionId")
 
 /*---------------------------------------------------------------------------*/
 /* Android AudioPlayer configuration                                         */

media/webrtc/trunk/webrtc/modules/audio_device/android/opensles_input.cc

@@ -22,6 +22,13 @@
 #include "webrtc/system_wrappers/interface/thread_wrapper.h"
 #include "webrtc/system_wrappers/interface/trace.h"
 
+#if defined(WEBRTC_GONK) && defined(WEBRTC_HARDWARE_AEC_NS)
+#include <media/AudioSystem.h>
+#include <audio_effects/effect_aec.h>
+#include <audio_effects/effect_ns.h>
+#include <utils/Errors.h>
+#endif
+
 #define VOID_RETURN
 #define OPENSL_RETURN_ON_FAILURE(op, ret_val)                    \
   do {                                                           \
@@ -65,6 +72,10 @@ OpenSlesInput::OpenSlesInput(
       active_queue_(0),
       rec_sampling_rate_(0),
       agc_enabled_(false),
+#if defined(WEBRTC_GONK) && defined(WEBRTC_HARDWARE_AEC_NS)
+      aec_(NULL),
+      ns_(NULL),
+#endif
       recording_delay_(0),
       opensles_lib_(NULL) {
 }
@@ -390,6 +401,104 @@ bool OpenSlesInput::EnqueueAllBuffers() {
   return true;
 }
 
+#if defined(WEBRTC_GONK) && defined(WEBRTC_HARDWARE_AEC_NS)
+bool OpenSlesInput::CheckPlatformAEC() {
+  effect_descriptor_t fxDesc;
+  uint32_t numFx;
+
+  if (android::AudioEffect::queryNumberEffects(&numFx) != android::NO_ERROR) {
+    return false;
+  }
+
+  WEBRTC_TRACE(kTraceError, kTraceAudioDevice, id_, "Platform has %d effects", numFx);
+
+  for (uint32_t i = 0; i < numFx; i++) {
+    if (android::AudioEffect::queryEffect(i, &fxDesc) != android::NO_ERROR) {
+      continue;
+    }
+    if (memcmp(&fxDesc.type, FX_IID_AEC, sizeof(fxDesc.type)) == 0) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+void OpenSlesInput::SetupVoiceMode() {
+  SLAndroidConfigurationItf configItf;
+  SLresult res = (*sles_recorder_)->GetInterface(sles_recorder_, SL_IID_ANDROIDCONFIGURATION_,
+                                                 (void*)&configItf);
+  WEBRTC_TRACE(kTraceError, kTraceAudioDevice, id_, "OpenSL GetInterface: %d", res);
+
+  if (res == SL_RESULT_SUCCESS) {
+    SLuint32 voiceMode = SL_ANDROID_RECORDING_PRESET_VOICE_COMMUNICATION;
+    SLuint32 voiceSize = sizeof(voiceMode);
+
+    res = (*configItf)->SetConfiguration(configItf,
+                                         SL_ANDROID_KEY_RECORDING_PRESET,
+                                         &voiceMode, voiceSize);
+    WEBRTC_TRACE(kTraceError, kTraceAudioDevice, id_, "OpenSL Set Voice mode res: %d", res);
+  }
+}
+
+void OpenSlesInput::SetupAECAndNS() {
+  bool hasAec = CheckPlatformAEC();
+  WEBRTC_TRACE(kTraceError, kTraceAudioDevice, id_, "Platform has AEC: %d", hasAec);
+  // This code should not have been enabled if this fails, because it means the
+  // software AEC has will have been disabled as well. If you hit this, you need
+  // to fix your B2G config or fix the hardware AEC on your device.
+  assert(hasAec);
+
+  SLAndroidConfigurationItf configItf;
+  SLresult res = (*sles_recorder_)->GetInterface(sles_recorder_, SL_IID_ANDROIDCONFIGURATION_,
+                                                 (void*)&configItf);
+  WEBRTC_TRACE(kTraceError, kTraceAudioDevice, id_, "OpenSL GetInterface: %d", res);
+
+  if (res == SL_RESULT_SUCCESS) {
+    SLuint32 sessionId = 0;
+    SLuint32 idSize = sizeof(sessionId);
+    res = (*configItf)->GetConfiguration(configItf,
+                                         SL_ANDROID_KEY_RECORDING_SESSION_ID,
+                                         &idSize, &sessionId);
+    WEBRTC_TRACE(kTraceError, kTraceAudioDevice, id_, "OpenSL Get sessionId res: %d", res);
+
+    if (res == SL_RESULT_SUCCESS && idSize == sizeof(sessionId)) {
+      WEBRTC_TRACE(kTraceError, kTraceAudioDevice, id_, "OpenSL sessionId: %d", sessionId);
+
+      aec_ = new android::AudioEffect(FX_IID_AEC, NULL, 0, 0, 0, sessionId, 0);
+      WEBRTC_TRACE(kTraceError, kTraceAudioDevice, id_, "OpenSL aec: %p", aec_);
+
+      if (aec_) {
+        android::status_t status = aec_->initCheck();
+        if (status == android::NO_ERROR || status == android::ALREADY_EXISTS) {
+          WEBRTC_TRACE(kTraceError, kTraceAudioDevice, id_, "OpenSL aec enabled");
+          aec_->setEnabled(true);
+        } else {
+          WEBRTC_TRACE(kTraceError, kTraceAudioDevice, id_, "OpenSL aec disabled: %d", status);
+          delete aec_;
+          aec_ = NULL;
+        }
+      }
+
+      ns_ = new android::AudioEffect(FX_IID_NS, NULL, 0, 0, 0, sessionId, 0);
+      WEBRTC_TRACE(kTraceError, kTraceAudioDevice, id_, "OpenSL ns: %p", ns_);
+
+      if (ns_) {
+        android::status_t status = ns_->initCheck();
+        if (status == android::NO_ERROR || status == android::ALREADY_EXISTS) {
+          WEBRTC_TRACE(kTraceError, kTraceAudioDevice, id_, "OpenSL ns enabled");
+          ns_->setEnabled(true);
+        } else {
+          WEBRTC_TRACE(kTraceError, kTraceAudioDevice, id_, "OpenSL ns disabled: %d", status);
+          delete ns_;
+          ns_ = NULL;
+        }
+      }
+    }
+  }
+}
+#endif
+
 bool OpenSlesInput::CreateAudioRecorder() {
   if (!event_.Start()) {
     assert(false);
@@ -425,10 +534,19 @@ bool OpenSlesInput::CreateAudioRecorder() {
                                                req),
       false);
 
+#if defined(WEBRTC_GONK) && defined(WEBRTC_HARDWARE_AEC_NS)
+  SetupVoiceMode();
+#endif
+
   // Realize the recorder in synchronous mode.
   OPENSL_RETURN_ON_FAILURE((*sles_recorder_)->Realize(sles_recorder_,
                                                       SL_BOOLEAN_FALSE),
                            false);
+
+#if defined(WEBRTC_GONK) && defined(WEBRTC_HARDWARE_AEC_NS)
+  SetupAECAndNS();
+#endif
+
   OPENSL_RETURN_ON_FAILURE(
       (*sles_recorder_)->GetInterface(sles_recorder_, SL_IID_RECORD_,
                                       static_cast<void*>(&sles_recorder_itf_)),
@@ -444,6 +562,14 @@ bool OpenSlesInput::CreateAudioRecorder() {
 
 void OpenSlesInput::DestroyAudioRecorder() {
   event_.Stop();
+
+#if defined(WEBRTC_GONK) && defined(WEBRTC_HARDWARE_AEC_NS)
+  delete aec_;
+  delete ns_;
+  aec_ = NULL;
+  ns_ = NULL;
+#endif
+
   if (sles_recorder_sbq_itf_) {
     // Release all buffers currently queued up.
     OPENSL_RETURN_ON_FAILURE(

media/webrtc/trunk/webrtc/modules/audio_device/android/opensles_input.h

@@ -17,6 +17,8 @@
 
 #if !defined(WEBRTC_GONK)
 #include "webrtc/modules/audio_device/android/audio_manager_jni.h"
+#else
+#include "media/AudioEffect.h"
 #endif
 #include "webrtc/modules/audio_device/android/low_latency_event.h"
 #include "webrtc/modules/audio_device/include/audio_device.h"
@@ -143,6 +145,11 @@ class OpenSlesInput {
   // etc, so it should be called when starting recording.
   bool CreateAudioRecorder();
   void DestroyAudioRecorder();
+#if defined(WEBRTC_GONK) && defined(WEBRTC_HARDWARE_AEC_NS)
+  void SetupAECAndNS();
+  void SetupVoiceMode();
+  bool CheckPlatformAEC();
+#endif
 
   // When overrun happens there will be more frames received from OpenSL than
   // the desired number of buffers. It is possible to expand the number of
@@ -220,6 +227,10 @@ class OpenSlesInput {
   uint32_t rec_sampling_rate_;
   bool agc_enabled_;
 
+#if defined(WEBRTC_GONK) && defined(WEBRTC_HARDWARE_AEC_NS)
+  android::AudioEffect* aec_;
+  android::AudioEffect* ns_;
+#endif
   // Audio status
   uint16_t recording_delay_;
 

media/webrtc/trunk/webrtc/modules/audio_device/audio_device.gypi

@@ -52,6 +52,11 @@
             '$(NSPR_CFLAGS)',
           ],
         }],
+        ['hardware_aec_ns==1', {
+          'defines': [
+            'WEBRTC_HARDWARE_AEC_NS',
+          ],
+        }],
         ['OS=="linux" or include_alsa_audio==1 or include_pulse_audio==1', {
           'include_dirs': [
             'linux',
@@ -81,7 +86,10 @@
         ['moz_widget_toolkit_gonk==1', {
           'cflags_mozilla': [
             '-I$(ANDROID_SOURCE)/frameworks/wilhelm/include',
+            '-I$(ANDROID_SOURCE)/frameworks/av/include',
             '-I$(ANDROID_SOURCE)/system/media/wilhelm/include',
+            '-I$(ANDROID_SOURCE)/system/media/audio_effects/include',
+            '-I$(ANDROID_SOURCE)/frameworks/native/include',
           ],
           'include_dirs': [
             'android',

modules/libpref/init/all.js

@@ -353,11 +353,16 @@ pref("media.peerconnection.identity.timeout", 10000);
 // kXxxUnchanged = 0, kXxxDefault = 1, and higher values are specific to each
 // setting (for Xxx = Ec, Agc, or Ns).  Defaults are all set to kXxxDefault here.
 pref("media.peerconnection.turn.disable", false);
+#if defined(MOZ_WEBRTC_HARDWARE_AEC_NS)
+pref("media.getusermedia.aec_enabled", false);
+pref("media.getusermedia.noise_enabled", false);
+#else
 pref("media.getusermedia.aec_enabled", true);
+pref("media.getusermedia.noise_enabled", true);
+#endif
+pref("media.getusermedia.noise", 1);
 pref("media.getusermedia.agc_enabled", false);
 pref("media.getusermedia.agc", 1);
-pref("media.getusermedia.noise_enabled", true);
-pref("media.getusermedia.noise", 1);
 // Adjustments for OS-specific input delay (lower bound)
 // Adjustments for OS-specific AudioStream+cubeb+output delay (lower bound)
 #if defined(XP_MACOSX)