From 4452f56e09e0fb3878d74eea6bf588605383a21e Mon Sep 17 00:00:00 2001 From: Paul Adenot Date: Fri, 11 Sep 2020 13:15:57 +0000 Subject: [PATCH] Bug 1663917 - Update libcubeb f39ce8a. r=cubeb-reviewers,kinetik Differential Revision: https://phabricator.services.mozilla.com/D89590 --- media/libcubeb/moz.yaml | 2 +- media/libcubeb/src/cubeb_resampler_internal.h | 33 ++- media/libcubeb/src/cubeb_sun.c | 4 +- media/libcubeb/src/cubeb_wasapi.cpp | 200 ++++++++++++++---- 4 files changed, 194 insertions(+), 45 deletions(-) diff --git a/media/libcubeb/moz.yaml b/media/libcubeb/moz.yaml index fdc6ba1e461d..60191d0147f9 100644 --- a/media/libcubeb/moz.yaml +++ b/media/libcubeb/moz.yaml @@ -19,5 +19,5 @@ origin: license: "ISC" # update.sh will update this value - release: "a971bf1a045b0e5dcaffd2a15c3255677f43cd2d (2020-07-07 11:53:43 +1200)" + release: "f39ce8a726448d6dd4b08949b49ed6937bc080d7 (2020-09-11 15:06:44 +0200)" diff --git a/media/libcubeb/src/cubeb_resampler_internal.h b/media/libcubeb/src/cubeb_resampler_internal.h index 4ed6c7501c8e..3e8b097be0ba 100644 --- a/media/libcubeb/src/cubeb_resampler_internal.h +++ b/media/libcubeb/src/cubeb_resampler_internal.h @@ -35,6 +35,7 @@ MOZ_END_STD_NAMESPACE #include "cubeb_utils.h" #include "cubeb-speex-resampler.h" #include "cubeb_resampler.h" +#include "cubeb_log.h" #include /* This header file contains the internal C++ API of the resamplers, for testing. */ @@ -191,6 +192,19 @@ public: speex_resampler = speex_resampler_init(channels, source_rate, target_rate, quality, &r); assert(r == RESAMPLER_ERR_SUCCESS && "resampler allocation failure"); + + uint32_t input_latency = speex_resampler_get_input_latency(speex_resampler); + const size_t LATENCY_SAMPLES = 8192; + T input_buffer[LATENCY_SAMPLES] = {}; + T output_buffer[LATENCY_SAMPLES] = {}; + uint32_t input_frame_count = input_latency; + uint32_t output_frame_count = LATENCY_SAMPLES; + assert(input_latency * channels <= LATENCY_SAMPLES); + speex_resample( + input_buffer, + &input_frame_count, + output_buffer, + &output_frame_count); } /** Destructor, deallocate the resampler */ @@ -254,7 +268,14 @@ public: speex_resample(resampling_in_buffer.data(), &in_len, resampling_out_buffer.data(), &out_len); - // assert(out_len == output_frame_count); + if (out_len < output_frame_count) { + LOGV("underrun during resampling: got %u frames, expected %zu", (unsigned)out_len, output_frame_count); + // silence the rightmost part + T* data = resampling_out_buffer.data(); + for (uint32_t i = frames_to_samples(out_len); i < frames_to_samples(output_frame_count); i++) { + data[i] = 0; + } + } /* This shifts back any unresampled samples to the beginning of the input buffer. */ @@ -286,11 +307,9 @@ public: uint32_t input_needed_for_output(int32_t output_frame_count) const { assert(output_frame_count >= 0); // Check overflow - int32_t unresampled_frames_left = samples_to_frames(resampling_in_buffer.length()); int32_t resampled_frames_left = samples_to_frames(resampling_out_buffer.length()); - float input_frames_needed = - (output_frame_count - unresampled_frames_left) * resampling_ratio - - resampled_frames_left; + float input_frames_needed = + output_frame_count * resampling_ratio - resampled_frames_left; if (input_frames_needed < 0) { return 0; } @@ -528,6 +547,7 @@ cubeb_resampler_create_internal(cubeb_stream * stream, (output_params && output_params->rate == target_rate)) || (input_params && !output_params && (input_params->rate == target_rate)) || (output_params && !input_params && (output_params->rate == target_rate))) { + LOG("Input and output sample-rate match, target rate of %dHz", target_rate); return new passthrough_resampler(stream, callback, user_ptr, input_params ? input_params->channels : 0, @@ -578,6 +598,7 @@ cubeb_resampler_create_internal(cubeb_stream * stream, } if (input_resampler && output_resampler) { + LOG("Resampling input (%d) and output (%d) to target rate of %dHz", input_params->rate, output_params->rate, target_rate); return new cubeb_resampler_speex, cubeb_resampler_speex_one_way> @@ -585,6 +606,7 @@ cubeb_resampler_create_internal(cubeb_stream * stream, output_resampler.release(), stream, callback, user_ptr); } else if (input_resampler) { + LOG("Resampling input (%d) to target and output rate of %dHz", input_params->rate, target_rate); return new cubeb_resampler_speex, delay_line> @@ -592,6 +614,7 @@ cubeb_resampler_create_internal(cubeb_stream * stream, output_delay.release(), stream, callback, user_ptr); } else { + LOG("Resampling output (%dHz) to target and input rate of %dHz", output_params->rate, target_rate); return new cubeb_resampler_speex, cubeb_resampler_speex_one_way> diff --git a/media/libcubeb/src/cubeb_sun.c b/media/libcubeb/src/cubeb_sun.c index f023242f666d..a0d77ee653ed 100644 --- a/media/libcubeb/src/cubeb_sun.c +++ b/media/libcubeb/src/cubeb_sun.c @@ -362,8 +362,8 @@ sun_stream_stop(cubeb_stream * s) static void sun_stream_destroy(cubeb_stream * s) { - pthread_mutex_destroy(&s->mutex); sun_stream_stop(s); + pthread_mutex_destroy(&s->mutex); if (s->play.fd != -1) { close(s->play.fd); } @@ -666,7 +666,7 @@ sun_stream_get_latency(cubeb_stream * s, uint32_t * latency) #else cubeb_stream_params params; - params.rate = stream->play.info.play.sample_rate; + params.rate = s->play.info.play.sample_rate; return sun_get_min_latency(NULL, params, latency); #endif diff --git a/media/libcubeb/src/cubeb_wasapi.cpp b/media/libcubeb/src/cubeb_wasapi.cpp index 341a6a4e5a54..449b8b56b0ab 100644 --- a/media/libcubeb/src/cubeb_wasapi.cpp +++ b/media/libcubeb/src/cubeb_wasapi.cpp @@ -193,6 +193,10 @@ int wasapi_stream_start(cubeb_stream * stm); void close_wasapi_stream(cubeb_stream * stm); int setup_wasapi_stream(cubeb_stream * stm); ERole pref_to_role(cubeb_stream_prefs param); +int wasapi_create_device(cubeb * ctx, cubeb_device_info& ret, IMMDeviceEnumerator * enumerator, IMMDevice * dev); +void wasapi_destroy_device(cubeb_device_info * device_info); +static int wasapi_enumerate_devices(cubeb * context, cubeb_device_type type, cubeb_device_collection * out); +static int wasapi_device_collection_destroy(cubeb * ctx, cubeb_device_collection * collection); static char const * wstr_to_utf8(wchar_t const * str); static std::unique_ptr utf8_to_wstr(char const * str); @@ -245,9 +249,15 @@ struct cubeb_stream { cubeb_stream_params output_stream_params = { CUBEB_SAMPLE_FLOAT32NE, 0, 0, CUBEB_LAYOUT_UNDEFINED, CUBEB_STREAM_PREF_NONE }; /* A MMDevice role for this stream: either communication or console here. */ ERole role; + /* True if this stream will transport voice-data. */ + bool voice; + /* True if the input device of this stream is using bluetooth handsfree. */ + bool input_bluetooth_handsfree; /* The input and output device, or NULL for default. */ - std::unique_ptr input_device; - std::unique_ptr output_device; + std::unique_ptr input_device_id; + std::unique_ptr output_device_id; + com_ptr input_device; + com_ptr output_device; /* The latency initially requested for this stream, in frames. */ unsigned latency = 0; cubeb_state_callback state_callback = nullptr; @@ -773,6 +783,12 @@ frames_to_hns(cubeb_stream * stm, uint32_t frames) return std::ceil(frames * 10000000.0 / get_rate(stm)); } +REFERENCE_TIME +frames_to_hns(uint32_t rate, uint32_t frames) +{ + return std::ceil(frames * 10000000.0 / rate); +} + /* This returns the size of a frame in the stream, before the eventual upmix occurs. */ static size_t @@ -1037,9 +1053,6 @@ refill_callback_duplex(cubeb_stream * stm) } input_frames = stm->linear_input_buffer->length() / stm->input_stream_params.channels; - if (!input_frames) { - return true; - } rv = get_output_buffer(stm, output_buffer, output_frames); if (!rv) { @@ -1332,17 +1345,11 @@ void wasapi_destroy(cubeb * context); HRESULT register_notification_client(cubeb_stream * stm) { - HRESULT hr = CoCreateInstance(__uuidof(MMDeviceEnumerator), - NULL, CLSCTX_INPROC_SERVER, - IID_PPV_ARGS(stm->device_enumerator.receive())); - if (FAILED(hr)) { - LOG("Could not get device enumerator: %lx", hr); - return hr; - } + assert(stm->device_enumerator); stm->notification_client.reset(new wasapi_endpoint_notification_client(stm->reconfigure_event, stm->role)); - hr = stm->device_enumerator->RegisterEndpointNotificationCallback(stm->notification_client.get()); + HRESULT hr = stm->device_enumerator->RegisterEndpointNotificationCallback(stm->notification_client.get()); if (FAILED(hr)) { LOG("Could not register endpoint notification callback: %lx", hr); stm->notification_client = nullptr; @@ -1370,7 +1377,6 @@ HRESULT unregister_notification_client(cubeb_stream * stm) } stm->notification_client = nullptr; - stm->device_enumerator = nullptr; return S_OK; } @@ -1920,9 +1926,9 @@ int setup_wasapi_stream_one_side(cubeb_stream * stm, uint32_t * buffer_frame_count, HANDLE & event, T & render_or_capture_client, - cubeb_stream_params * mix_params) + cubeb_stream_params * mix_params, + com_ptr& device) { - com_ptr device; HRESULT hr; bool is_loopback = stream_params->prefs & CUBEB_STREAM_PREF_LOOPBACK; if (is_loopback && direction != eCapture) { @@ -2035,6 +2041,43 @@ int setup_wasapi_stream_one_side(cubeb_stream * stm, flags |= AUDCLNT_STREAMFLAGS_EVENTCALLBACK; } + // Sanity check the latency, it may be that the device doesn't support it. + REFERENCE_TIME minimum_period; + REFERENCE_TIME default_period; + hr = audio_client->GetDevicePeriod(&default_period, &minimum_period); + if (FAILED(hr)) { + LOG("Could not get device period: %lx", hr); + return CUBEB_ERROR; + } + + cubeb_device_info device_info; + int rv = wasapi_create_device(stm->context, device_info, stm->device_enumerator.get(), device.get()); + if (rv != CUBEB_OK) { + LOG("Could not get cubeb_device_info."); + } + + uint32_t latency_frames = stm->latency; + + const char* HANDSFREE_TAG = "BTHHFEENUM"; + size_t len = sizeof(HANDSFREE_TAG); + if (direction == eCapture && strncmp(device_info.group_id, HANDSFREE_TAG, len) == 0) { + // Rather high-latency to prevent constant under-runs in this particular + // case of an input device using bluetooth handsfree. + uint32_t default_period_frames = hns_to_frames(device_info.default_rate, default_period); + latency_frames = default_period_frames * 4; + stm->input_bluetooth_handsfree = true; + LOG("Input is a bluetooth device in handsfree, latency increased to %u frames from a default of %u", latency_frames, default_period_frames); + } else { + uint32_t minimum_period_frames = hns_to_frames(device_info.default_rate, minimum_period); + LOG("Input is a not bluetooth handsfree, latency %s to %u frames (minimum %u)", latency_frames < minimum_period_frames ? "increased" : "set", latency_frames, minimum_period_frames); + latency_frames = std::max(latency_frames, minimum_period_frames); + stm->input_bluetooth_handsfree = false; + } + + REFERENCE_TIME latency_hns = frames_to_hns(device_info.default_rate, latency_frames); + + wasapi_destroy_device(&device_info); + #if 0 // See https://bugzilla.mozilla.org/show_bug.cgi?id=1590902 if (initialize_iaudioclient3(audio_client, stm, mix_format, flags, direction)) { LOG("Initialized with IAudioClient3"); @@ -2042,7 +2085,7 @@ int setup_wasapi_stream_one_side(cubeb_stream * stm, #endif hr = audio_client->Initialize(AUDCLNT_SHAREMODE_SHARED, flags, - frames_to_hns(stm, stm->latency), + latency_hns, 0, mix_format.get(), NULL); @@ -2082,6 +2125,54 @@ int setup_wasapi_stream_one_side(cubeb_stream * stm, #undef DIRECTION_NAME +void wasapi_find_matching_output_device(cubeb_stream * stm) { + HRESULT hr; + cubeb_device_info * input_device; + cubeb_device_collection collection; + + // Only try to match to an output device if the input device is a bluetooth + // device that is using the handsfree protocol + if (!stm->input_bluetooth_handsfree) { + return; + } + + wchar_t * tmp = nullptr; + hr = stm->input_device->GetId(&tmp); + if (FAILED(hr)) { + LOG("Couldn't get input device id in wasapi_find_matching_output_device"); + return; + } + com_heap_ptr device_id(tmp); + cubeb_devid input_device_id = intern_device_id(stm->context, device_id.get()); + if (!input_device_id) { + return; + } + + int rv = wasapi_enumerate_devices(stm->context, (cubeb_device_type)(CUBEB_DEVICE_TYPE_INPUT|CUBEB_DEVICE_TYPE_OUTPUT), &collection); + + // Find the input device, and then find the output device with the same group + // id and the same rate. + for (uint32_t i = 0; i < collection.count; i++) { + cubeb_device_info dev = collection.device[i]; + if (dev.devid == input_device_id) { + input_device = &dev; + break; + } + } + + for (uint32_t i = 0; i < collection.count; i++) { + cubeb_device_info dev = collection.device[i]; + if (dev.type == CUBEB_DEVICE_TYPE_OUTPUT && + dev.group_id && !strcmp(dev.group_id, input_device->group_id) && + dev.default_rate == input_device->default_rate) { + LOG("Found matching device for %s: %s", input_device->friendly_name, dev.friendly_name); + stm->output_device_id = utf8_to_wstr(reinterpret_cast(dev.devid)); + } + } + + wasapi_device_collection_destroy(stm->context, &collection); +} + int setup_wasapi_stream(cubeb_stream * stm) { int rv; @@ -2091,17 +2182,18 @@ int setup_wasapi_stream(cubeb_stream * stm) XASSERT((!stm->output_client || !stm->input_client) && "WASAPI stream already setup, close it first."); if (has_input(stm)) { - LOG("(%p) Setup capture: device=%p", stm, stm->input_device.get()); + LOG("(%p) Setup capture: device=%p", stm, stm->input_device_id.get()); rv = setup_wasapi_stream_one_side(stm, &stm->input_stream_params, - stm->input_device.get(), + stm->input_device_id.get(), eCapture, __uuidof(IAudioCaptureClient), stm->input_client, &stm->input_buffer_frame_count, stm->input_available_event, stm->capture_client, - &stm->input_mix_params); + &stm->input_mix_params, + stm->input_device); if (rv != CUBEB_OK) { LOG("Failure to open the input side."); return rv; @@ -2120,6 +2212,14 @@ int setup_wasapi_stream(cubeb_stream * stm) stm->linear_input_buffer->push_silence(stm->input_buffer_frame_count * stm->input_stream_params.channels * silent_buffer_count); + + // If this is a bluetooth device, and the output device is the default + // device, and the default device is the same bluetooth device, pick the + // right output device, running at the same rate and with the same protocol + // as the input. + if (!stm->output_device_id) { + wasapi_find_matching_output_device(stm); + } } // If we don't have an output device but are requesting a loopback device, @@ -2130,31 +2230,32 @@ int setup_wasapi_stream(cubeb_stream * stm) stm->output_stream_params.rate = stm->input_stream_params.rate; stm->output_stream_params.channels = stm->input_stream_params.channels; stm->output_stream_params.layout = stm->input_stream_params.layout; - if (stm->input_device) { - size_t len = wcslen(stm->input_device.get()); + if (stm->input_device_id) { + size_t len = wcslen(stm->input_device_id.get()); std::unique_ptr tmp(new wchar_t[len + 1]); - if (wcsncpy_s(tmp.get(), len + 1, stm->input_device.get(), len) != 0) { + if (wcsncpy_s(tmp.get(), len + 1, stm->input_device_id.get(), len) != 0) { LOG("Failed to copy device identifier while copying input stream" " configuration to output stream configuration to drive loopback."); return CUBEB_ERROR; } - stm->output_device = move(tmp); + stm->output_device_id = move(tmp); } stm->has_dummy_output = true; } if (has_output(stm)) { - LOG("(%p) Setup render: device=%p", stm, stm->output_device.get()); + LOG("(%p) Setup render: device=%p", stm, stm->output_device_id.get()); rv = setup_wasapi_stream_one_side(stm, &stm->output_stream_params, - stm->output_device.get(), + stm->output_device_id.get(), eRender, __uuidof(IAudioRenderClient), stm->output_client, &stm->output_buffer_frame_count, stm->refill_event, stm->render_client, - &stm->output_mix_params); + &stm->output_mix_params, + stm->output_device); if (rv != CUBEB_OK) { LOG("Failure to open the output side."); return rv; @@ -2213,7 +2314,7 @@ int setup_wasapi_stream(cubeb_stream * stm) target_sample_rate, stm->data_callback, stm->user_ptr, - CUBEB_RESAMPLER_QUALITY_DESKTOP)); + stm->voice ? CUBEB_RESAMPLER_QUALITY_VOIP : CUBEB_RESAMPLER_QUALITY_DESKTOP)); if (!stm->resampler) { LOG("Could not get a resampler"); return CUBEB_ERROR; @@ -2300,21 +2401,31 @@ wasapi_stream_init(cubeb * context, cubeb_stream ** stream, stm->data_callback = data_callback; stm->state_callback = state_callback; stm->user_ptr = user_ptr; + stm->role = eConsole; + stm->input_bluetooth_handsfree = false; - if (stm->output_stream_params.prefs & CUBEB_STREAM_PREF_VOICE || - stm->input_stream_params.prefs & CUBEB_STREAM_PREF_VOICE) { - stm->role = eCommunications; - } else { - stm->role = eConsole; + HRESULT hr = CoCreateInstance(__uuidof(MMDeviceEnumerator), + NULL, CLSCTX_INPROC_SERVER, + IID_PPV_ARGS(stm->device_enumerator.receive())); + if (FAILED(hr)) { + LOG("Could not get device enumerator: %lx", hr); + return hr; } if (input_stream_params) { stm->input_stream_params = *input_stream_params; - stm->input_device = utf8_to_wstr(reinterpret_cast(input_device)); + stm->input_device_id = utf8_to_wstr(reinterpret_cast(input_device)); } if (output_stream_params) { stm->output_stream_params = *output_stream_params; - stm->output_device = utf8_to_wstr(reinterpret_cast(output_device)); + stm->output_device_id = utf8_to_wstr(reinterpret_cast(output_device)); + } + + if (stm->output_stream_params.prefs & CUBEB_STREAM_PREF_VOICE || + stm->input_stream_params.prefs & CUBEB_STREAM_PREF_VOICE) { + stm->voice = true; + } else { + stm->voice = false; } switch (output_stream_params ? output_stream_params->format : input_stream_params->format) { @@ -2431,6 +2542,8 @@ void wasapi_stream_destroy(cubeb_stream * stm) // must be destroyed in wasapi_stream_destroy. stm->linear_input_buffer.reset(); + stm->device_enumerator = nullptr; + { auto_lock lock(stm->stream_reset_lock); close_wasapi_stream(stm); @@ -2629,16 +2742,19 @@ int wasapi_stream_get_latency(cubeb_stream * stm, uint32_t * latency) /* The GetStreamLatency method only works if the AudioClient has been initialized. */ if (!stm->output_client) { + LOG("get_latency: No output_client."); return CUBEB_ERROR; } REFERENCE_TIME latency_hns; HRESULT hr = stm->output_client->GetStreamLatency(&latency_hns); if (FAILED(hr)) { + LOG("GetStreamLatency failed %lx.", hr); return CUBEB_ERROR; } // This happens on windows 10: no error, but always 0 for latency. if (latency_hns == 0) { + LOG("GetStreamLatency returned 0, using workaround."); double delay_s = current_stream_delay(stm); // convert to sample-frames *latency = delay_s * stm->output_stream_params.rate; @@ -2646,6 +2762,8 @@ int wasapi_stream_get_latency(cubeb_stream * stm, uint32_t * latency) *latency = hns_to_frames(stm, latency_hns); } + LOG("Output latency %u frames.", *latency); + return CUBEB_OK; } @@ -2655,12 +2773,14 @@ int wasapi_stream_get_input_latency(cubeb_stream * stm, uint32_t * latency) XASSERT(stm && latency); if (!has_input(stm)) { + LOG("Input latency queried on an output-only stream."); return CUBEB_ERROR; } auto_lock lock(stm->stream_reset_lock); if (stm->input_latency_hns == LATENCY_NOT_AVAILABLE_YET) { + LOG("Input latency not available yet."); return CUBEB_ERROR; } @@ -2868,6 +2988,13 @@ wasapi_create_device(cubeb * ctx, cubeb_device_info& ret, IMMDeviceEnumerator * return CUBEB_OK; } +void +wasapi_destroy_device(cubeb_device_info * device) +{ + delete [] device->friendly_name; + delete [] device->group_id; +} + static int wasapi_enumerate_devices(cubeb * context, cubeb_device_type type, cubeb_device_collection * out) @@ -2931,8 +3058,7 @@ wasapi_device_collection_destroy(cubeb * /*ctx*/, cubeb_device_collection * coll for (size_t n = 0; n < collection->count; n++) { cubeb_device_info& dev = collection->device[n]; - delete [] dev.friendly_name; - delete [] dev.group_id; + wasapi_destroy_device(&dev); } delete [] collection->device;