From c888e581c18eb94a73747e329d038a421c1d6e57 Mon Sep 17 00:00:00 2001 From: Paul Adenot Date: Tue, 2 Feb 2016 18:24:16 +0100 Subject: [PATCH] Add a resampler that can synchronously resample audio. Depending on if its used for input or output audio, it can be used in two different ways, so it has to expose its input buffer to minimise copy (when the input of the resampler is output audio, directly written inside an audio callback), and to be able to expose an internal buffer so that the output of the resampling process can directly be passed to a callback. When created, additional latency can be added to the pipeline so that multiple resamplers of different filter length (=latency) can be synchronized. --- .gitignore | 7 +- Makefile.am | 3 + configure.ac | 2 +- src/cubeb_resampler.cpp | 152 +++++++-------------- src/cubeb_resampler_internal.h | 242 +++++++++++++++++++++++++++++++++ test/test_resampler.cpp | 225 ++++++++++++++++++++++++++++++ 6 files changed, 526 insertions(+), 105 deletions(-) create mode 100644 src/cubeb_resampler_internal.h create mode 100644 test/test_resampler.cpp diff --git a/.gitignore b/.gitignore index d00e847..2152806 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,10 @@ *.o *.swp *~ +*.trs +*.raw +*.wav +*.log .deps .dirstamp .libs @@ -47,10 +51,11 @@ test/test_tone test/test_tone.exe test/test_devices test/test_devices.exe +test/test_resampler +test/test_resampler.exe test/test_utils test/test_utils.exe include/cubeb/cubeb-stdint.h test-suite.log test/test_sanity.log test/test_sanity.trs - diff --git a/Makefile.am b/Makefile.am index 95c51e7..c060e7b 100644 --- a/Makefile.am +++ b/Makefile.am @@ -80,6 +80,7 @@ check_PROGRAMS = test/test_sanity \ test/test_audio \ test/test_latency \ test/test_devices \ + test/test_resampler \ test/test_utils \ $(NULL) @@ -98,6 +99,8 @@ test_test_latency_LDADD = -lm src/libcubeb.la $(platform_lib) test_test_devices_SOURCES = test/test_devices.cpp test_test_devices_LDADD = -lm src/libcubeb.la $(platform_lib) +test_test_resampler_SOURCES = test/test_resampler.cpp +test_test_resampler_LDADD = -lm src/libcubeb.la $(platform_lib) src/cubeb_resampler.o test_test_resampler_SOURCES = test/test_utils.cpp diff --git a/configure.ac b/configure.ac index cc257af..41f700e 100644 --- a/configure.ac +++ b/configure.ac @@ -46,7 +46,7 @@ AM_PROG_CC_C_O AC_LIBTOOL_WIN32_DLL AM_PROG_LIBTOOL -NEED_SPEEX=0 +NEED_SPEEX=1 AC_ARG_WITH([pulse], AS_HELP_STRING([--with-pulse], [with PulseAudio @<:@default=check@:>@])) diff --git a/src/cubeb_resampler.cpp b/src/cubeb_resampler.cpp index c41b29e..de2e4fa 100644 --- a/src/cubeb_resampler.cpp +++ b/src/cubeb_resampler.cpp @@ -14,45 +14,8 @@ #endif #include "cubeb_resampler.h" #include "cubeb-speex-resampler.h" - -namespace { - -template -class auto_array -{ -public: - auto_array(uint32_t size) - : data(new T[size]) - {} - - ~auto_array() - { - delete [] data; - } - - T * get() const - { - return data; - } - -private: - T * data; -}; - -long -frame_count_at_rate(long frame_count, float rate) -{ - return static_cast(ceilf(rate * frame_count) + 1); -} - -size_t -frames_to_bytes(cubeb_stream_params params, size_t frames) -{ - assert(params.format == CUBEB_SAMPLE_S16NE || params.format == CUBEB_SAMPLE_FLOAT32NE); - size_t sample_size = params.format == CUBEB_SAMPLE_S16NE ? sizeof(short) : sizeof(float); - size_t frame_size = params.channels * sample_size; - return frame_size * frames; -} +#include "cubeb_resampler_internal.h" +#include "cubeb_utils.h" int to_speex_quality(cubeb_resampler_quality q) @@ -69,72 +32,55 @@ to_speex_quality(cubeb_resampler_quality q) return 0XFFFFFFFF; } } + +template +cubeb_resampler_speex_one_way::cubeb_resampler_speex_one_way(int32_t channels, + int32_t source_rate, + int32_t target_rate, + int quality) + : processor(channels) + , resampling_ratio(static_cast(source_rate) / target_rate) + , additional_latency(0) +{ + int r; + speex_resampler = speex_resampler_init(channels, source_rate, + target_rate, quality, &r); + assert(r == RESAMPLER_ERR_SUCCESS && "resampler allocation failure"); +} + +template +cubeb_resampler_speex_one_way::~cubeb_resampler_speex_one_way() +{ + speex_resampler_destroy(speex_resampler); +} + +long noop_resampler::fill(void * input_buffer, long * input_frames_count, + void * output_buffer, long output_frames) +{ + assert(input_buffer && output_buffer && + *input_frames_count >= output_frames|| + !input_buffer && input_frames_count == 0 || + !output_buffer && output_frames== 0); + + if (*input_frames_count != output_frames) { + assert(*input_frames_count > output_frames); + *input_frames_count = output_frames; + } + + return data_callback(stream, user_ptr, + input_buffer, output_buffer, output_frames); +} + +namespace { + +long +frame_count_at_rate(long frame_count, float rate) +{ + return static_cast(ceilf(rate * frame_count) + 1); +} + } // end of anonymous namespace -struct cubeb_resampler { - virtual long fill(void * input_buffer, void * output_buffer, long frames_needed) = 0; - virtual ~cubeb_resampler() {} -}; - -class noop_resampler : public cubeb_resampler { -public: - noop_resampler(cubeb_stream * s, - cubeb_data_callback cb, - void * ptr) - : stream(s) - , data_callback(cb) - , user_ptr(ptr) - { - } - - virtual long fill(void * input_buffer, void * output_buffer, long frames_needed) - { - long got = data_callback(stream, user_ptr, input_buffer, output_buffer, frames_needed); - assert(got <= frames_needed); - return got; - } - -private: - cubeb_stream * const stream; - const cubeb_data_callback data_callback; - void * const user_ptr; -}; - -class cubeb_resampler_speex : public cubeb_resampler { -public: - cubeb_resampler_speex(SpeexResamplerState * r, cubeb_stream * s, - cubeb_stream_params params, uint32_t out_rate, - cubeb_data_callback cb, long max_count, - void * ptr); - - virtual ~cubeb_resampler_speex(); - - virtual long fill(void * input_buffer, void * output_buffer, long frames_needed); - -private: - SpeexResamplerState * const speex_resampler; - cubeb_stream * const stream; - const cubeb_stream_params stream_params; - const cubeb_data_callback data_callback; - void * const user_ptr; - - // Maximum number of frames we can be requested in a callback. - const long buffer_frame_count; - // input rate / output rate - const float resampling_ratio; - // Maximum frames that can be stored in |leftover_frames_buffer|. - const uint32_t leftover_frame_size; - // Number of leftover frames stored in |leftover_frames_buffer|. - uint32_t leftover_frame_count; - - // A little buffer to store the leftover frames, - // that is, the samples not consumed by the resampler that we will end up - // using next time fill() is called. - auto_array leftover_frames_buffer; - // A buffer to store frames that will be consumed by the resampler. - auto_array resampling_src_buffer; -}; - cubeb_resampler_speex::cubeb_resampler_speex(SpeexResamplerState * r, cubeb_stream * s, cubeb_stream_params params, diff --git a/src/cubeb_resampler_internal.h b/src/cubeb_resampler_internal.h new file mode 100644 index 0000000..56e8c8f --- /dev/null +++ b/src/cubeb_resampler_internal.h @@ -0,0 +1,242 @@ +/* + * Copyright © 2016 Mozilla Foundation + * + * This program is made available under an ISC-style license. See the + * accompanying file LICENSE for details. + */ + +#if !defined(CUBEB_RESAMPLER_INTERNAL) +#define CUBEB_RESAMPLER_INTERNAL + +#include +#include +#include +#include "cubeb/cubeb.h" +#include "cubeb_utils.h" +#include "cubeb-speex-resampler.h" +#include "cubeb_resampler.h" +#include + +/* This header file contains the internal C++ API of the resamplers, for testing. */ + +int to_speex_quality(cubeb_resampler_quality q); + +template +class cubeb_resampler_speex_one_way; + +struct cubeb_resampler { + virtual long fill(void * input_buffer, long * input_frames_count, + void * output_buffer, long frames_needed) = 0; + virtual long latency() = 0; + virtual ~cubeb_resampler() {} +}; + +class noop_resampler : public cubeb_resampler { +public: + noop_resampler(cubeb_stream * s, + cubeb_data_callback cb, + void * ptr) + : stream(s) + , data_callback(cb) + , user_ptr(ptr) + { + } + + virtual long fill(void * input_buffer, long * input_frames_count, + void * output_buffer, long output_frames); + + virtual long latency() + { + return 0; + } + +private: + cubeb_stream * const stream; + const cubeb_data_callback data_callback; + void * const user_ptr; +}; + +/** Base class for processors. This is just used to share methods for now. */ +class processor { +public: + processor(uint32_t channels) + : channels(channels) + {} +protected: + size_t frames_to_samples(size_t frames) + { + return frames * channels; + } + size_t samples_to_frames(size_t samples) + { + assert(!(samples % channels)); + return samples / channels; + } + /** The number of channel of the audio buffers to be resampled. */ + const uint32_t channels; +}; + +/** Handles one way of a (possibly) duplex resampler, working on interleaved + * audio buffers of type T. This class is designed so that the number of frames + * coming out of the resampler can be precisely controled. It manages its own + * input buffer, and can use the caller's output buffer, or allocate its own. */ +template +class cubeb_resampler_speex_one_way : public processor { +public: + /** The sample type of this resampler, either 16-bit integers or 32-bit + * floats. */ + typedef T sample_type; + /** Construct a resampler resampling from #source_rate to #target_rate, that + * can be arbitrary, strictly positive number. + * @parameter channels The number of channels this resampler will resample. + * @parameter source_rate The sample-rate of the audio input. + * @parameter target_rate The sample-rate of the audio output. + * @parameter quality A number between 0 (fast, low quality) and 10 (slow, + * high quality). */ + cubeb_resampler_speex_one_way(int32_t channels, + int32_t source_rate, + int32_t target_rate, + int quality); + + /** Destructor, deallocate the resampler */ + virtual ~cubeb_resampler_speex_one_way(); + + /** Sometimes, it is necessary to add latency on one way of a two-way + * resampler so that the stream are synchronized. This must be called only on + * a fresh resampler, otherwise, silent samples will be inserted in the + * stream. + * @param frames the number of frames of latency to add. */ + void add_latency(size_t frames) + { + additional_latency += frames; + resampling_in_buffer.push(frames_to_samples(frames)); + } + + /* Fill the resampler with `input_frame_count` frames. */ + void input(T * input_buffer, size_t input_frame_count) + { + resampling_in_buffer.push(input_buffer, + frames_to_samples(input_frame_count)); + } + + /** Outputs exactly `output_frame_count` into `output_buffer`. + * `output_buffer` has to be at least `output_frame_count` long. */ + void output(T * output_buffer, size_t output_frame_count) + { + uint32_t in_len = samples_to_frames(resampling_in_buffer.length()); + uint32_t out_len = output_frame_count; + + speex_resample(resampling_in_buffer.data(), &in_len, + output_buffer, &out_len); + + assert(out_len == output_frame_count); + + /* This shifts back any unresampled samples to the beginning of the input + buffer. */ + resampling_in_buffer.pop(nullptr, frames_to_samples(in_len)); + } + + /** Drains the resampler, emptying the input buffer, and returning the number + * of frames written to `output_buffer`, that can be less than + * `output_frame_count`. */ + size_t drain(T * output_buffer, size_t output_frame_count) + { + uint32_t in_len = samples_to_frames(resampling_in_buffer.length()); + uint32_t out_len = output_frame_count; + + speex_resample(resampling_in_buffer.data(), &in_len, + output_buffer, &out_len); + + /* This shifts back any unresampled samples to the beginning of the input + buffer. */ + resampling_in_buffer.pop(nullptr, frames_to_samples(in_len)); + + // assert(resampling_in_buffer.length() == 0); + + return out_len; + } + + /** Returns a buffer containing exactly `output_frame_count` resampled frames. + * The consumer should not hold onto the pointer. */ + T * output(size_t output_frame_count) + { + if (resampling_out_buffer.capacity() < frames_to_samples(output_frame_count)) { + resampling_out_buffer.resize(frames_to_samples(output_frame_count)); + } + + uint32_t in_len = samples_to_frames(resampling_in_buffer.length()); + uint32_t out_len = output_frame_count; + + speex_resample(resampling_in_buffer.data(), &in_len, + resampling_out_buffer.data(), &out_len); + + assert(out_len == output_frame_count); + + /* This shifts back any unresampled samples to the beginning of the input + buffer. */ + resampling_in_buffer.pop(nullptr, frames_to_samples(in_len)); + + return resampling_out_buffer.data(); + } + + /** Get the l atency of the resampler, in output frames. */ + uint32_t latency() const + { + /* The documentation of the resampler talks about "samples" here, but it + * only consider a single channel here so it's the same number of frames. */ + return speex_resampler_get_output_latency(speex_resampler) + additional_latency; + } + + /** Returns the number of frames to pass in the input of the resampler to have + * exactly `output_frame_count` resampled frames. This can return a number + * slightly bigger than what is strictly necessary, but it guaranteed that the + * number of output frames will be exactly equal. */ + uint32_t input_needed_for_output(uint32_t output_frame_count) + { + return ceil(output_frame_count * resampling_ratio) + 1 + - resampling_in_buffer.length() / channels; + } + + /** Returns a pointer to the input buffer, that contains empty space for at + * least `frame_count` elements. This is useful so that consumer can directly + * write into the input buffer of the resampler. The pointer returned is + * adjusted so that leftover data are not overwritten. + */ + T * input_buffer(size_t frame_count) + { + size_t prev_length = resampling_in_buffer.length(); + resampling_in_buffer.push(frames_to_samples(frame_count)); + return resampling_in_buffer.data() + prev_length; + } +private: + /** Wrapper for the speex resampling functions to have a typed + * interface. */ + void speex_resample(float * input_buffer, uint32_t * input_frame_count, + float * output_buffer, uint32_t * output_frame_count) + { + speex_resampler_process_interleaved_float(speex_resampler, + input_buffer, input_frame_count, + output_buffer, output_frame_count); + } + + void speex_resample(short * input_buffer, uint32_t * input_frame_count, + short * output_buffer, uint32_t * output_frame_count) + { + speex_resampler_process_interleaved_int(speex_resampler, + input_buffer, input_frame_count, + output_buffer, output_frame_count); + } + /** The state for the speex resampler used internaly. */ + SpeexResamplerState * speex_resampler; + /** Source rate / target rate. */ + const float resampling_ratio; + /** Storage for the input frames, to be resampled. Also contains + * any unresampled frames after resampling. */ + auto_array resampling_in_buffer; + /* Storage for the resampled frames, to be passed back to the caller. */ + auto_array resampling_out_buffer; + /** Additional latency inserted into the pipeline for synchronisation. */ + uint32_t additional_latency; +}; + +#endif /* CUBEB_RESAMPLER_INTERNAL */ diff --git a/test/test_resampler.cpp b/test/test_resampler.cpp new file mode 100644 index 0000000..2fefef1 --- /dev/null +++ b/test/test_resampler.cpp @@ -0,0 +1,225 @@ +/* + * Copyright © 2016 Mozilla Foundation + * + * This program is made available under an ISC-style license. See the + * accompanying file LICENSE for details. + */ + +#define OUTSIDE_SPEEX +#define RANDOM_PREFIX speex + +#include "cubeb/cubeb.h" +#include "cubeb_utils.h" +#include "cubeb_resampler.h" +#include "cubeb_resampler_internal.h" +#include +#include +#include +#include + +/* Windows cmath USE_MATH_DEFINE thing... */ +const float PI = 3.14159265359f; +/* Some standard sample rates we're testing with. */ +const int sample_rates[] = { + 8000, + 16000, + 32000, + 44100, + 48000, + 88200, + 96000, + 192000 +}; +/* The maximum number of channels we're resampling. */ +const uint32_t max_channels = 2; +/* The minimum an maximum number of milliseconds we're resampling for. This is + * used to simulate the fact that the audio stream is resampled in chunks, + * because audio is delivered using callbacks. */ +const uint32_t min_chunks = 10; /* ms */ +const uint32_t max_chunks = 30; /* ms */ + +#define DUMP_ARRAYS +#ifdef DUMP_ARRAYS +/** + * Files produced by dump(...) can be converted to .wave files using: + * + * sox -c -r -e float -b 32 file.raw file.wav + * + * for floating-point audio, or: + * + * sox -c -r -e unsigned -b 16 file.raw file.wav + * + * for 16bit integer audio. + */ + +/* Use the correct implementation of fopen, depending on the platform. */ +void fopen_portable(FILE ** f, const char * name, const char * mode) +{ +#ifdef WIN32 + fopen_s(f, name, mode); +#else + *f = fopen(name, mode); +#endif +} + +template +void dump(const char * name, T * frames, size_t count) +{ + FILE * file; + fopen_portable(&file, name, "wb"); + + if (!file) { + fprintf(stderr, "error opening %s\n", name); + return; + } + + if (count != fwrite(frames, sizeof(T), count, file)) { + fprintf(stderr, "error writing to %s\n", name); + return; + } + fclose(file); +} +#else +template +void dump(const char * name, T * frames, size_t count) +{ } +#endif + +// The more the ratio is far from 1, the more we accept a big error. +float epsilon_tweak_ratio(float ratio) +{ + return ratio >= 1 ? ratio : 1 / ratio; +} + +// Epsilon values for comparing resampled data to expected data. +// The bigger the resampling ratio is, the more lax we are about errors. +template +T epsilon(float ratio); + +template<> +float epsilon(float ratio) { + return 0.08f * epsilon_tweak_ratio(ratio); +} + +template<> +int16_t epsilon(float ratio) { + return static_cast(10 * epsilon_tweak_ratio(ratio)); +} + +/** + * This takes sine waves with a certain `channels` count, `source_rate`, and + * resample them, by chunk of `chunk_duration` milliseconds, to `target_rate`. + * Then a sample-wise comparison is performed against a sine wave generated at + * the correct rate. + */ +template +void test_resampler_one_way(uint32_t channels, int32_t source_rate, int32_t target_rate, float chunk_duration) +{ + size_t chunk_duration_in_source_frames = static_cast(ceil(chunk_duration * source_rate / 1000.)); + float resampling_ratio = static_cast(source_rate) / target_rate; + cubeb_resampler_speex_one_way resampler(channels, source_rate, target_rate, 3); + auto_array source(channels * source_rate * 10); + auto_array destination(channels * target_rate * 10); + auto_array expected(channels * target_rate * 10); + uint32_t phase_index = 0; + uint32_t offset = 0; + const uint32_t buf_len = 2; /* seconds */ + + // generate a sine wave in each channel, at the source sample rate + source.push(channels * source_rate * buf_len); + while(offset != source.length()) { + float p = phase_index++ / static_cast(source_rate); + for (uint32_t j = 0; j < channels; j++) { + source.data()[offset++] = 0.5 * sin(440. * 2 * PI * p); + } + } + + dump("input.raw", source.data(), source.length()); + + expected.push(channels * target_rate * buf_len); + // generate a sine wave in each channel, at the target sample rate. + // Insert silent samples at the beginning to account for the resampler latency. + offset = resampler.latency() * channels; + for (uint32_t i = 0; i < offset; i++) { + expected.data()[i] = 0.0f; + } + phase_index = 0; + while (offset != expected.length()) { + float p = phase_index++ / static_cast(target_rate); + for (uint32_t j = 0; j < channels; j++) { + expected.data()[offset++] = 0.5 * sin(440. * 2 * PI * p); + } + } + + dump("expected.raw", expected.data(), expected.length()); + + // resample by chunk + uint32_t write_offset = 0; + destination.push(channels * target_rate * buf_len); + while (write_offset < destination.length()) + { + size_t output_frames = static_cast(floor(chunk_duration_in_source_frames / resampling_ratio)); + uint32_t input_frames = resampler.input_needed_for_output(output_frames); + resampler.input(source.data(), input_frames); + source.pop(nullptr, input_frames * channels); + resampler.output(destination.data() + write_offset, + std::min(output_frames, (destination.length() - write_offset) / channels)); + write_offset += output_frames * channels; + } + + dump("output.raw", destination.data(), expected.length()); + + // compare, taking the latency into account + bool fuzzy_equal = true; + for (uint32_t i = resampler.latency() + 1; i < expected.length(); i++) { + float diff = abs(expected.data()[i] - destination.data()[i]); + if (diff > epsilon(resampling_ratio)) { + fprintf(stderr, "divergence at %d: %f %f (delta %f)\n", i, expected.data()[i], destination.data()[i], diff); + fuzzy_equal = false; + } + } + assert(fuzzy_equal); +} + +template +cubeb_sample_format cubeb_format(); + +template<> +cubeb_sample_format cubeb_format() +{ + return CUBEB_SAMPLE_FLOAT32NE; +} + +template<> +cubeb_sample_format cubeb_format() +{ + return CUBEB_SAMPLE_S16NE; +} + + +#define array_size(x) (sizeof(x) / sizeof(x[0])) + +void test_resamplers_one_way() +{ + /* Test one way resamplers */ + for (uint32_t channels = 1; channels <= max_channels; channels++) { + for (uint32_t source_rate = 0; source_rate < array_size(sample_rates); source_rate++) { + for (uint32_t dest_rate = 0; dest_rate < array_size(sample_rates); dest_rate++) { + for (uint32_t chunk_duration = min_chunks; chunk_duration < max_chunks; chunk_duration++) { + printf("one_way: channels: %d, source_rate: %d, dest_rate: %d, chunk_duration: %d\n", + channels, sample_rates[source_rate], sample_rates[dest_rate], chunk_duration); + test_resampler_one_way(channels, sample_rates[source_rate], + sample_rates[dest_rate], chunk_duration); + } + } + } + } +} + + +int main() +{ + test_resamplers_one_way(); + + return 0; +}