Add a resampler that can synchronously resample audio.

Depending on if its used for input or output audio, it can be used in two
different ways, so it has to expose its input buffer to minimise copy (when the
input of the resampler is output audio, directly written inside an audio
callback), and to be able to expose an internal buffer so that the output of
the resampling process can directly be passed to a callback.

When created, additional latency can be added to the pipeline so that multiple
resamplers of different filter length (=latency) can be synchronized.
This commit is contained in:
Paul Adenot 2016-02-02 18:24:16 +01:00
Родитель 1f13325871
Коммит c888e581c1
6 изменённых файлов: 526 добавлений и 105 удалений

7
.gitignore поставляемый
Просмотреть файл

@ -2,6 +2,10 @@
*.o
*.swp
*~
*.trs
*.raw
*.wav
*.log
.deps
.dirstamp
.libs
@ -47,10 +51,11 @@ test/test_tone
test/test_tone.exe
test/test_devices
test/test_devices.exe
test/test_resampler
test/test_resampler.exe
test/test_utils
test/test_utils.exe
include/cubeb/cubeb-stdint.h
test-suite.log
test/test_sanity.log
test/test_sanity.trs

Просмотреть файл

@ -80,6 +80,7 @@ check_PROGRAMS = test/test_sanity \
test/test_audio \
test/test_latency \
test/test_devices \
test/test_resampler \
test/test_utils \
$(NULL)
@ -98,6 +99,8 @@ test_test_latency_LDADD = -lm src/libcubeb.la $(platform_lib)
test_test_devices_SOURCES = test/test_devices.cpp
test_test_devices_LDADD = -lm src/libcubeb.la $(platform_lib)
test_test_resampler_SOURCES = test/test_resampler.cpp
test_test_resampler_LDADD = -lm src/libcubeb.la $(platform_lib) src/cubeb_resampler.o
test_test_resampler_SOURCES = test/test_utils.cpp

Просмотреть файл

@ -46,7 +46,7 @@ AM_PROG_CC_C_O
AC_LIBTOOL_WIN32_DLL
AM_PROG_LIBTOOL
NEED_SPEEX=0
NEED_SPEEX=1
AC_ARG_WITH([pulse],
AS_HELP_STRING([--with-pulse], [with PulseAudio @<:@default=check@:>@]))

Просмотреть файл

@ -14,45 +14,8 @@
#endif
#include "cubeb_resampler.h"
#include "cubeb-speex-resampler.h"
namespace {
template<typename T>
class auto_array
{
public:
auto_array(uint32_t size)
: data(new T[size])
{}
~auto_array()
{
delete [] data;
}
T * get() const
{
return data;
}
private:
T * data;
};
long
frame_count_at_rate(long frame_count, float rate)
{
return static_cast<long>(ceilf(rate * frame_count) + 1);
}
size_t
frames_to_bytes(cubeb_stream_params params, size_t frames)
{
assert(params.format == CUBEB_SAMPLE_S16NE || params.format == CUBEB_SAMPLE_FLOAT32NE);
size_t sample_size = params.format == CUBEB_SAMPLE_S16NE ? sizeof(short) : sizeof(float);
size_t frame_size = params.channels * sample_size;
return frame_size * frames;
}
#include "cubeb_resampler_internal.h"
#include "cubeb_utils.h"
int
to_speex_quality(cubeb_resampler_quality q)
@ -69,72 +32,55 @@ to_speex_quality(cubeb_resampler_quality q)
return 0XFFFFFFFF;
}
}
template<typename T>
cubeb_resampler_speex_one_way<T>::cubeb_resampler_speex_one_way(int32_t channels,
int32_t source_rate,
int32_t target_rate,
int quality)
: processor(channels)
, resampling_ratio(static_cast<float>(source_rate) / target_rate)
, additional_latency(0)
{
int r;
speex_resampler = speex_resampler_init(channels, source_rate,
target_rate, quality, &r);
assert(r == RESAMPLER_ERR_SUCCESS && "resampler allocation failure");
}
template<typename T>
cubeb_resampler_speex_one_way<T>::~cubeb_resampler_speex_one_way()
{
speex_resampler_destroy(speex_resampler);
}
long noop_resampler::fill(void * input_buffer, long * input_frames_count,
void * output_buffer, long output_frames)
{
assert(input_buffer && output_buffer &&
*input_frames_count >= output_frames||
!input_buffer && input_frames_count == 0 ||
!output_buffer && output_frames== 0);
if (*input_frames_count != output_frames) {
assert(*input_frames_count > output_frames);
*input_frames_count = output_frames;
}
return data_callback(stream, user_ptr,
input_buffer, output_buffer, output_frames);
}
namespace {
long
frame_count_at_rate(long frame_count, float rate)
{
return static_cast<long>(ceilf(rate * frame_count) + 1);
}
} // end of anonymous namespace
struct cubeb_resampler {
virtual long fill(void * input_buffer, void * output_buffer, long frames_needed) = 0;
virtual ~cubeb_resampler() {}
};
class noop_resampler : public cubeb_resampler {
public:
noop_resampler(cubeb_stream * s,
cubeb_data_callback cb,
void * ptr)
: stream(s)
, data_callback(cb)
, user_ptr(ptr)
{
}
virtual long fill(void * input_buffer, void * output_buffer, long frames_needed)
{
long got = data_callback(stream, user_ptr, input_buffer, output_buffer, frames_needed);
assert(got <= frames_needed);
return got;
}
private:
cubeb_stream * const stream;
const cubeb_data_callback data_callback;
void * const user_ptr;
};
class cubeb_resampler_speex : public cubeb_resampler {
public:
cubeb_resampler_speex(SpeexResamplerState * r, cubeb_stream * s,
cubeb_stream_params params, uint32_t out_rate,
cubeb_data_callback cb, long max_count,
void * ptr);
virtual ~cubeb_resampler_speex();
virtual long fill(void * input_buffer, void * output_buffer, long frames_needed);
private:
SpeexResamplerState * const speex_resampler;
cubeb_stream * const stream;
const cubeb_stream_params stream_params;
const cubeb_data_callback data_callback;
void * const user_ptr;
// Maximum number of frames we can be requested in a callback.
const long buffer_frame_count;
// input rate / output rate
const float resampling_ratio;
// Maximum frames that can be stored in |leftover_frames_buffer|.
const uint32_t leftover_frame_size;
// Number of leftover frames stored in |leftover_frames_buffer|.
uint32_t leftover_frame_count;
// A little buffer to store the leftover frames,
// that is, the samples not consumed by the resampler that we will end up
// using next time fill() is called.
auto_array<uint8_t> leftover_frames_buffer;
// A buffer to store frames that will be consumed by the resampler.
auto_array<uint8_t> resampling_src_buffer;
};
cubeb_resampler_speex::cubeb_resampler_speex(SpeexResamplerState * r,
cubeb_stream * s,
cubeb_stream_params params,

Просмотреть файл

@ -0,0 +1,242 @@
/*
* Copyright © 2016 Mozilla Foundation
*
* This program is made available under an ISC-style license. See the
* accompanying file LICENSE for details.
*/
#if !defined(CUBEB_RESAMPLER_INTERNAL)
#define CUBEB_RESAMPLER_INTERNAL
#include <cmath>
#include <cassert>
#include <algorithm>
#include "cubeb/cubeb.h"
#include "cubeb_utils.h"
#include "cubeb-speex-resampler.h"
#include "cubeb_resampler.h"
#include <stdio.h>
/* This header file contains the internal C++ API of the resamplers, for testing. */
int to_speex_quality(cubeb_resampler_quality q);
template<typename T>
class cubeb_resampler_speex_one_way;
struct cubeb_resampler {
virtual long fill(void * input_buffer, long * input_frames_count,
void * output_buffer, long frames_needed) = 0;
virtual long latency() = 0;
virtual ~cubeb_resampler() {}
};
class noop_resampler : public cubeb_resampler {
public:
noop_resampler(cubeb_stream * s,
cubeb_data_callback cb,
void * ptr)
: stream(s)
, data_callback(cb)
, user_ptr(ptr)
{
}
virtual long fill(void * input_buffer, long * input_frames_count,
void * output_buffer, long output_frames);
virtual long latency()
{
return 0;
}
private:
cubeb_stream * const stream;
const cubeb_data_callback data_callback;
void * const user_ptr;
};
/** Base class for processors. This is just used to share methods for now. */
class processor {
public:
processor(uint32_t channels)
: channels(channels)
{}
protected:
size_t frames_to_samples(size_t frames)
{
return frames * channels;
}
size_t samples_to_frames(size_t samples)
{
assert(!(samples % channels));
return samples / channels;
}
/** The number of channel of the audio buffers to be resampled. */
const uint32_t channels;
};
/** Handles one way of a (possibly) duplex resampler, working on interleaved
* audio buffers of type T. This class is designed so that the number of frames
* coming out of the resampler can be precisely controled. It manages its own
* input buffer, and can use the caller's output buffer, or allocate its own. */
template<typename T>
class cubeb_resampler_speex_one_way : public processor {
public:
/** The sample type of this resampler, either 16-bit integers or 32-bit
* floats. */
typedef T sample_type;
/** Construct a resampler resampling from #source_rate to #target_rate, that
* can be arbitrary, strictly positive number.
* @parameter channels The number of channels this resampler will resample.
* @parameter source_rate The sample-rate of the audio input.
* @parameter target_rate The sample-rate of the audio output.
* @parameter quality A number between 0 (fast, low quality) and 10 (slow,
* high quality). */
cubeb_resampler_speex_one_way(int32_t channels,
int32_t source_rate,
int32_t target_rate,
int quality);
/** Destructor, deallocate the resampler */
virtual ~cubeb_resampler_speex_one_way();
/** Sometimes, it is necessary to add latency on one way of a two-way
* resampler so that the stream are synchronized. This must be called only on
* a fresh resampler, otherwise, silent samples will be inserted in the
* stream.
* @param frames the number of frames of latency to add. */
void add_latency(size_t frames)
{
additional_latency += frames;
resampling_in_buffer.push(frames_to_samples(frames));
}
/* Fill the resampler with `input_frame_count` frames. */
void input(T * input_buffer, size_t input_frame_count)
{
resampling_in_buffer.push(input_buffer,
frames_to_samples(input_frame_count));
}
/** Outputs exactly `output_frame_count` into `output_buffer`.
* `output_buffer` has to be at least `output_frame_count` long. */
void output(T * output_buffer, size_t output_frame_count)
{
uint32_t in_len = samples_to_frames(resampling_in_buffer.length());
uint32_t out_len = output_frame_count;
speex_resample(resampling_in_buffer.data(), &in_len,
output_buffer, &out_len);
assert(out_len == output_frame_count);
/* This shifts back any unresampled samples to the beginning of the input
buffer. */
resampling_in_buffer.pop(nullptr, frames_to_samples(in_len));
}
/** Drains the resampler, emptying the input buffer, and returning the number
* of frames written to `output_buffer`, that can be less than
* `output_frame_count`. */
size_t drain(T * output_buffer, size_t output_frame_count)
{
uint32_t in_len = samples_to_frames(resampling_in_buffer.length());
uint32_t out_len = output_frame_count;
speex_resample(resampling_in_buffer.data(), &in_len,
output_buffer, &out_len);
/* This shifts back any unresampled samples to the beginning of the input
buffer. */
resampling_in_buffer.pop(nullptr, frames_to_samples(in_len));
// assert(resampling_in_buffer.length() == 0);
return out_len;
}
/** Returns a buffer containing exactly `output_frame_count` resampled frames.
* The consumer should not hold onto the pointer. */
T * output(size_t output_frame_count)
{
if (resampling_out_buffer.capacity() < frames_to_samples(output_frame_count)) {
resampling_out_buffer.resize(frames_to_samples(output_frame_count));
}
uint32_t in_len = samples_to_frames(resampling_in_buffer.length());
uint32_t out_len = output_frame_count;
speex_resample(resampling_in_buffer.data(), &in_len,
resampling_out_buffer.data(), &out_len);
assert(out_len == output_frame_count);
/* This shifts back any unresampled samples to the beginning of the input
buffer. */
resampling_in_buffer.pop(nullptr, frames_to_samples(in_len));
return resampling_out_buffer.data();
}
/** Get the l atency of the resampler, in output frames. */
uint32_t latency() const
{
/* The documentation of the resampler talks about "samples" here, but it
* only consider a single channel here so it's the same number of frames. */
return speex_resampler_get_output_latency(speex_resampler) + additional_latency;
}
/** Returns the number of frames to pass in the input of the resampler to have
* exactly `output_frame_count` resampled frames. This can return a number
* slightly bigger than what is strictly necessary, but it guaranteed that the
* number of output frames will be exactly equal. */
uint32_t input_needed_for_output(uint32_t output_frame_count)
{
return ceil(output_frame_count * resampling_ratio) + 1
- resampling_in_buffer.length() / channels;
}
/** Returns a pointer to the input buffer, that contains empty space for at
* least `frame_count` elements. This is useful so that consumer can directly
* write into the input buffer of the resampler. The pointer returned is
* adjusted so that leftover data are not overwritten.
*/
T * input_buffer(size_t frame_count)
{
size_t prev_length = resampling_in_buffer.length();
resampling_in_buffer.push(frames_to_samples(frame_count));
return resampling_in_buffer.data() + prev_length;
}
private:
/** Wrapper for the speex resampling functions to have a typed
* interface. */
void speex_resample(float * input_buffer, uint32_t * input_frame_count,
float * output_buffer, uint32_t * output_frame_count)
{
speex_resampler_process_interleaved_float(speex_resampler,
input_buffer, input_frame_count,
output_buffer, output_frame_count);
}
void speex_resample(short * input_buffer, uint32_t * input_frame_count,
short * output_buffer, uint32_t * output_frame_count)
{
speex_resampler_process_interleaved_int(speex_resampler,
input_buffer, input_frame_count,
output_buffer, output_frame_count);
}
/** The state for the speex resampler used internaly. */
SpeexResamplerState * speex_resampler;
/** Source rate / target rate. */
const float resampling_ratio;
/** Storage for the input frames, to be resampled. Also contains
* any unresampled frames after resampling. */
auto_array<T> resampling_in_buffer;
/* Storage for the resampled frames, to be passed back to the caller. */
auto_array<T> resampling_out_buffer;
/** Additional latency inserted into the pipeline for synchronisation. */
uint32_t additional_latency;
};
#endif /* CUBEB_RESAMPLER_INTERNAL */

225
test/test_resampler.cpp Normal file
Просмотреть файл

@ -0,0 +1,225 @@
/*
* Copyright © 2016 Mozilla Foundation
*
* This program is made available under an ISC-style license. See the
* accompanying file LICENSE for details.
*/
#define OUTSIDE_SPEEX
#define RANDOM_PREFIX speex
#include "cubeb/cubeb.h"
#include "cubeb_utils.h"
#include "cubeb_resampler.h"
#include "cubeb_resampler_internal.h"
#include <assert.h>
#include <stdio.h>
#include <algorithm>
#include <iostream>
/* Windows cmath USE_MATH_DEFINE thing... */
const float PI = 3.14159265359f;
/* Some standard sample rates we're testing with. */
const int sample_rates[] = {
8000,
16000,
32000,
44100,
48000,
88200,
96000,
192000
};
/* The maximum number of channels we're resampling. */
const uint32_t max_channels = 2;
/* The minimum an maximum number of milliseconds we're resampling for. This is
* used to simulate the fact that the audio stream is resampled in chunks,
* because audio is delivered using callbacks. */
const uint32_t min_chunks = 10; /* ms */
const uint32_t max_chunks = 30; /* ms */
#define DUMP_ARRAYS
#ifdef DUMP_ARRAYS
/**
* Files produced by dump(...) can be converted to .wave files using:
*
* sox -c <channel_count> -r <rate> -e float -b 32 file.raw file.wav
*
* for floating-point audio, or:
*
* sox -c <channel_count> -r <rate> -e unsigned -b 16 file.raw file.wav
*
* for 16bit integer audio.
*/
/* Use the correct implementation of fopen, depending on the platform. */
void fopen_portable(FILE ** f, const char * name, const char * mode)
{
#ifdef WIN32
fopen_s(f, name, mode);
#else
*f = fopen(name, mode);
#endif
}
template<typename T>
void dump(const char * name, T * frames, size_t count)
{
FILE * file;
fopen_portable(&file, name, "wb");
if (!file) {
fprintf(stderr, "error opening %s\n", name);
return;
}
if (count != fwrite(frames, sizeof(T), count, file)) {
fprintf(stderr, "error writing to %s\n", name);
return;
}
fclose(file);
}
#else
template<typename T>
void dump(const char * name, T * frames, size_t count)
{ }
#endif
// The more the ratio is far from 1, the more we accept a big error.
float epsilon_tweak_ratio(float ratio)
{
return ratio >= 1 ? ratio : 1 / ratio;
}
// Epsilon values for comparing resampled data to expected data.
// The bigger the resampling ratio is, the more lax we are about errors.
template<typename T>
T epsilon(float ratio);
template<>
float epsilon(float ratio) {
return 0.08f * epsilon_tweak_ratio(ratio);
}
template<>
int16_t epsilon(float ratio) {
return static_cast<int16_t>(10 * epsilon_tweak_ratio(ratio));
}
/**
* This takes sine waves with a certain `channels` count, `source_rate`, and
* resample them, by chunk of `chunk_duration` milliseconds, to `target_rate`.
* Then a sample-wise comparison is performed against a sine wave generated at
* the correct rate.
*/
template<typename T>
void test_resampler_one_way(uint32_t channels, int32_t source_rate, int32_t target_rate, float chunk_duration)
{
size_t chunk_duration_in_source_frames = static_cast<uint32_t>(ceil(chunk_duration * source_rate / 1000.));
float resampling_ratio = static_cast<float>(source_rate) / target_rate;
cubeb_resampler_speex_one_way<T> resampler(channels, source_rate, target_rate, 3);
auto_array<T> source(channels * source_rate * 10);
auto_array<T> destination(channels * target_rate * 10);
auto_array<T> expected(channels * target_rate * 10);
uint32_t phase_index = 0;
uint32_t offset = 0;
const uint32_t buf_len = 2; /* seconds */
// generate a sine wave in each channel, at the source sample rate
source.push(channels * source_rate * buf_len);
while(offset != source.length()) {
float p = phase_index++ / static_cast<float>(source_rate);
for (uint32_t j = 0; j < channels; j++) {
source.data()[offset++] = 0.5 * sin(440. * 2 * PI * p);
}
}
dump("input.raw", source.data(), source.length());
expected.push(channels * target_rate * buf_len);
// generate a sine wave in each channel, at the target sample rate.
// Insert silent samples at the beginning to account for the resampler latency.
offset = resampler.latency() * channels;
for (uint32_t i = 0; i < offset; i++) {
expected.data()[i] = 0.0f;
}
phase_index = 0;
while (offset != expected.length()) {
float p = phase_index++ / static_cast<float>(target_rate);
for (uint32_t j = 0; j < channels; j++) {
expected.data()[offset++] = 0.5 * sin(440. * 2 * PI * p);
}
}
dump("expected.raw", expected.data(), expected.length());
// resample by chunk
uint32_t write_offset = 0;
destination.push(channels * target_rate * buf_len);
while (write_offset < destination.length())
{
size_t output_frames = static_cast<uint32_t>(floor(chunk_duration_in_source_frames / resampling_ratio));
uint32_t input_frames = resampler.input_needed_for_output(output_frames);
resampler.input(source.data(), input_frames);
source.pop(nullptr, input_frames * channels);
resampler.output(destination.data() + write_offset,
std::min(output_frames, (destination.length() - write_offset) / channels));
write_offset += output_frames * channels;
}
dump("output.raw", destination.data(), expected.length());
// compare, taking the latency into account
bool fuzzy_equal = true;
for (uint32_t i = resampler.latency() + 1; i < expected.length(); i++) {
float diff = abs(expected.data()[i] - destination.data()[i]);
if (diff > epsilon<T>(resampling_ratio)) {
fprintf(stderr, "divergence at %d: %f %f (delta %f)\n", i, expected.data()[i], destination.data()[i], diff);
fuzzy_equal = false;
}
}
assert(fuzzy_equal);
}
template<typename T>
cubeb_sample_format cubeb_format();
template<>
cubeb_sample_format cubeb_format<float>()
{
return CUBEB_SAMPLE_FLOAT32NE;
}
template<>
cubeb_sample_format cubeb_format<short>()
{
return CUBEB_SAMPLE_S16NE;
}
#define array_size(x) (sizeof(x) / sizeof(x[0]))
void test_resamplers_one_way()
{
/* Test one way resamplers */
for (uint32_t channels = 1; channels <= max_channels; channels++) {
for (uint32_t source_rate = 0; source_rate < array_size(sample_rates); source_rate++) {
for (uint32_t dest_rate = 0; dest_rate < array_size(sample_rates); dest_rate++) {
for (uint32_t chunk_duration = min_chunks; chunk_duration < max_chunks; chunk_duration++) {
printf("one_way: channels: %d, source_rate: %d, dest_rate: %d, chunk_duration: %d\n",
channels, sample_rates[source_rate], sample_rates[dest_rate], chunk_duration);
test_resampler_one_way<float>(channels, sample_rates[source_rate],
sample_rates[dest_rate], chunk_duration);
}
}
}
}
}
int main()
{
test_resamplers_one_way();
return 0;
}