From ec3a23c754c52071257176b27889a0bdafd98200 Mon Sep 17 00:00:00 2001 From: Andreas Pehrson Date: Wed, 7 Oct 2020 22:19:37 +0000 Subject: [PATCH] Bug 1667728 - Use only one notion of sample rate for AudioTrackEncoder. r=bryce Differential Revision: https://phabricator.services.mozilla.com/D91951 --- dom/media/encoder/OpusTrackEncoder.cpp | 31 ++++++++---------- dom/media/encoder/OpusTrackEncoder.h | 2 +- dom/media/encoder/TrackEncoder.cpp | 6 ++-- dom/media/encoder/TrackEncoder.h | 10 ++---- dom/media/gtest/TestAudioTrackEncoder.cpp | 31 +++++++++--------- dom/media/gtest/TestWebMWriter.cpp | 40 ++++++++++------------- 6 files changed, 54 insertions(+), 66 deletions(-) diff --git a/dom/media/encoder/OpusTrackEncoder.cpp b/dom/media/encoder/OpusTrackEncoder.cpp index 0e9c4b947162..2781fa3ad116 100644 --- a/dom/media/encoder/OpusTrackEncoder.cpp +++ b/dom/media/encoder/OpusTrackEncoder.cpp @@ -129,7 +129,7 @@ OpusTrackEncoder::~OpusTrackEncoder() { } } -nsresult OpusTrackEncoder::Init(int aChannels, int aSamplingRate) { +nsresult OpusTrackEncoder::Init(int aChannels) { NS_ENSURE_TRUE((aChannels <= MAX_SUPPORTED_AUDIO_CHANNELS) && (aChannels > 0), NS_ERROR_FAILURE); @@ -139,8 +139,8 @@ nsresult OpusTrackEncoder::Init(int aChannels, int aSamplingRate) { mChannels = aChannels > MAX_CHANNELS ? MAX_CHANNELS : aChannels; // Reject non-audio sample rates. - NS_ENSURE_TRUE(aSamplingRate >= 8000, NS_ERROR_INVALID_ARG); - NS_ENSURE_TRUE(aSamplingRate <= 192000, NS_ERROR_INVALID_ARG); + NS_ENSURE_TRUE(mTrackRate >= 8000, NS_ERROR_INVALID_ARG); + NS_ENSURE_TRUE(mTrackRate <= 192000, NS_ERROR_INVALID_ARG); // According to www.opus-codec.org, creating an opus encoder requires the // sampling rate of source signal be one of 8000, 12000, 16000, 24000, or @@ -149,18 +149,15 @@ nsresult OpusTrackEncoder::Init(int aChannels, int aSamplingRate) { supportedSamplingRates.AppendElements( kOpusSupportedInputSamplingRates, ArrayLength(kOpusSupportedInputSamplingRates)); - if (!supportedSamplingRates.Contains(aSamplingRate)) { + if (!supportedSamplingRates.Contains(mTrackRate)) { int error; - mResampler = - speex_resampler_init(mChannels, aSamplingRate, kOpusSamplingRate, - SPEEX_RESAMPLER_QUALITY_DEFAULT, &error); + mResampler = speex_resampler_init(mChannels, mTrackRate, kOpusSamplingRate, + SPEEX_RESAMPLER_QUALITY_DEFAULT, &error); if (error != RESAMPLER_ERR_SUCCESS) { return NS_ERROR_FAILURE; } } - mSamplingRate = aSamplingRate; - NS_ENSURE_TRUE(mSamplingRate > 0, NS_ERROR_FAILURE); int error = 0; mEncoder = opus_encoder_create(GetOutputSampleRate(), mChannels, @@ -196,7 +193,7 @@ nsresult OpusTrackEncoder::Init(int aChannels, int aSamplingRate) { } int OpusTrackEncoder::GetOutputSampleRate() { - return mResampler ? kOpusSamplingRate : mSamplingRate; + return mResampler ? kOpusSamplingRate : mTrackRate; } int OpusTrackEncoder::GetPacketDuration() { @@ -218,12 +215,12 @@ already_AddRefed OpusTrackEncoder::GetMetadata() { RefPtr meta = new OpusMetadata(); meta->mChannels = mChannels; - meta->mSamplingFrequency = mSamplingRate; + meta->mSamplingFrequency = mTrackRate; - // The ogg time stamping and pre-skip is always timed at 48000. + // Ogg and Webm timestamps are always sampled at 48k for Opus. SerializeOpusIdHeader( mChannels, mLookahead * (kOpusSamplingRate / GetOutputSampleRate()), - mSamplingRate, &meta->mIdHeader); + mTrackRate, &meta->mIdHeader); nsCString vendor; vendor.AppendASCII(opus_get_version_string()); @@ -273,7 +270,7 @@ nsresult OpusTrackEncoder::GetEncodedTrack( const int framesToFetch = !mResampler ? GetPacketDuration() : (GetPacketDuration() - framesLeft) * - mSamplingRate / kOpusSamplingRate + + mTrackRate / kOpusSamplingRate + frameRoundUp; if (!mEndOfStream && mSourceSegment.GetDuration() < framesToFetch) { @@ -341,7 +338,7 @@ nsresult OpusTrackEncoder::GetEncodedTrack( // We want to consume all the input data, so we slightly oversize the // resampled data buffer so we can fit the output data in. We cannot // really predict the output frame count at each call. - uint32_t outframes = frameCopied * kOpusSamplingRate / mSamplingRate + 1; + uint32_t outframes = frameCopied * kOpusSamplingRate / mTrackRate + 1; uint32_t inframes = frameCopied; resamplingDest.SetLength(outframes * mChannels); @@ -378,8 +375,8 @@ nsresult OpusTrackEncoder::GetEncodedTrack( framesInPCM = framesLeft + outframesToCopy; duration = framesInPCM; } else { - // The ogg time stamping and pre-skip is always timed at 48000. - duration = frameCopied * (kOpusSamplingRate / mSamplingRate); + // Ogg and Webm timestamps are always sampled at 48k for Opus. + duration = frameCopied * (kOpusSamplingRate / mTrackRate); } // Remove the raw data which has been pulled to pcm buffer. diff --git a/dom/media/encoder/OpusTrackEncoder.h b/dom/media/encoder/OpusTrackEncoder.h index 630aafb7d6f1..142937458edb 100644 --- a/dom/media/encoder/OpusTrackEncoder.h +++ b/dom/media/encoder/OpusTrackEncoder.h @@ -42,7 +42,7 @@ class OpusTrackEncoder : public AudioTrackEncoder { protected: int GetPacketDuration() override; - nsresult Init(int aChannels, int aSamplingRate) override; + nsresult Init(int aChannels) override; /** * Get the samplerate of the data to be fed to the Opus encoder. This might be diff --git a/dom/media/encoder/TrackEncoder.cpp b/dom/media/encoder/TrackEncoder.cpp index 9fc1db7fbb52..cb3eefa8fe7f 100644 --- a/dom/media/encoder/TrackEncoder.cpp +++ b/dom/media/encoder/TrackEncoder.cpp @@ -178,7 +178,7 @@ void AudioTrackEncoder::TryInit(const AudioSegment& aSegment, continue; } - nsresult rv = Init(iter->mChannelData.Length(), mTrackRate); + nsresult rv = Init(iter->mChannelData.Length()); if (NS_SUCCEEDED(rv)) { TRACK_LOG(LogLevel::Info, @@ -204,7 +204,7 @@ void AudioTrackEncoder::TryInit(const AudioSegment& aSegment, ("[AudioTrackEncoder]: Initialize failed for %ds. Attempting to " "init with %d (default) channels!", AUDIO_INIT_FAILED_DURATION, DEFAULT_CHANNELS)); - nsresult rv = Init(DEFAULT_CHANNELS, mTrackRate); + nsresult rv = Init(DEFAULT_CHANNELS); Telemetry::Accumulate( Telemetry::MEDIA_RECORDER_TRACK_ENCODER_INIT_TIMEOUT_TYPE, 0); if (NS_FAILED(rv)) { @@ -231,7 +231,7 @@ void AudioTrackEncoder::NotifyEndOfStream() { if (!mCanceled && !mInitialized) { // If source audio track is completely silent till the end of encoding, // initialize the encoder with a default channel count. - Init(DEFAULT_CHANNELS, mTrackRate); + Init(DEFAULT_CHANNELS); } mEndOfStream = true; diff --git a/dom/media/encoder/TrackEncoder.h b/dom/media/encoder/TrackEncoder.h index f9717c145422..72a202ca4136 100644 --- a/dom/media/encoder/TrackEncoder.h +++ b/dom/media/encoder/TrackEncoder.h @@ -175,7 +175,7 @@ class TrackEncoder { /** * The track rate of source media. */ - TrackRate mTrackRate; + const TrackRate mTrackRate; /** * If set we assert that all methods are called on this thread. @@ -190,7 +190,6 @@ class AudioTrackEncoder : public TrackEncoder { explicit AudioTrackEncoder(TrackRate aTrackRate) : TrackEncoder(aTrackRate), mChannels(0), - mSamplingRate(0), mNotInitDuration(0), mAudioBitrate(0) {} @@ -297,7 +296,7 @@ class AudioTrackEncoder : public TrackEncoder { * Initializes the audio encoder. The call of this method is delayed until we * have received the first valid track from MediaTrackGraph. */ - virtual nsresult Init(int aChannels, int aSamplingRate) = 0; + virtual nsresult Init(int aChannels) = 0; /** * The number of channels are used for processing PCM data in the audio @@ -307,11 +306,6 @@ class AudioTrackEncoder : public TrackEncoder { */ int mChannels; - /** - * The sampling rate of source audio data. - */ - int mSamplingRate; - /** * A segment queue of outgoing audio track data to the encoder. * The contents of mOutgoingBuffer will always be what has been appended on diff --git a/dom/media/gtest/TestAudioTrackEncoder.cpp b/dom/media/gtest/TestAudioTrackEncoder.cpp index e481d74b4883..aaed613284ec 100644 --- a/dom/media/gtest/TestAudioTrackEncoder.cpp +++ b/dom/media/gtest/TestAudioTrackEncoder.cpp @@ -13,11 +13,12 @@ using namespace mozilla; class TestOpusTrackEncoder : public OpusTrackEncoder { public: - TestOpusTrackEncoder() : OpusTrackEncoder(90000) {} + explicit TestOpusTrackEncoder(TrackRate aTrackRate) + : OpusTrackEncoder(aTrackRate) {} // Return true if it has successfully initialized the Opus encoder. - bool TestOpusRawCreation(int aChannels, int aSamplingRate) { - if (Init(aChannels, aSamplingRate) == NS_OK) { + bool TestOpusRawCreation(int aChannels) { + if (Init(aChannels) == NS_OK) { if (IsInitialized()) { return true; } @@ -33,9 +34,9 @@ class TestOpusTrackEncoder : public OpusTrackEncoder { } }; -static bool TestOpusInit(int aChannels, int aSamplingRate) { - TestOpusTrackEncoder encoder; - return encoder.TestOpusRawCreation(aChannels, aSamplingRate); +static bool TestOpusInit(int aChannels, TrackRate aSamplingRate) { + TestOpusTrackEncoder encoder(aSamplingRate); + return encoder.TestOpusRawCreation(aChannels); } TEST(OpusAudioTrackEncoder, InitRaw) @@ -153,9 +154,9 @@ TEST(OpusAudioTrackEncoder, Init) } } -static int TestOpusResampler(int aChannels, int aSamplingRate) { - TestOpusTrackEncoder encoder; - EXPECT_TRUE(encoder.TestOpusRawCreation(aChannels, aSamplingRate)); +static int TestOpusResampler(int aChannels, TrackRate aTrackRate) { + TestOpusTrackEncoder encoder(aTrackRate); + EXPECT_TRUE(encoder.TestOpusRawCreation(aChannels)); return encoder.TestGetOutputSampleRate(); } @@ -178,9 +179,9 @@ TEST(OpusAudioTrackEncoder, Resample) TEST(OpusAudioTrackEncoder, FetchMetadata) { const int32_t channels = 1; - const int32_t sampleRate = 44100; - TestOpusTrackEncoder encoder; - EXPECT_TRUE(encoder.TestOpusRawCreation(channels, sampleRate)); + const TrackRate sampleRate = 44100; + TestOpusTrackEncoder encoder(sampleRate); + EXPECT_TRUE(encoder.TestOpusRawCreation(channels)); RefPtr metadata = encoder.GetMetadata(); ASSERT_EQ(TrackMetadataBase::METADATA_OPUS, metadata->GetKind()); @@ -193,9 +194,9 @@ TEST(OpusAudioTrackEncoder, FetchMetadata) TEST(OpusAudioTrackEncoder, FrameEncode) { const int32_t channels = 1; - const int32_t sampleRate = 44100; - TestOpusTrackEncoder encoder; - EXPECT_TRUE(encoder.TestOpusRawCreation(channels, sampleRate)); + const TrackRate sampleRate = 44100; + TestOpusTrackEncoder encoder(sampleRate); + EXPECT_TRUE(encoder.TestOpusRawCreation(channels)); // Generate five seconds of raw audio data. AudioGenerator generator(channels, sampleRate); diff --git a/dom/media/gtest/TestWebMWriter.cpp b/dom/media/gtest/TestWebMWriter.cpp index f13cc518e16b..176c9f0808e2 100644 --- a/dom/media/gtest/TestWebMWriter.cpp +++ b/dom/media/gtest/TestWebMWriter.cpp @@ -18,8 +18,8 @@ class WebMOpusTrackEncoder : public OpusTrackEncoder { public: explicit WebMOpusTrackEncoder(TrackRate aTrackRate) : OpusTrackEncoder(aTrackRate) {} - bool TestOpusCreation(int aChannels, int aSamplingRate) { - if (NS_SUCCEEDED(Init(aChannels, aSamplingRate))) { + bool TestOpusCreation(int aChannels) { + if (NS_SUCCEEDED(Init(aChannels))) { return true; } return false; @@ -40,11 +40,10 @@ class WebMVP8TrackEncoder : public VP8TrackEncoder { } }; -static void GetOpusMetadata(int aChannels, int aSampleRate, - TrackRate aTrackRate, +static void GetOpusMetadata(int aChannels, TrackRate aTrackRate, nsTArray>& aMeta) { WebMOpusTrackEncoder opusEncoder(aTrackRate); - EXPECT_TRUE(opusEncoder.TestOpusCreation(aChannels, aSampleRate)); + EXPECT_TRUE(opusEncoder.TestOpusCreation(aChannels)); aMeta.AppendElement(opusEncoder.GetMetadata()); } @@ -106,18 +105,18 @@ TEST(WebMWriter, Metadata) nsTArray> meta; + TrackRate trackRate = 44100; + // Get opus metadata. int channel = 1; - int sampleRate = 44100; - TrackRate aTrackRate = 90000; - GetOpusMetadata(channel, sampleRate, aTrackRate, meta); + GetOpusMetadata(channel, trackRate, meta); // Get vp8 metadata int32_t width = 640; int32_t height = 480; int32_t displayWidth = 640; int32_t displayHeight = 480; - GetVP8Metadata(width, height, displayWidth, displayHeight, aTrackRate, meta); + GetVP8Metadata(width, height, displayWidth, displayHeight, trackRate, meta); // Set metadata writer.SetMetadata(meta); @@ -130,17 +129,16 @@ TEST(WebMWriter, Cluster) { TestWebMWriter writer; nsTArray> meta; + TrackRate trackRate = 48000; // Get opus metadata. int channel = 1; - int sampleRate = 48000; - TrackRate aTrackRate = 90000; - GetOpusMetadata(channel, sampleRate, aTrackRate, meta); + GetOpusMetadata(channel, trackRate, meta); // Get vp8 metadata int32_t width = 320; int32_t height = 240; int32_t displayWidth = 320; int32_t displayHeight = 240; - GetVP8Metadata(width, height, displayWidth, displayHeight, aTrackRate, meta); + GetVP8Metadata(width, height, displayWidth, displayHeight, trackRate, meta); writer.SetMetadata(meta); nsTArray> encodedBuf; @@ -173,17 +171,16 @@ TEST(WebMWriter, FLUSH_NEEDED) { TestWebMWriter writer; nsTArray> meta; + TrackRate trackRate = 44100; // Get opus metadata. int channel = 2; - int sampleRate = 44100; - TrackRate aTrackRate = 100000; - GetOpusMetadata(channel, sampleRate, aTrackRate, meta); + GetOpusMetadata(channel, trackRate, meta); // Get vp8 metadata int32_t width = 176; int32_t height = 352; int32_t displayWidth = 176; int32_t displayHeight = 352; - GetVP8Metadata(width, height, displayWidth, displayHeight, aTrackRate, meta); + GetVP8Metadata(width, height, displayWidth, displayHeight, trackRate, meta); writer.SetMetadata(meta); // write the first I-Frame. @@ -294,17 +291,16 @@ TEST(WebMWriter, bug970774_aspect_ratio) { TestWebMWriter writer; nsTArray> meta; + TrackRate trackRate = 44100; // Get opus metadata. int channel = 1; - int sampleRate = 44100; - TrackRate aTrackRate = 90000; - GetOpusMetadata(channel, sampleRate, aTrackRate, meta); + GetOpusMetadata(channel, trackRate, meta); // Set vp8 metadata int32_t width = 640; int32_t height = 480; int32_t displayWidth = 1280; int32_t displayHeight = 960; - GetVP8Metadata(width, height, displayWidth, displayHeight, aTrackRate, meta); + GetVP8Metadata(width, height, displayWidth, displayHeight, trackRate, meta); writer.SetMetadata(meta); // write the first I-Frame. @@ -353,7 +349,7 @@ TEST(WebMWriter, bug970774_aspect_ratio) rv = nestegg_track_audio_params(context, track, ¶ms); EXPECT_EQ(rv, 0); EXPECT_EQ(channel, static_cast(params.channels)); - EXPECT_EQ(static_cast(sampleRate), params.rate); + EXPECT_EQ(static_cast(trackRate), params.rate); } } if (context) {