Bug 1747760 - P8: Handle Opus data using Opus codec specific variant. r=kinetik.

This means as of this patch
- All mp4 specific audio codecs are handled. There's some more we could
  theoretically get, stuff like ALAC, but it's not clear to me we handle them
  following demuxing. I've left a catch all in the mp4 demuxing code just in
  case.
- We no longer pack the codec-delay/preskip at the head of the opus binary blob.
  This means that the binary blob is just the opus header data and the container
  specific preskip has its own member. My hope is this is clearer and easier to
  understand. It also means we can drop some of the code we had for packing the
  delay/preskip into a binary blob.

Differential Revision: https://phabricator.services.mozilla.com/D145521

dom/media/mp4/DecoderData.cpp

@@ -156,9 +156,9 @@ MediaResult MP4AudioInfo::Update(const Mp4parseTrackInfo* track,
   Mp4parseByteData extraData = audio->sample_info[0].extra_data;
   MOZ_ASSERT(mCodecSpecificConfig.is<NoCodecSpecificData>(),
              "Should have no codec specific data yet");
-  AudioCodecSpecificBinaryBlob codecSpecificBinaryBlob;
   if (codecType == MP4PARSE_CODEC_OPUS) {
     mMimeType = "audio/opus"_ns;
+    OpusCodecSpecificData opusCodecSpecificData{};
     // The Opus decoder expects the container's codec delay or
     // pre-skip value, in microseconds, as a 64-bit int at the
     // start of the codec-specific config blob.
@@ -166,14 +166,16 @@ MediaResult MP4AudioInfo::Update(const Mp4parseTrackInfo* track,
         mp4ParseSampleCodecSpecific.length >= 12) {
       uint16_t preskip = mozilla::LittleEndian::readUint16(
           mp4ParseSampleCodecSpecific.data + 10);
-      mozilla::OpusDataDecoder::AppendCodecDelay(
-          codecSpecificBinaryBlob.mBinaryBlob,
-          mozilla::FramesToUsecs(preskip, 48000).value());
+      opusCodecSpecificData.mContainerCodecDelayMicroSeconds =
+          mozilla::FramesToUsecs(preskip, 48000).value();
     } else {
       // This file will error later as it will be rejected by the opus decoder.
-      mozilla::OpusDataDecoder::AppendCodecDelay(
-          codecSpecificBinaryBlob.mBinaryBlob, 0);
+      opusCodecSpecificData.mContainerCodecDelayMicroSeconds = 0;
     }
+    opusCodecSpecificData.mHeadersBinaryBlob->AppendElements(
+        mp4ParseSampleCodecSpecific.data, mp4ParseSampleCodecSpecific.length);
+    mCodecSpecificConfig =
+        AudioCodecSpecificVariant{std::move(opusCodecSpecificData)};
   } else if (codecType == MP4PARSE_CODEC_AAC) {
     mMimeType = "audio/mp4a-latm"_ns;
     AacCodecSpecificData aacCodecSpecificData{};
@@ -216,9 +218,12 @@ MediaResult MP4AudioInfo::Update(const Mp4parseTrackInfo* track,
     mProfile = audio->sample_info[0].profile;
   }
 
-  // If length is 0 we append nothing
-  mExtraData->AppendElements(extraData.data, extraData.length);
   if (mCodecSpecificConfig.is<NoCodecSpecificData>()) {
+    // Handle codecs that are not explicitly handled above.
+    MOZ_ASSERT(
+        extraData.length == 0,
+        "Codecs that use extra data should be explicitly handled already");
+    AudioCodecSpecificBinaryBlob codecSpecificBinaryBlob;
     // No codec specific metadata set, use the generic form.
     codecSpecificBinaryBlob.mBinaryBlob->AppendElements(
         mp4ParseSampleCodecSpecific.data, mp4ParseSampleCodecSpecific.length);

dom/media/ogg/OggCodecState.cpp

@@ -1022,18 +1022,15 @@ bool OpusState::Init(void) {
   mInfo.mChannels = mParser->mChannels;
   mInfo.mBitDepth = 16;
   // Save preskip & the first header packet for the Opus decoder
-  AudioCodecSpecificBinaryBlob blob;
-  OpusDataDecoder::AppendCodecDelay(blob.mBinaryBlob,
-                                    Time(0, mParser->mPreSkip));
+  OpusCodecSpecificData opusData;
+  opusData.mContainerCodecDelayMicroSeconds = Time(0, mParser->mPreSkip);
+
   if (!mHeaders.PeekFront()) {
-    mInfo.mCodecSpecificConfig = AudioCodecSpecificVariant{
-        std::move(blob)};  // maybe not needed but do it just in case for this
-                           // temporary state.
     return false;
   }
-  blob.mBinaryBlob->AppendElements(mHeaders.PeekFront()->packet,
-                                   mHeaders.PeekFront()->bytes);
-  mInfo.mCodecSpecificConfig = AudioCodecSpecificVariant{std::move(blob)};
+  opusData.mHeadersBinaryBlob->AppendElements(mHeaders.PeekFront()->packet,
+                                              mHeaders.PeekFront()->bytes);
+  mInfo.mCodecSpecificConfig = AudioCodecSpecificVariant{std::move(opusData)};
 
   mHeaders.Erase();
   LOG(LogLevel::Debug, ("Opus decoder init"));

dom/media/platforms/agnostic/OpusDecoder.cpp

@@ -51,30 +51,23 @@ RefPtr<ShutdownPromise> OpusDataDecoder::Shutdown() {
   return ShutdownPromise::CreateAndResolve(true, __func__);
 }
 
-void OpusDataDecoder::AppendCodecDelay(MediaByteBuffer* config,
-                                       uint64_t codecDelayUS) {
-  uint8_t buffer[sizeof(uint64_t)];
-  BigEndian::writeUint64(buffer, codecDelayUS);
-  config->AppendElements(buffer, sizeof(uint64_t));
-}
-
 RefPtr<MediaDataDecoder::InitPromise> OpusDataDecoder::Init() {
   mThread = GetCurrentSerialEventTarget();
-  RefPtr<MediaByteBuffer> audioCodecSpecificBinaryBlob =
-      ForceGetAudioCodecSpecificBlob(mInfo.mCodecSpecificConfig);
-  size_t length = audioCodecSpecificBinaryBlob->Length();
-  uint8_t* p = audioCodecSpecificBinaryBlob->Elements();
-  if (length < sizeof(uint64_t)) {
-    OPUS_DEBUG("CodecSpecificConfig too short to read codecDelay!");
+  if (!mInfo.mCodecSpecificConfig.is<OpusCodecSpecificData>()) {
+    MOZ_ASSERT_UNREACHABLE();
+    OPUS_DEBUG("Opus decoder got non-opus codec specific data");
     return InitPromise::CreateAndReject(
         MediaResult(
             NS_ERROR_DOM_MEDIA_FATAL_ERR,
-            RESULT_DETAIL("CodecSpecificConfig too short to read codecDelay!")),
+            RESULT_DETAIL("Opus decoder got non-opus codec specific data!")),
         __func__);
   }
-  int64_t codecDelay = BigEndian::readUint64(p);
-  length -= sizeof(uint64_t);
-  p += sizeof(uint64_t);
+  const OpusCodecSpecificData opusCodecSpecificData =
+      mInfo.mCodecSpecificConfig.as<OpusCodecSpecificData>();
+  RefPtr<MediaByteBuffer> opusHeaderBlob =
+      opusCodecSpecificData.mHeadersBinaryBlob;
+  size_t length = opusHeaderBlob->Length();
+  uint8_t* p = opusHeaderBlob->Elements();
   if (NS_FAILED(DecodeHeader(p, length))) {
     OPUS_DEBUG("Error decoding header!");
     return InitPromise::CreateAndReject(
@@ -110,7 +103,7 @@ RefPtr<MediaDataDecoder::InitPromise> OpusDataDecoder::Init() {
   mSkip = mOpusParser->mPreSkip;
   mPaddingDiscarded = false;
 
-  if (codecDelay !=
+  if (opusCodecSpecificData.mContainerCodecDelayMicroSeconds !=
       FramesToUsecs(mOpusParser->mPreSkip, mOpusParser->mRate).value()) {
     NS_WARNING(
         "Invalid Opus header: container CodecDelay and Opus pre-skip do not "

dom/media/platforms/agnostic/OpusDecoder.h

@@ -37,13 +37,6 @@ class OpusDataDecoder : public MediaDataDecoder,
   // Return true if mimetype is Opus
   static bool IsOpus(const nsACString& aMimeType);
 
-  // Pack pre-skip/CodecDelay, given in microseconds, into a
-  // MediaByteBuffer. The decoder expects this value to come
-  // from the container (if any) and to precede the OpusHead
-  // block in the CodecSpecificConfig buffer to verify the
-  // values match.
-  static void AppendCodecDelay(MediaByteBuffer* config, uint64_t codecDelayUS);
-
  private:
   nsresult DecodeHeader(const unsigned char* aData, size_t aLength);
 

dom/media/webm/WebMDemuxer.cpp

@@ -397,10 +397,10 @@ nsresult WebMDemuxer::ReadMetadata() {
       } else if (mAudioCodec == NESTEGG_CODEC_OPUS) {
         uint64_t codecDelayUs = params.codec_delay / 1000;
         mInfo.mAudio.mMimeType = "audio/opus";
-        AudioCodecSpecificBinaryBlob blob;
-        OpusDataDecoder::AppendCodecDelay(blob.mBinaryBlob, codecDelayUs);
+        OpusCodecSpecificData opusCodecSpecificData;
+        opusCodecSpecificData.mContainerCodecDelayMicroSeconds = codecDelayUs;
         mInfo.mAudio.mCodecSpecificConfig =
-            AudioCodecSpecificVariant{std::move(blob)};
+            AudioCodecSpecificVariant{std::move(opusCodecSpecificData)};
       }
       mSeekPreroll = params.seek_preroll;
       mInfo.mAudio.mRate = params.rate;