Bug 1406503 - P1. Abstract FFmpeg decoding so that an av_parser can also be used for audio. r=jwwang

MozReview-Commit-ID: 4bNxLhYKqVG --HG-- extra : rebase_source : fb444f2b51fc6f23d0a4c76db337fd7324c6cc1c
2017-10-26 22:08:33 +02:00 · 2017-10-26 22:08:33 +02:00 · e147a460d9
--- a/dom/media/platforms/ffmpeg/FFmpegAudioDecoder.cpp
+++ b/dom/media/platforms/ffmpeg/FFmpegAudioDecoder.cpp
@ -119,27 +119,32 @@ CopyAndPackAudio(AVFrame* aFrame, uint32_t aNumChannels, uint32_t aNumAFrames)
  return audio;
 }

-RefPtr<MediaDataDecoder::DecodePromise>
-FFmpegAudioDecoder<LIBAV_VER>::ProcessDecode(MediaRawData* aSample)
+MediaResult
+FFmpegAudioDecoder<LIBAV_VER>::DoDecode(MediaRawData* aSample,
+                                        uint8_t* aData,
+                                        int aSize,
+                                        bool* aGotFrame,
+                                        DecodedData& aResults)
 {
  AVPacket packet;
  mLib->av_init_packet(&packet);

-  packet.data = const_cast<uint8_t*>(aSample->Data());
-  packet.size = aSample->Size();
+  packet.data = const_cast<uint8_t*>(aData);
+  packet.size = aSize;
+
+  if (aGotFrame) {
+    *aGotFrame = false;
+  }

  if (!PrepareFrame()) {
-    return DecodePromise::CreateAndReject(
-      MediaResult(
-        NS_ERROR_OUT_OF_MEMORY,
-        RESULT_DETAIL("FFmpeg audio decoder failed to allocate frame")),
-      __func__);
+    return MediaResult(
+      NS_ERROR_OUT_OF_MEMORY,
+      RESULT_DETAIL("FFmpeg audio decoder failed to allocate frame"));
  }

  int64_t samplePosition = aSample->mOffset;
  media::TimeUnit pts = aSample->mTime;

-  DecodedData results;
  while (packet.size > 0) {
    int decoded;
    int bytesConsumed =
@ -147,10 +152,8 @@ FFmpegAudioDecoder<LIBAV_VER>::ProcessDecode(MediaRawData* aSample)

    if (bytesConsumed < 0) {
      NS_WARNING("FFmpeg audio decoder error.");
-      return DecodePromise::CreateAndReject(
-        MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
-                    RESULT_DETAIL("FFmpeg audio error:%d", bytesConsumed)),
-        __func__);
+      return MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
+                         RESULT_DETAIL("FFmpeg audio error:%d", bytesConsumed));
    }

    if (decoded) {
@ -160,20 +163,17 @@ FFmpegAudioDecoder<LIBAV_VER>::ProcessDecode(MediaRawData* aSample)
          mFrame->format != AV_SAMPLE_FMT_S16P &&
          mFrame->format != AV_SAMPLE_FMT_S32 &&
          mFrame->format != AV_SAMPLE_FMT_S32P) {
-        return DecodePromise::CreateAndReject(
-          MediaResult(
-            NS_ERROR_DOM_MEDIA_DECODE_ERR,
-            RESULT_DETAIL(
-              "FFmpeg audio decoder outputs unsupported audio format")),
-          __func__);
+        return MediaResult(
+          NS_ERROR_DOM_MEDIA_DECODE_ERR,
+          RESULT_DETAIL(
+            "FFmpeg audio decoder outputs unsupported audio format"));
      }
      uint32_t numChannels = mCodecContext->channels;
      AudioConfig::ChannelLayout layout(numChannels);
      if (!layout.IsValid()) {
-        return DecodePromise::CreateAndReject(
-          MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
-                      RESULT_DETAIL("Unsupported channel layout:%u", numChannels)),
-          __func__);
+        return MediaResult(
+          NS_ERROR_DOM_MEDIA_FATAL_ERR,
+          RESULT_DETAIL("Unsupported channel layout:%u", numChannels));
      }

      uint32_t samplingRate = mCodecContext->sample_rate;
@ -181,39 +181,38 @@ FFmpegAudioDecoder<LIBAV_VER>::ProcessDecode(MediaRawData* aSample)
      AlignedAudioBuffer audio =
        CopyAndPackAudio(mFrame, numChannels, mFrame->nb_samples);
      if (!audio) {
-        return DecodePromise::CreateAndReject(
-          MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__), __func__);
+        return MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__);
      }

      media::TimeUnit duration =
        FramesToTimeUnit(mFrame->nb_samples, samplingRate);
      if (!duration.IsValid()) {
-        return DecodePromise::CreateAndReject(
-          MediaResult(NS_ERROR_DOM_MEDIA_OVERFLOW_ERR,
-                      RESULT_DETAIL("Invalid sample duration")),
-          __func__);
+        return MediaResult(NS_ERROR_DOM_MEDIA_OVERFLOW_ERR,
+                           RESULT_DETAIL("Invalid sample duration"));
      }

      media::TimeUnit newpts = pts + duration;
      if (!newpts.IsValid()) {
-        return DecodePromise::CreateAndReject(
-          MediaResult(
-            NS_ERROR_DOM_MEDIA_OVERFLOW_ERR,
-            RESULT_DETAIL("Invalid count of accumulated audio samples")),
-          __func__);
+        return MediaResult(
+          NS_ERROR_DOM_MEDIA_OVERFLOW_ERR,
+          RESULT_DETAIL("Invalid count of accumulated audio samples"));
      }

-      results.AppendElement(new AudioData(
+      aResults.AppendElement(new AudioData(
        samplePosition, pts, duration,
        mFrame->nb_samples, Move(audio), numChannels, samplingRate));

      pts = newpts;
+
+      if (aGotFrame) {
+        *aGotFrame = true;
+      }
    }
    packet.data += bytesConsumed;
    packet.size -= bytesConsumed;
    samplePosition += bytesConsumed;
  }
-  return DecodePromise::CreateAndResolve(Move(results), __func__);
+  return NS_OK;
 }

 RefPtr<MediaDataDecoder::DecodePromise>
--- a/dom/media/platforms/ffmpeg/FFmpegAudioDecoder.h
+++ b/dom/media/platforms/ffmpeg/FFmpegAudioDecoder.h
@ -33,8 +33,12 @@ public:
  }

 private:
-  RefPtr<DecodePromise> ProcessDecode(MediaRawData* aSample) override;
  RefPtr<DecodePromise> ProcessDrain() override;
+  MediaResult DoDecode(MediaRawData* aSample,
+                       uint8_t* aData,
+                       int aSize,
+                       bool* aGotFrame,
+                       DecodedData& aResults) override;
 };

 } // namespace mozilla
--- a/dom/media/platforms/ffmpeg/FFmpegDataDecoder.cpp
+++ b/dom/media/platforms/ffmpeg/FFmpegDataDecoder.cpp
@ -24,6 +24,7 @@ FFmpegDataDecoder<LIBAV_VER>::FFmpegDataDecoder(FFmpegLibWrapper* aLib,
                                                AVCodecID aCodecID)
  : mLib(aLib)
  , mCodecContext(nullptr)
+  , mCodecParser(nullptr)
  , mFrame(NULL)
  , mExtraData(nullptr)
  , mCodecID(aCodecID)
@ -36,6 +37,10 @@ FFmpegDataDecoder<LIBAV_VER>::FFmpegDataDecoder(FFmpegLibWrapper* aLib,
 FFmpegDataDecoder<LIBAV_VER>::~FFmpegDataDecoder()
 {
  MOZ_COUNT_DTOR(FFmpegDataDecoder);
+  if (mCodecParser) {
+    mLib->av_parser_close(mCodecParser);
+    mCodecParser = nullptr;
+  }
 }

 MediaResult
@ -56,6 +61,13 @@ FFmpegDataDecoder<LIBAV_VER>::InitDecoder()
                       RESULT_DETAIL("Couldn't init ffmpeg context"));
  }

+  if (NeedParser()) {
+    MOZ_ASSERT(mCodecParser == nullptr);
+    mCodecParser = mLib->av_parser_init(mCodecID);
+    if (mCodecParser) {
+      mCodecParser->flags |= ParserFlags();
+    }
+  }
  mCodecContext->opaque = this;

  InitCodecContext();
@ -106,6 +118,54 @@ FFmpegDataDecoder<LIBAV_VER>::Decode(MediaRawData* aSample)
                                    &FFmpegDataDecoder::ProcessDecode, aSample);
 }

+RefPtr<MediaDataDecoder::DecodePromise>
+FFmpegDataDecoder<LIBAV_VER>::ProcessDecode(MediaRawData* aSample)
+{
+  bool gotFrame = false;
+  DecodedData results;
+  MediaResult rv = DoDecode(aSample, &gotFrame, results);
+  if (NS_FAILED(rv)) {
+    return DecodePromise::CreateAndReject(rv, __func__);
+  }
+  return DecodePromise::CreateAndResolve(Move(results), __func__);
+}
+
+MediaResult
+FFmpegDataDecoder<LIBAV_VER>::DoDecode(MediaRawData* aSample, bool* aGotFrame,
+                                       MediaDataDecoder::DecodedData& aResults)
+{
+  uint8_t* inputData = const_cast<uint8_t*>(aSample->Data());
+  size_t inputSize = aSample->Size();
+
+  if (inputSize && mCodecParser) {
+    while (inputSize) {
+      uint8_t* data = inputData;
+      int size = inputSize;
+      int len = mLib->av_parser_parse2(
+        mCodecParser, mCodecContext, &data, &size, inputData, inputSize,
+        aSample->mTime.ToMicroseconds(), aSample->mTimecode.ToMicroseconds(),
+        aSample->mOffset);
+      if (size_t(len) > inputSize) {
+        return NS_ERROR_DOM_MEDIA_DECODE_ERR;
+      }
+      inputData += len;
+      inputSize -= len;
+      if (size) {
+        bool gotFrame = false;
+        MediaResult rv = DoDecode(aSample, data, size, &gotFrame, aResults);
+        if (NS_FAILED(rv)) {
+          return rv;
+        }
+        if (gotFrame && aGotFrame) {
+          *aGotFrame = true;
+        }
+      }
+    }
+    return NS_OK;
+  }
+  return DoDecode(aSample, inputData, inputSize, aGotFrame, aResults);
+}
+
 RefPtr<MediaDataDecoder::FlushPromise>
 FFmpegDataDecoder<LIBAV_VER>::Flush()
 {
--- a/dom/media/platforms/ffmpeg/FFmpegDataDecoder.h
+++ b/dom/media/platforms/ffmpeg/FFmpegDataDecoder.h
@ -42,19 +42,30 @@ protected:
  virtual RefPtr<FlushPromise> ProcessFlush();
  virtual void ProcessShutdown();
  virtual void InitCodecContext() { }
-  AVFrame*        PrepareFrame();
-  MediaResult     InitDecoder();
+  AVFrame* PrepareFrame();
+  MediaResult InitDecoder();
+  MediaResult DoDecode(MediaRawData* aSample,
+                       bool* aGotFrame,
+                       DecodedData& aOutResults);

  FFmpegLibWrapper* mLib;

  AVCodecContext* mCodecContext;
-  AVFrame*        mFrame;
+  AVCodecParserContext* mCodecParser;
+  AVFrame* mFrame;
  RefPtr<MediaByteBuffer> mExtraData;
  AVCodecID mCodecID;

 private:
-  virtual RefPtr<DecodePromise> ProcessDecode(MediaRawData* aSample) = 0;
+  RefPtr<DecodePromise> ProcessDecode(MediaRawData* aSample);
  virtual RefPtr<DecodePromise> ProcessDrain() = 0;
+  virtual MediaResult DoDecode(MediaRawData* aSample,
+                               uint8_t* aData,
+                               int aSize,
+                               bool* aGotFrame,
+                               MediaDataDecoder::DecodedData& aOutResults) = 0;
+  virtual bool NeedParser() const { return false; }
+  virtual int ParserFlags() const { return PARSER_FLAG_COMPLETE_FRAMES; }

  static StaticMutex sMonitor;
  const RefPtr<TaskQueue> mTaskQueue;
--- a/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp
+++ b/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp
@ -128,11 +128,9 @@ FFmpegVideoDecoder<LIBAV_VER>::FFmpegVideoDecoder(
  , mImageAllocator(aAllocator)
  , mImageContainer(aImageContainer)
  , mInfo(aConfig)
-  , mCodecParser(nullptr)
  , mLastInputDts(INT64_MIN)
  , mLowLatency(aLowLatency)
 {
-  MOZ_COUNT_CTOR(FFmpegVideoDecoder);
  // Use a new MediaByteBuffer as the object will be modified during
  // initialization.
  mExtraData = new MediaByteBuffer;
@ -184,65 +182,6 @@ FFmpegVideoDecoder<LIBAV_VER>::InitCodecContext()

  // FFmpeg will call back to this to negotiate a video pixel format.
  mCodecContext->get_format = ChoosePixelFormat;
-
-  mCodecParser = mLib->av_parser_init(mCodecID);
-  if (mCodecParser) {
-    mCodecParser->flags |= PARSER_FLAG_COMPLETE_FRAMES;
-  }
-}
-
-RefPtr<MediaDataDecoder::DecodePromise>
-FFmpegVideoDecoder<LIBAV_VER>::ProcessDecode(MediaRawData* aSample)
-{
-  bool gotFrame = false;
-  DecodedData results;
-  MediaResult rv = DoDecode(aSample, &gotFrame, results);
-  if (NS_FAILED(rv)) {
-    return DecodePromise::CreateAndReject(rv, __func__);
-  }
-  return DecodePromise::CreateAndResolve(Move(results), __func__);
-}
-
-MediaResult
-FFmpegVideoDecoder<LIBAV_VER>::DoDecode(MediaRawData* aSample, bool* aGotFrame,
-                                        MediaDataDecoder::DecodedData& aResults)
-{
-  uint8_t* inputData = const_cast<uint8_t*>(aSample->Data());
-  size_t inputSize = aSample->Size();
-
-#if LIBAVCODEC_VERSION_MAJOR >= 54
-  if (inputSize && mCodecParser && (mCodecID == AV_CODEC_ID_VP8
-#if LIBAVCODEC_VERSION_MAJOR >= 55
-      || mCodecID == AV_CODEC_ID_VP9
-#endif
-      )) {
-    while (inputSize) {
-      uint8_t* data = inputData;
-      int size = inputSize;
-      int len = mLib->av_parser_parse2(
-        mCodecParser, mCodecContext, &data, &size, inputData, inputSize,
-        aSample->mTime.ToMicroseconds(), aSample->mTimecode.ToMicroseconds(),
-        aSample->mOffset);
-      if (size_t(len) > inputSize) {
-        return NS_ERROR_DOM_MEDIA_DECODE_ERR;
-      }
-      inputData += len;
-      inputSize -= len;
-      if (size) {
-        bool gotFrame = false;
-        MediaResult rv = DoDecode(aSample, data, size, &gotFrame, aResults);
-        if (NS_FAILED(rv)) {
-          return rv;
-        }
-        if (gotFrame && aGotFrame) {
-          *aGotFrame = true;
-        }
-      }
-    }
-    return NS_OK;
-  }
-#endif
-  return DoDecode(aSample, inputData, inputSize, aGotFrame, aResults);
 }

 MediaResult
@ -426,7 +365,8 @@ FFmpegVideoDecoder<LIBAV_VER>::ProcessDrain()
  empty->mTimecode = TimeUnit::FromMicroseconds(mLastInputDts);
  bool gotFrame = false;
  DecodedData results;
-  while (NS_SUCCEEDED(DoDecode(empty, &gotFrame, results)) && gotFrame) {
+  while (NS_SUCCEEDED(DoDecode(empty, nullptr, 0, &gotFrame, results)) &&
+         gotFrame) {
  }
  return DecodePromise::CreateAndResolve(Move(results), __func__);
 }
@ -439,15 +379,6 @@ FFmpegVideoDecoder<LIBAV_VER>::ProcessFlush()
  return FFmpegDataDecoder::ProcessFlush();
 }

-FFmpegVideoDecoder<LIBAV_VER>::~FFmpegVideoDecoder()
-{
-  MOZ_COUNT_DTOR(FFmpegVideoDecoder);
-  if (mCodecParser) {
-    mLib->av_parser_close(mCodecParser);
-    mCodecParser = nullptr;
-  }
-}
-
 AVCodecID
 FFmpegVideoDecoder<LIBAV_VER>::GetCodecId(const nsACString& aMimeType)
 {
--- a/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.h
+++ b/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.h
@ -33,7 +33,6 @@ public:
                     KnowsCompositor* aAllocator,
                     ImageContainer* aImageContainer,
                     bool aLowLatency);
-  virtual ~FFmpegVideoDecoder();

  RefPtr<InitPromise> Init() override;
  void InitCodecContext() override;
@ -53,14 +52,22 @@ public:
  static AVCodecID GetCodecId(const nsACString& aMimeType);

 private:
-  RefPtr<DecodePromise> ProcessDecode(MediaRawData* aSample) override;
  RefPtr<DecodePromise> ProcessDrain() override;
  RefPtr<FlushPromise> ProcessFlush() override;
-  MediaResult DoDecode(MediaRawData* aSample, bool* aGotFrame,
-                       DecodedData& aResults);
-  MediaResult DoDecode(MediaRawData* aSample, uint8_t* aData, int aSize,
-                       bool* aGotFrame, DecodedData& aResults);
+  MediaResult DoDecode(MediaRawData* aSample,
+                       uint8_t* aData,
+                       int aSize,
+                       bool* aGotFrame,
+                       DecodedData& aResults) override;
  void OutputDelayedFrames();
+  bool NeedParser() const override
+  {
+    return
+#if LIBAVCODEC_VERSION_MAJOR >= 55
+      mCodecID == AV_CODEC_ID_VP9 ||
+#endif
+      mCodecID == AV_CODEC_ID_VP8;
+  }

  /**
   * This method allocates a buffer for FFmpeg's decoder, wrapped in an Image.
@ -75,9 +82,6 @@ private:
  RefPtr<ImageContainer> mImageContainer;
  VideoInfo mInfo;

-  // Parser used for VP8 and VP9 decoding.
-  AVCodecParserContext* mCodecParser;
-
  class PtsCorrectionContext
  {
  public: