From 4dbb2503ff37d2f22b398f03d2a63ec3d105862c Mon Sep 17 00:00:00 2001 From: Martin Stransky Date: Sat, 13 Jun 2020 18:38:38 +0000 Subject: [PATCH] Bug 1629788 [Wayland] Implement SW decoding to WaylandDMABufSurface, r=jya - Use WaylandDMABufSurface for SW decoded frames when they can be converted to NV12 format. - Rename VAAPIFrameHolder to DMABufSurface and use it as general placeholder for WaylandDMABufSurface surface. It's used for VA-API/SW video playback and holds decoded video images until thery are used by gecko rendering engine. - Implmenet a linked list of DMABufSurface where recently used frames are stored. The frames are recycled by ffmpeg decoder for VA-API and SW video playback. Differential Revision: https://phabricator.services.mozilla.com/D78292 --- .../platforms/ffmpeg/FFmpegVideoDecoder.cpp | 214 ++++++++++++------ .../platforms/ffmpeg/FFmpegVideoDecoder.h | 58 ++++- 2 files changed, 196 insertions(+), 76 deletions(-) diff --git a/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp b/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp index 59a82f03ccbc..ab9f4e90ae65 100644 --- a/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp +++ b/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp @@ -122,28 +122,43 @@ static AVPixelFormat ChooseVAAPIPixelFormat(AVCodecContext* aCodecContext, return AV_PIX_FMT_NONE; } -VAAPIFrameHolder::VAAPIFrameHolder(FFmpegLibWrapper* aLib, - WaylandDMABufSurface* aSurface, - AVCodecContext* aAVCodecContext, - AVFrame* aAVFrame) - : mLib(aLib), - mSurface(aSurface), - mAVHWFramesContext(mLib->av_buffer_ref(aAVCodecContext->hw_frames_ctx)), - mHWAVBuffer(mLib->av_buffer_ref(aAVFrame->buf[0])) { - FFMPEG_LOG("VAAPIFrameHolder is adding dmabuf surface UID = %d", - mSurface->GetUID()); - +DMABufSurface::DMABufSurface(WaylandDMABufSurface* aSurface, + FFmpegLibWrapper* aLib) + : mSurface(aSurface), + mLib(aLib), + mAVHWFramesContext(nullptr), + mHWAVBuffer(nullptr) { // Create global refcount object to track mSurface usage over // gects rendering engine. We can't release it until it's used // by GL compositor / WebRender. mSurface->GlobalRefCountCreate(); + FFMPEG_LOG("DMABufSurface: creating surface UID = %d", mSurface->GetUID()); } -VAAPIFrameHolder::~VAAPIFrameHolder() { - FFMPEG_LOG("VAAPIFrameHolder is releasing dmabuf surface UID = %d", +void DMABufSurface::LockVAAPIData(AVCodecContext* aAVCodecContext, + AVFrame* aAVFrame) { + FFMPEG_LOG("DMABufSurface: VAAPI locking dmabuf surface UID = %d", mSurface->GetUID()); - mLib->av_buffer_unref(&mHWAVBuffer); - mLib->av_buffer_unref(&mAVHWFramesContext); + if (aAVCodecContext && aAVFrame) { + mAVHWFramesContext = mLib->av_buffer_ref(aAVCodecContext->hw_frames_ctx); + mHWAVBuffer = mLib->av_buffer_ref(aAVFrame->buf[0]); + } +} + +void DMABufSurface::ReleaseVAAPIData() { + FFMPEG_LOG("DMABufSurface: VAAPI releasing dmabuf surface UID = %d", + mSurface->GetUID()); + if (mHWAVBuffer && mAVHWFramesContext) { + mLib->av_buffer_unref(&mHWAVBuffer); + mLib->av_buffer_unref(&mAVHWFramesContext); + } + mSurface->ReleaseSurface(); +} + +DMABufSurface::~DMABufSurface() { + FFMPEG_LOG("DMABufSurface: deleting dmabuf surface UID = %d", + mSurface->GetUID()); + ReleaseVAAPIData(); } AVCodec* FFmpegVideoDecoder::FindVAAPICodec() { @@ -225,14 +240,6 @@ MediaResult FFmpegVideoDecoder::InitVAAPIDecoder() { return NS_ERROR_NOT_AVAILABLE; } - auto layersBackend = mImageAllocator - ? mImageAllocator->GetCompositorBackendType() - : layers::LayersBackend::LAYERS_BASIC; - if (layersBackend != layers::LayersBackend::LAYERS_WR) { - FFMPEG_LOG("VA-API works with WebRender only!"); - return NS_ERROR_NOT_AVAILABLE; - } - AVCodec* codec = FindVAAPICodec(); if (!codec) { FFMPEG_LOG("Couldn't find ffmpeg VA-API decoder"); @@ -318,6 +325,7 @@ FFmpegVideoDecoder::FFmpegVideoDecoder( mVAAPIDeviceContext(nullptr), mDisableHardwareDecoding(aDisableHardwareDecoding), mDisplay(nullptr), + mUseDMABufSurfaces(false), #endif mImageAllocator(aAllocator), mImageContainer(aImageContainer), @@ -327,13 +335,25 @@ FFmpegVideoDecoder::FFmpegVideoDecoder( // initialization. mExtraData = new MediaByteBuffer; mExtraData->AppendElements(*aConfig.mExtraData); + +#ifdef MOZ_WAYLAND_USE_VAAPI + mUseDMABufSurfaces = + gfxPlatformGtk::GetPlatform()->UseWaylandDMABufVideoTextures() && + mImageAllocator && + (mImageAllocator->GetCompositorBackendType() == + layers::LayersBackend::LAYERS_WR); + + if (!mUseDMABufSurfaces) { + FFMPEG_LOG("DMA-BUF/VA-API can't be used, WebRender/Wayland is disabled"); + } +#endif } RefPtr FFmpegVideoDecoder::Init() { MediaResult rv; #ifdef MOZ_WAYLAND_USE_VAAPI - if (!mDisableHardwareDecoding) { + if (mUseDMABufSurfaces && !mDisableHardwareDecoding) { rv = InitVAAPIDecoder(); if (NS_SUCCEEDED(rv)) { return InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, __func__); @@ -433,6 +453,8 @@ MediaResult FFmpegVideoDecoder::DoDecode( } # ifdef MOZ_WAYLAND_USE_VAAPI + // Release unused VA-API surfaces before avcodec_receive_frame() as + // ffmpeg recycles VASurface for HW decoding. if (mVAAPIDeviceContext) { ReleaseUnusedVAAPIFrames(); } @@ -453,10 +475,19 @@ MediaResult FFmpegVideoDecoder::DoDecode( MediaResult rv; # ifdef MOZ_WAYLAND_USE_VAAPI - if (mVAAPIDeviceContext) { - MOZ_ASSERT(mFrame->format == AV_PIX_FMT_VAAPI_VLD); - rv = CreateImageVAAPI(mFrame->pkt_pos, mFrame->pkt_pts, - mFrame->pkt_duration, aResults); + if (mVAAPIDeviceContext || mUseDMABufSurfaces) { + rv = CreateImageDMABuf(mFrame->pkt_pos, mFrame->pkt_pts, + mFrame->pkt_duration, aResults); + + // If VA-API playback failed, just quit. Decoder is going to be restarted + // without VA-API. + // If VA-API is already off, disable DMABufSurfaces and fallback to + // default. + if (NS_FAILED(rv) && !mVAAPIDeviceContext) { + mUseDMABufSurfaces = false; + rv = CreateImage(mFrame->pkt_pos, mFrame->pkt_pts, mFrame->pkt_duration, + aResults); + } } else # endif { @@ -645,69 +676,122 @@ MediaResult FFmpegVideoDecoder::CreateImage( #ifdef MOZ_WAYLAND_USE_VAAPI void FFmpegVideoDecoder::ReleaseUnusedVAAPIFrames() { - std::list>::iterator holder = - mFrameHolders.begin(); - while (holder != mFrameHolders.end()) { - if (!(*holder)->IsUsed()) { - holder = mFrameHolders.erase(holder); - } else { - holder++; + int len = mDMABufSurfaces.Length(); + for (int i = 0; i < len; i++) { + if (!mDMABufSurfaces[i].IsUsed()) { + mDMABufSurfaces[i].ReleaseVAAPIData(); } } } -void FFmpegVideoDecoder::ReleaseAllVAAPIFrames() { - mFrameHolders.clear(); +DMABufSurface* FFmpegVideoDecoder::GetUnusedDMABufSurface() { + int len = mDMABufSurfaces.Length(); + for (int i = 0; i < len; i++) { + if (!mDMABufSurfaces[i].IsUsed()) { + return &(mDMABufSurfaces[i]); + } + } + return nullptr; } -MediaResult FFmpegVideoDecoder::CreateImageVAAPI( - int64_t aOffset, int64_t aPts, int64_t aDuration, - MediaDataDecoder::DecodedData& aResults) { - FFMPEG_LOG("Got one VAAPI frame output with pts=%" PRId64 " dts=%" PRId64 - " duration=%" PRId64 " opaque=%" PRId64, - aPts, mFrame->pkt_dts, aDuration, mCodecContext->reordered_opaque); +void FFmpegVideoDecoder::ReleaseDMABufSurfaces() { + mDMABufSurfaces.Clear(); +} - VADRMPRIMESurfaceDescriptor va_desc; +bool FFmpegVideoDecoder::GetVAAPISurfaceDescriptor( + VADRMPRIMESurfaceDescriptor& aVaDesc) { VASurfaceID surface_id = (VASurfaceID)(uintptr_t)mFrame->data[3]; VAStatus vas = mLib->vaExportSurfaceHandle( mDisplay, surface_id, VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2, VA_EXPORT_SURFACE_READ_ONLY | VA_EXPORT_SURFACE_SEPARATE_LAYERS, - &va_desc); + &aVaDesc); if (vas != VA_STATUS_SUCCESS) { - return MediaResult( - NS_ERROR_OUT_OF_MEMORY, - RESULT_DETAIL("Unable to get frame by vaExportSurfaceHandle()")); + return false; } vas = mLib->vaSyncSurface(mDisplay, surface_id); if (vas != VA_STATUS_SUCCESS) { NS_WARNING("vaSyncSurface() failed."); } - va_desc.width = mFrame->width; - va_desc.height = mFrame->height; + aVaDesc.width = mFrame->width; + aVaDesc.height = mFrame->height; - RefPtr surface = - WaylandDMABufSurfaceNV12::CreateNV12Surface(va_desc); - if (!surface) { + return true; +} + +MediaResult FFmpegVideoDecoder::CreateImageDMABuf( + int64_t aOffset, int64_t aPts, int64_t aDuration, + MediaDataDecoder::DecodedData& aResults) { + FFMPEG_LOG("DMABUF/VA-API Got one frame output with pts=%" PRId64 + "dts=%" PRId64 " duration=%" PRId64 " opaque=%" PRId64, + aPts, mFrame->pkt_dts, aDuration, mCodecContext->reordered_opaque); + + // With SW decode we support only YUV420P format with DMABuf surfaces. + if (!mVAAPIDeviceContext && mCodecContext->pix_fmt != AV_PIX_FMT_YUV420P) { return MediaResult( - NS_ERROR_OUT_OF_MEMORY, - RESULT_DETAIL("Unable to allocate WaylandDMABufSurfaceNV12.")); + NS_ERROR_NOT_IMPLEMENTED, + RESULT_DETAIL("DMA-BUF textures supports YUV420P format only")); } + VADRMPRIMESurfaceDescriptor vaDesc; + if (mVAAPIDeviceContext && !GetVAAPISurfaceDescriptor(vaDesc)) { + return MediaResult( + NS_ERROR_OUT_OF_MEMORY, + RESULT_DETAIL("Unable to get frame by vaExportSurfaceHandle()")); + } + + RefPtr waylandSurface; + + DMABufSurface* surface = GetUnusedDMABufSurface(); + if (!surface) { + if (mVAAPIDeviceContext) { + waylandSurface = WaylandDMABufSurfaceNV12::CreateNV12Surface(vaDesc); + } else { + waylandSurface = WaylandDMABufSurfaceNV12::CreateNV12Surface( + mFrame->width, mFrame->height, (void**)mFrame->data, + mFrame->linesize); + } + if (!waylandSurface) { + return MediaResult( + NS_ERROR_OUT_OF_MEMORY, + RESULT_DETAIL("Unable to get WaylandDMABufSurfaceNV12")); + } + # ifdef MOZ_LOGGING - static int uid = 0; - surface->SetUID(++uid); - FFMPEG_LOG("Created dmabuf UID = %d HW surface %x", uid, surface_id); + static int uid = 0; + waylandSurface->SetUID(++uid); + FFMPEG_LOG("Created new WaylandDMABufSurface UID = %d", uid); # endif + mDMABufSurfaces.AppendElement(DMABufSurface(waylandSurface, mLib)); + surface = &(mDMABufSurfaces[mDMABufSurfaces.Length() - 1]); + } else { + waylandSurface = surface->GetWaylandDMABufSurface(); + bool ret; - surface->SetYUVColorSpace(GetFrameColorSpace()); + if (mVAAPIDeviceContext) { + ret = waylandSurface->UpdateNV12Data(vaDesc); + } else { + ret = waylandSurface->UpdateNV12Data((void**)mFrame->data, + mFrame->linesize); + } - // Store reference to the decoded HW buffer, see VAAPIFrameHolder struct. - auto holder = - MakeUnique(mLib, surface, mCodecContext, mFrame); - mFrameHolders.push_back(std::move(holder)); + if (!ret) { + return MediaResult( + NS_ERROR_OUT_OF_MEMORY, + RESULT_DETAIL("Unable to upload data to WaylandDMABufSurfaceNV12")); + } + FFMPEG_LOG("Reusing WaylandDMABufSurface UID = %d", + waylandSurface->GetUID()); + } - RefPtr im = new layers::WaylandDMABUFSurfaceImage(surface); + if (mVAAPIDeviceContext) { + surface->LockVAAPIData(mCodecContext, mFrame); + } + + waylandSurface->SetYUVColorSpace(GetFrameColorSpace()); + + RefPtr im = + new layers::WaylandDMABUFSurfaceImage(waylandSurface); RefPtr vp = VideoData::CreateFromImage( mInfo.mDisplay, aOffset, TimeUnit::FromMicroseconds(aPts), @@ -758,8 +842,8 @@ AVCodecID FFmpegVideoDecoder::GetCodecId( void FFmpegVideoDecoder::ProcessShutdown() { #ifdef MOZ_WAYLAND_USE_VAAPI + ReleaseDMABufSurfaces(); if (mVAAPIDeviceContext) { - ReleaseAllVAAPIFrames(); mLib->av_buffer_unref(&mVAAPIDeviceContext); } #endif diff --git a/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.h b/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.h index 02fa4d73523c..bf510d8e90a4 100644 --- a/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.h +++ b/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.h @@ -12,12 +12,26 @@ #include "SimpleMap.h" #ifdef MOZ_WAYLAND_USE_VAAPI # include "mozilla/widget/WaylandDMABufSurface.h" -# include +# include "mozilla/LinkedList.h" #endif namespace mozilla { #ifdef MOZ_WAYLAND_USE_VAAPI +// DMABufSurface holds a reference to GPU data with a video frame. +// +// Actual GPU pixel data are stored at WaylandDMABufSurface and +// WaylandDMABufSurfaces is passed to gecko GL rendering pipeline via. +// WaylandDMABUFSurfaceImage. +// +// DMABufSurface can optionally hold VA-API ffmpeg related data to keep +// GPU data locked untill we need them. +// +// DMABufSurface is used for both HW accelerated video decoding (VA-API) +// and ffmpeg SW decoding. +// +// VA-API scenario +// // When VA-API decoding is running, ffmpeg allocates AVHWFramesContext - a pool // of "hardware" frames. Every "hardware" frame (VASurface) is backed // by actual piece of GPU memory which holds the decoded image data. @@ -33,20 +47,37 @@ namespace mozilla { // Unfortunately there isn't any obvious way how to mark particular VASurface // as used. The best we can do is to hold a reference to particular AVBuffer // from decoded AVFrame and AVHWFramesContext which owns the AVBuffer. - -class VAAPIFrameHolder final { +// +// FFmpeg SW decoding scenario +// +// When SW ffmpeg decoding is running, DMABufSurface contains only +// a WaylandDMABufSurface reference and VA-API related members are null. +// We own the WaylandDMABufSurface underlying GPU data and we use it for +// repeated rendering of video frames. +// +class DMABufSurface final { public: - VAAPIFrameHolder(FFmpegLibWrapper* aLib, WaylandDMABufSurface* aSurface, - AVCodecContext* aAVCodecContext, AVFrame* aAVFrame); - ~VAAPIFrameHolder(); + DMABufSurface(WaylandDMABufSurface* aSurface, FFmpegLibWrapper* aLib); + ~DMABufSurface(); + + // Lock VAAPI related data + void LockVAAPIData(AVCodecContext* aAVCodecContext, AVFrame* aAVFrame); + + // Release VAAPI related data, DMABufSurface can be reused + // for another frame. + void ReleaseVAAPIData(); // Check if WaylandDMABufSurface is used by any gecko rendering process // (WebRender or GL compositor) or by WaylandDMABUFSurfaceImage/VideoData. bool IsUsed() const { return mSurface->IsGlobalRefSet(); } + const RefPtr GetWaylandDMABufSurface() { + return mSurface->GetAsWaylandDMABufSurfaceNV12(); + } + private: - const FFmpegLibWrapper* mLib; const RefPtr mSurface; + const FFmpegLibWrapper* mLib; AVBufferRef* mAVHWFramesContext; AVBufferRef* mHWAVBuffer; }; @@ -118,11 +149,15 @@ class FFmpegVideoDecoder void InitVAAPICodecContext(); AVCodec* FindVAAPICodec(); bool IsHardwareAccelerated(nsACString& aFailureReason) const override; + bool GetVAAPISurfaceDescriptor(VADRMPRIMESurfaceDescriptor& aVaDesc); + + MediaResult CreateImageDMABuf(int64_t aOffset, int64_t aPts, + int64_t aDuration, + MediaDataDecoder::DecodedData& aResults); - MediaResult CreateImageVAAPI(int64_t aOffset, int64_t aPts, int64_t aDuration, - MediaDataDecoder::DecodedData& aResults); void ReleaseUnusedVAAPIFrames(); - void ReleaseAllVAAPIFrames(); + DMABufSurface* GetUnusedDMABufSurface(); + void ReleaseDMABufSurfaces(); #endif /** @@ -138,7 +173,8 @@ class FFmpegVideoDecoder AVBufferRef* mVAAPIDeviceContext; const bool mDisableHardwareDecoding; VADisplay mDisplay; - std::list> mFrameHolders; + bool mUseDMABufSurfaces; + nsTArray mDMABufSurfaces; #endif RefPtr mImageAllocator; RefPtr mImageContainer;