Bug 938034 - Add new GonkCameraImage image type. r=roc

dom/media/webrtc/GonkCameraImage.cpp

@@ -0,0 +1,81 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "GonkCameraImage.h"
+#include "stagefright/MediaBuffer.h"
+
+namespace mozilla {
+
+GonkCameraImage::GonkCameraImage(layers::BufferRecycleBin* aRecycleBin)
+  : PlanarYCbCrImage(aRecycleBin)
+  , mMonitor("GonkCameraImage.Monitor")
+  , mMediaBuffer(nullptr)
+  , mThread(nullptr)
+{
+  mFormat = ImageFormat::GONK_CAMERA_IMAGE;
+}
+
+GonkCameraImage::~GonkCameraImage()
+{
+  ReentrantMonitorAutoEnter mon(mMonitor);
+  // mMediaBuffer must be cleared before destructor.
+  MOZ_ASSERT(mMediaBuffer == nullptr);
+}
+
+nsresult
+GonkCameraImage::GetBuffer(android::MediaBuffer** aBuffer)
+{
+  ReentrantMonitorAutoEnter mon(mMonitor);
+
+  if (!mMediaBuffer) {
+    return NS_ERROR_FAILURE;
+  }
+
+  MOZ_ASSERT(NS_GetCurrentThread() == mThread);
+
+  *aBuffer = mMediaBuffer;
+  mMediaBuffer->add_ref();
+
+  return NS_OK;
+}
+
+bool
+GonkCameraImage::HasMediaBuffer()
+{
+  ReentrantMonitorAutoEnter mon(mMonitor);
+  return mMediaBuffer != nullptr;
+}
+
+nsresult
+GonkCameraImage::SetBuffer(android::MediaBuffer* aBuffer)
+{
+  ReentrantMonitorAutoEnter mon(mMonitor);
+  MOZ_ASSERT(!mMediaBuffer);
+
+  mMediaBuffer = aBuffer;
+  mMediaBuffer->add_ref();
+  mThread = NS_GetCurrentThread();
+
+  return NS_OK;
+}
+
+nsresult
+GonkCameraImage::ClearBuffer()
+{
+  ReentrantMonitorAutoEnter mon(mMonitor);
+
+  if (mMediaBuffer) {
+    MOZ_ASSERT(NS_GetCurrentThread() == mThread);
+    mMediaBuffer->release();
+    mMediaBuffer = nullptr;
+    mThread = nullptr;
+  }
+  return NS_OK;
+}
+
+} // namespace mozilla
+
+

dom/media/webrtc/GonkCameraImage.h

@@ -0,0 +1,74 @@
+/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef GONKCAMERAIMAGE_H
+#define GONKCAMERAIMAGE_H
+
+#include "mozilla/ReentrantMonitor.h"
+#include "ImageLayers.h"
+#include "ImageContainer.h"
+
+namespace android {
+class MOZ_EXPORT MediaBuffer;
+}
+
+namespace mozilla {
+
+/**
+ * GonkCameraImage has two parts. One is preview image which will be saved in
+ * PlanarYCbCrImage, another kind is the MediaBuffer keeps in mMediaBuffer
+ * which is from gonk camera recording callback. The data in MediaBuffer is Gonk
+ * shared memory based on android binder (IMemory), the actual format in IMemory
+ * is platform dependent.
+ * This instance is created in MediaEngine when the preview image arrives.
+ * The MediaBuffer is attached to the current created GonkCameraImage via SetBuffer().
+ * After sending this image to MediaStreamGraph by AppendToTrack(), ClearBuffer()
+ * must be called to clear MediaBuffer to avoid MediaBuffer be kept in MSG thread.
+ * The reason to keep MediaBuffer be accessed from MSG thread is MediaBuffer is
+ * limited resource and it could cause frame rate jitter if MediaBuffer stay too
+ * long in other threads.
+ * So there will be 3 threads to accessed this class. First is camera preview
+ * thread which creates an instance of this class and initialize the preview
+ * image in the base class PlanarYCbCrImage. Second is the camera recording
+ * thread which attaches MediaBuffer and sends this image to MediaStreamDirectListener.
+ * Third is the MSG thread via NotifyPull, the image should have preview image
+ * only in NotifyPull.
+ *
+ * Note: SetBuffer() and GetBuffer() should be called from the same thread. It
+ *       is forbidden to call GetBuffer() from other threads.
+ */
+class GonkCameraImage : public layers::PlanarYCbCrImage
+{
+public:
+  GonkCameraImage(layers::BufferRecycleBin* aRecycleBin);
+
+  // The returned aBuffer has called aBuffer->add_ref() already, so it is caller's
+  // duty to release aBuffer. It should be called from the same thread which
+  // called SetBuffer().
+  nsresult GetBuffer(android::MediaBuffer** aBuffer);
+
+  // Set MediaBuffer to image. It is caller's responsibility to call ClearBuffer()
+  // after the MediaBuffer is sent via MediaStreamGraph.
+  nsresult SetBuffer(android::MediaBuffer* aBuffer);
+
+  // It should be called from the same thread which called SetBuffer().
+  nsresult ClearBuffer();
+
+  bool HasMediaBuffer();
+
+protected:
+  virtual ~GonkCameraImage();
+
+  // mMonitor protects mMediaBuffer and mThread.
+  ReentrantMonitor mMonitor;
+  android::MediaBuffer* mMediaBuffer;
+  // Check if current thread is the same one which called SetBuffer().
+  // It doesn't need to hold reference count.
+  DebugOnly<nsIThread*> mThread;
+};
+
+} // namespace mozilla
+
+#endif /* GONKCAMERAIMAGE_H */

dom/media/webrtc/moz.build

@@ -36,8 +36,12 @@ if CONFIG['MOZ_WEBRTC']:
     ]
     # Gonk camera source.
     if CONFIG['MOZ_B2G_CAMERA']:
-        EXPORTS += ['MediaEngineGonkVideoSource.h']
+        EXPORTS += [
+            'GonkCameraImage.h',
+            'MediaEngineGonkVideoSource.h',
+        ]
         UNIFIED_SOURCES += [
+            'GonkCameraImage.cpp',
             'MediaEngineGonkVideoSource.cpp',
         ]
 

gfx/layers/ImageContainer.cpp

@@ -19,6 +19,7 @@
 #include "YCbCrUtils.h"                 // for YCbCr conversions
 #ifdef MOZ_WIDGET_GONK
 #include "GrallocImages.h"
+#include "GonkCameraImage.h"
 #endif
 #include "gfx2DGlue.h"
 #include "mozilla/gfx/2D.h"
@@ -60,6 +61,10 @@ ImageFactory::CreateImage(ImageFormat aFormat,
     img = new OverlayImage();
     return img.forget();
   }
+  if (aFormat == ImageFormat::GONK_CAMERA_IMAGE) {
+    img = new GonkCameraImage(aRecycleBin);
+    return img.forget();
+  }
 #endif
   if (aFormat == ImageFormat::PLANAR_YCBCR) {
     img = new PlanarYCbCrImage(aRecycleBin);

gfx/layers/ImageContainer.h

@@ -105,6 +105,7 @@ class TextureClient;
 class CompositableClient;
 class CompositableForwarder;
 class SurfaceDescriptor;
+class PlanarYCbCrImage;
 
 struct ImageBackendData
 {
@@ -167,6 +168,11 @@ public:
 
   virtual TemporaryRef<gfx::SourceSurface> GetAsSourceSurface() = 0;
 
+  virtual PlanarYCbCrImage* AsPlanarYCbCrImage()
+  {
+    return nullptr;
+  }
+
 protected:
   Image(void* aImplData, ImageFormat aFormat) :
     mImplData(aImplData),
@@ -751,6 +757,11 @@ public:
 
   virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const;
 
+  virtual PlanarYCbCrImage* AsPlanarYCbCrImage() MOZ_OVERRIDE
+  {
+    return this;
+  }
+
 protected:
   /**
    * Make a copy of the YCbCr data into local storage.

gfx/layers/ImageTypes.h

@@ -25,6 +25,16 @@ MOZ_BEGIN_ENUM_CLASS(ImageFormat)
    */
   GRALLOC_PLANAR_YCBCR,
 
+  /**
+   * The GONK_CAMERA_IMAGE format creates a GonkCameraImage, which contains two
+   * parts. One is PlanarYCbCr image for preview image. Another one is
+   * MediaBuffer from Gonk recording image. The preview image can be rendered in
+   * a layer for display. And the MediaBuffer will be used in component like OMX
+   * encoder. It is for GUM to support preview and recording image on Gonk
+   * camera.
+   */
+  GONK_CAMERA_IMAGE,
+
   /**
    * The SHARED_RGB format creates a SharedRGBImage, which stores RGB data in
    * shared memory. Some Android hardware video decoders require this format.

gfx/layers/client/ImageClient.cpp

@@ -165,9 +165,8 @@ ImageClientSingle::UpdateImage(ImageContainer* aContainer, uint32_t aContentFlag
     // Slow path, we should not be hitting it very often and if we do it means
     // we are using an Image class that is not backed by textureClient and we
     // should fix it.
-    if (image->GetFormat() == ImageFormat::PLANAR_YCBCR) {
-      PlanarYCbCrImage* ycbcr = static_cast<PlanarYCbCrImage*>(image);
-      const PlanarYCbCrData* data = ycbcr->GetData();
+    if (image->AsPlanarYCbCrImage()) {
+      const PlanarYCbCrData* data = image->AsPlanarYCbCrImage()->GetData();
       if (!data) {
         return false;
       }