Bug 486918. Part 2: Add the ability to pre-downscale using a high-quality scaler on a separate thread. r=joe,jlebar

gfx/thebes/gfx2DGlue.h

@@ -196,6 +196,23 @@ inline gfxASurface::gfxImageFormat SurfaceFormatToImageFormat(SurfaceFormat aFor
   }
 }
 
+inline SurfaceFormat ImageFormatToSurfaceFormat(gfxASurface::gfxImageFormat aFormat)
+{
+  switch (aFormat) {
+  case gfxASurface::ImageFormatARGB32:
+    return FORMAT_B8G8R8A8;
+  case gfxASurface::ImageFormatRGB24:
+    return FORMAT_B8G8R8X8;
+  case gfxASurface::ImageFormatRGB16_565:
+    return FORMAT_R5G6B5;
+  case gfxASurface::ImageFormatA8:
+    return FORMAT_A8;
+  default:
+  case gfxASurface::ImageFormatUnknown:
+    return FORMAT_B8G8R8A8;
+  }
+}
+
 inline gfxASurface::gfxContentType ContentForFormat(const SurfaceFormat &aFormat)
 {
   switch (aFormat) {

image/decoders/nsBMPDecoder.cpp

@@ -9,13 +9,13 @@
 
 #include <stdlib.h>
 
+#include "ImageLogging.h"
 #include "EndianMacros.h"
 #include "nsBMPDecoder.h"
 
 #include "nsIInputStream.h"
 #include "RasterImage.h"
 #include "imgIContainerObserver.h"
-#include "ImageLogging.h"
 
 namespace mozilla {
 namespace image {

image/decoders/nsJPEGDecoder.cpp

@@ -4,8 +4,8 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-#include "nsJPEGDecoder.h"
 #include "ImageLogging.h"
+#include "nsJPEGDecoder.h"
 
 #include "imgIContainerObserver.h"
 

image/decoders/nsPNGDecoder.cpp

@@ -4,8 +4,8 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-#include "nsPNGDecoder.h"
 #include "ImageLogging.h"
+#include "nsPNGDecoder.h"
 
 #include "nsMemory.h"
 #include "nsRect.h"

image/src/RasterImage.cpp

@@ -4,6 +4,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "base/histogram.h"
+#include "ImageLogging.h"
 #include "nsComponentManagerUtils.h"
 #include "imgIContainerObserver.h"
 #include "nsError.h"
@@ -16,7 +17,6 @@
 #include "nsStringStream.h"
 #include "prmem.h"
 #include "prenv.h"
-#include "ImageLogging.h"
 #include "ImageContainer.h"
 #include "Layers.h"
 
@@ -28,12 +28,66 @@
 #include "nsIconDecoder.h"
 
 #include "gfxContext.h"
+#include "gfx2DGlue.h"
 
 #include "mozilla/Preferences.h"
 #include "mozilla/StandardInteger.h"
 #include "mozilla/Telemetry.h"
 #include "mozilla/TimeStamp.h"
 #include "mozilla/ClearOnShutdown.h"
+#include "mozilla/gfx/Scale.h"
+
+// The high-quality scaler requires Skia.
+#ifdef MOZ_ENABLE_SKIA
+
+static bool
+ScaleFrameImage(imgFrame *aSrcFrame, imgFrame *aDstFrame,
+                const gfxSize &aScaleFactors)
+{
+  if (aScaleFactors.width <= 0 || aScaleFactors.height <= 0)
+    return false;
+
+  imgFrame *srcFrame = aSrcFrame;
+  nsIntRect srcRect = srcFrame->GetRect();
+  uint32_t dstWidth = NSToIntRoundUp(srcRect.width * aScaleFactors.width);
+  uint32_t dstHeight = NSToIntRoundUp(srcRect.height * aScaleFactors.height);
+
+  // Destination is unconditionally ARGB32 because that's what the scaler
+  // outputs.
+  nsresult rv = aDstFrame->Init(0, 0, dstWidth, dstHeight,
+                                gfxASurface::ImageFormatARGB32);
+  if (!NS_FAILED(rv)) {
+    uint8_t* srcData;
+    uint32_t srcDataLength;
+    // Source frame data is locked/unlocked on the main thread.
+    srcFrame->GetImageData(&srcData, &srcDataLength);
+    NS_ASSERTION(srcData != nullptr, "Source data is unavailable! Is it locked?");
+
+    uint8_t* dstData;
+    uint32_t dstDataLength;
+    aDstFrame->LockImageData();
+    aDstFrame->GetImageData(&dstData, &dstDataLength);
+
+    // This returns an SkBitmap backed by dstData; since it wrote to dstData,
+    // we don't need to look at that SkBitmap.
+    mozilla::gfx::Scale(srcData, srcRect.width, srcRect.height, aSrcFrame->GetImageBytesPerRow(),
+                        dstData, dstWidth, dstHeight, aDstFrame->GetImageBytesPerRow(),
+                        mozilla::gfx::ImageFormatToSurfaceFormat(aSrcFrame->GetFormat()));
+
+    aDstFrame->UnlockImageData();
+    return true;
+  }
+
+  return false;
+}
+#else // MOZ_ENABLE_SKIA
+static bool
+ScaleFrameImage(imgFrame *aSrcFrame, imgFrame *aDstFrame,
+                const gfxSize &aScaleFactors)
+{
+  return false;
+}
+#endif // MOZ_ENABLE_SKIA
 
 using namespace mozilla;
 using namespace mozilla::image;
@@ -136,6 +190,12 @@ namespace image {
 
 /* static */ StaticRefPtr<RasterImage::DecodeWorker> RasterImage::DecodeWorker::sSingleton;
 
+#define PRE_DOWNSCALE_MIN_FACTOR 0.9
+
+/* static */ nsRefPtr<RasterImage::ScaleWorker> RasterImage::ScaleWorker::sSingleton;
+/* static */ nsRefPtr<RasterImage::DrawWorker> RasterImage::DrawWorker::sSingleton;
+static nsCOMPtr<nsIThread> sScaleWorkerThread = nullptr;
+
 #ifndef DEBUG
 NS_IMPL_ISUPPORTS3(RasterImage, imgIContainer, nsIProperties,
                    nsISupportsWeakReference)
@@ -170,7 +230,8 @@ RasterImage::RasterImage(imgStatusTracker* aStatusTracker) :
   mInDecoder(false),
   mAnimationFinished(false),
   mFinishing(false),
-  mInUpdateImageContainer(false)
+  mInUpdateImageContainer(false),
+  mScaleRequest(this)
 {
   // Set up the discard tracker node.
   mDiscardTrackerNode.img = this;
@@ -184,6 +245,8 @@ RasterImage::RasterImage(imgStatusTracker* aStatusTracker) :
 //******************************************************************************
 RasterImage::~RasterImage()
 {
+  ScaleRequest::Stop(mScaleRequest.image);
+
   delete mAnim;
 
   for (unsigned int i = 0; i < mFrames.Length(); ++i)
@@ -226,6 +289,8 @@ RasterImage::Initialize()
   // Create our singletons now, so we don't have to worry about what thread
   // they're created on.
   DecodeWorker::Singleton();
+  DrawWorker::Singleton();
+  ScaleWorker::Singleton();
 }
 
 nsresult
@@ -2584,6 +2649,219 @@ RasterImage::SyncDecode()
   return mError ? NS_ERROR_FAILURE : NS_OK;
 }
 
+/* static */ RasterImage::ScaleWorker*
+RasterImage::ScaleWorker::Singleton()
+{
+  if (!sSingleton) {
+    sSingleton = new ScaleWorker();
+    ClearOnShutdown(&sSingleton);
+  }
+
+  return sSingleton;
+}
+
+nsresult
+RasterImage::ScaleWorker::Run()
+{
+  if (!mInitialized) {
+    PR_SetCurrentThreadName("Image Scaler");
+    mInitialized = true;
+  }
+
+  ScaleRequest* request;
+  gfxSize scale;
+  imgFrame* frame;
+  {
+    MutexAutoLock lock(ScaleWorker::Singleton()->mRequestsMutex);
+    request = mScaleRequests.popFirst();
+    if (!request)
+      return NS_OK;
+
+    scale = request->scale;
+    frame = request->srcFrame;
+  }
+
+  nsAutoPtr<imgFrame> scaledFrame(new imgFrame());
+  bool scaled = ScaleFrameImage(frame, scaledFrame, scale);
+
+  // OK, we've got a new scaled image. Let's get the main thread to unlock and
+  // redraw it.
+  {
+    MutexAutoLock lock(ScaleWorker::Singleton()->mRequestsMutex);
+    if (scaled && scale == request->scale && !request->isInList()) {
+      request->dstFrame = scaledFrame;
+      request->done = true;
+    }
+
+    DrawWorker::Singleton()->RequestDraw(request->image);
+  }
+  return NS_OK;
+}
+
+// Note: you MUST call RequestScale with the ScaleWorker mutex held.
+void
+RasterImage::ScaleWorker::RequestScale(RasterImage* aImg)
+{
+  mRequestsMutex.AssertCurrentThreadOwns();
+
+  ScaleRequest* request = &aImg->mScaleRequest;
+  if (request->isInList())
+    return;
+
+  mScaleRequests.insertBack(request);
+
+  if (!sScaleWorkerThread) {
+    NS_NewThread(getter_AddRefs(sScaleWorkerThread), this, NS_DISPATCH_NORMAL);
+    ClearOnShutdown(&sScaleWorkerThread);
+  }
+  else {
+    sScaleWorkerThread->Dispatch(this, NS_DISPATCH_NORMAL);
+  }
+}
+
+/* static */ RasterImage::DrawWorker*
+RasterImage::DrawWorker::Singleton()
+{
+  if (!sSingleton) {
+    sSingleton = new DrawWorker();
+    ClearOnShutdown(&sSingleton);
+  }
+
+  return sSingleton;
+}
+
+nsresult
+RasterImage::DrawWorker::Run()
+{
+  ScaleRequest* request;
+  {
+    MutexAutoLock lock(ScaleWorker::Singleton()->mRequestsMutex);
+    request = mDrawRequests.popFirst();
+  }
+  if (request) {
+    // ScaleWorker is finished with this request, so we can unlock the data now.
+    request->UnlockSourceData();
+    // We have to reset dstFrame if request was stopped while ScaleWorker was scaling.
+    if (request->stopped) {
+      ScaleRequest::Stop(request->image);
+    }
+    nsCOMPtr<imgIContainerObserver> observer(do_QueryReferent(request->image->mObserver));
+    if (request->done && observer) {
+      imgFrame *scaledFrame = request->dstFrame.get();
+      scaledFrame->ImageUpdated(scaledFrame->GetRect());
+      nsIntRect frameRect = request->srcFrame->GetRect();
+      observer->FrameChanged(nullptr, request->image, &frameRect);
+    }
+  }
+
+  return NS_OK;
+}
+
+void
+RasterImage::DrawWorker::RequestDraw(RasterImage* aImg)
+{
+  ScaleRequest* request = &aImg->mScaleRequest;
+  mDrawRequests.insertBack(request);
+  NS_DispatchToMainThread(this, NS_DISPATCH_NORMAL);
+}
+
+void
+RasterImage::ScaleRequest::Stop(RasterImage* aImg)
+{
+  ScaleRequest* request = &aImg->mScaleRequest;
+  // It's safe to unlock source image data only if request is in the list.
+  // Otherwise we may be reading from the source while performing scaling
+  // and can't interrupt immediately.
+  if (request->isInList()) {
+    request->remove();
+    request->UnlockSourceData();
+  }
+  // We have to check if request is finished before dropping the destination
+  // frame. Otherwise we may be writing to the dest while performing scaling.
+  if (request->done) {
+    request->done = false;
+    request->dstFrame = nullptr;
+    request->scale.width = 0;
+    request->scale.height = 0;
+  }
+  request->stopped = true;
+}
+
+bool
+RasterImage::CanScale(gfxPattern::GraphicsFilter aFilter,
+                      gfxSize aScale)
+{
+// The high-quality scaler requires Skia.
+#ifdef MOZ_ENABLE_SKIA
+  return (aFilter == gfxPattern::FILTER_GOOD) &&
+          !mAnim && mDecoded &&
+          (aScale.width <= 1.0 && aScale.height <= 1.0) &&
+          (aScale.width < PRE_DOWNSCALE_MIN_FACTOR ||
+           aScale.height < PRE_DOWNSCALE_MIN_FACTOR);
+#else
+  return false;
+#endif
+}
+
+void
+RasterImage::DrawWithPreDownscaleIfNeeded(imgFrame *aFrame,
+                                          gfxContext *aContext,
+                                          gfxPattern::GraphicsFilter aFilter,
+                                          const gfxMatrix &aUserSpaceToImageSpace,
+                                          const gfxRect &aFill,
+                                          const nsIntRect &aSubimage)
+{
+  imgFrame *frame = aFrame;
+  nsIntRect framerect = frame->GetRect();
+  gfxMatrix userSpaceToImageSpace = aUserSpaceToImageSpace;
+  gfxMatrix imageSpaceToUserSpace = aUserSpaceToImageSpace;
+  imageSpaceToUserSpace.Invert();
+  gfxSize scale = imageSpaceToUserSpace.ScaleFactors(true);
+  nsIntRect subimage = aSubimage;
+
+  if (CanScale(aFilter, scale)) {
+    MutexAutoLock lock(ScaleWorker::Singleton()->mRequestsMutex);
+    // If scale factor is still the same that we scaled for and
+    // ScaleWorker has done it's job, then we can use pre-downscaled frame.
+    // If scale factor has changed, order new request.
+    if (mScaleRequest.scale == scale) {
+      if (mScaleRequest.done) {
+        frame = mScaleRequest.dstFrame.get();
+        userSpaceToImageSpace.Multiply(gfxMatrix().Scale(scale.width, scale.height));
+
+        // Since we're switching to a scaled image, we we need to transform the
+        // area of the subimage to draw accordingly, since imgFrame::Draw()
+        // doesn't know about scaled frames.
+        subimage.ScaleRoundOut(scale.width, scale.height);
+      }
+    } else {
+      // FIXME: Current implementation doesn't support pre-downscale
+      // mechanism for multiple images from same src, since we cache
+      // pre-downscaled frame only for the latest requested scale.
+      // The solution is to cache more than one scaled image frame
+      // for each RasterImage.
+      int scaling = mScaleRequest.srcDataLocked ? 1 : 0;
+      if (mLockCount - scaling == 1) {
+        ScaleRequest::Stop(this);
+        mScaleRequest.srcFrame = frame;
+        mScaleRequest.scale = scale;
+        mScaleRequest.stopped = false;
+
+        // We need to make sure that source data is available before asking to scale.
+        if (mScaleRequest.LockSourceData()) {
+          ScaleWorker::Singleton()->RequestScale(this);
+        }
+      }
+    }
+  }
+
+  nsIntMargin padding(framerect.x, framerect.y,
+                      mSize.width - framerect.XMost(),
+                      mSize.height - framerect.YMost());
+
+  frame->Draw(aContext, aFilter, userSpaceToImageSpace, aFill, padding, subimage);
+}
+
 //******************************************************************************
 /* [noscript] void draw(in gfxContext aContext,
  *                      in gfxGraphicsFilter aFilter,
@@ -2655,12 +2933,7 @@ RasterImage::Draw(gfxContext *aContext,
     return NS_OK; // Getting the frame (above) touches the image and kicks off decoding
   }
 
-  nsIntRect framerect = frame->GetRect();
-  nsIntMargin padding(framerect.x, framerect.y, 
-                      mSize.width - framerect.XMost(),
-                      mSize.height - framerect.YMost());
-
-  frame->Draw(aContext, aFilter, aUserSpaceToImageSpace, aFill, padding, aSubimage, aFlags);
+  DrawWithPreDownscaleIfNeeded(frame, aContext, aFilter, aUserSpaceToImageSpace, aFill, aSubimage);
 
   if (mDecoded && !mDrawStartTime.IsNull()) {
       TimeDuration drawLatency = TimeStamp::Now() - mDrawStartTime;
@@ -2668,6 +2941,7 @@ RasterImage::Draw(gfxContext *aContext,
       // clear the value of mDrawStartTime
       mDrawStartTime = TimeStamp();
   }
+
   return NS_OK;
 }
 
@@ -2716,6 +2990,11 @@ RasterImage::UnlockImage()
   // Decrement our lock count
   mLockCount--;
 
+  if (ScaleWorker::sSingleton && mLockCount == 0) {
+    MutexAutoLock lock(ScaleWorker::Singleton()->mRequestsMutex);
+    ScaleRequest::Stop(this);
+  }
+
   // If we've decoded this image once before, we're currently decoding again,
   // and our lock count is now zero (so nothing is forcing us to keep the
   // decoded data around), try to cancel the decode and throw away whatever

image/src/RasterImage.h

@@ -17,6 +17,7 @@
 #ifndef mozilla_imagelib_RasterImage_h_
 #define mozilla_imagelib_RasterImage_h_
 
+#include "mozilla/Mutex.h"
 #include "Image.h"
 #include "nsCOMArray.h"
 #include "nsCOMPtr.h"
@@ -470,6 +471,109 @@ private:
     bool mPendingInEventLoop;
   };
 
+  struct ScaleRequest : public LinkedListElement<ScaleRequest>
+  {
+    ScaleRequest(RasterImage* aImage)
+      : image(aImage)
+      , srcFrame(nullptr)
+      , dstFrame(nullptr)
+      , scale(0, 0)
+      , done(false)
+      , stopped(false)
+      , srcDataLocked(false)
+    {};
+
+    bool LockSourceData()
+    {
+      if (!srcDataLocked) {
+        bool success = true;
+        success = success && NS_SUCCEEDED(image->LockImage());
+        success = success && NS_SUCCEEDED(srcFrame->LockImageData());
+        srcDataLocked = success;
+      }
+      return srcDataLocked;
+    }
+
+    bool UnlockSourceData()
+    {
+      bool success = true;
+      if (srcDataLocked) {
+        success = success && NS_SUCCEEDED(image->UnlockImage());
+        success = success && NS_SUCCEEDED(srcFrame->UnlockImageData());
+
+        // If unlocking fails, there's nothing we can do to make it work, so we
+        // claim that we're not locked regardless.
+        srcDataLocked = false;
+      }
+      return success;
+    }
+
+    static void Stop(RasterImage* aImg);
+
+    RasterImage* const image;
+    imgFrame *srcFrame;
+    nsAutoPtr<imgFrame> dstFrame;
+    gfxSize scale;
+    bool done;
+    bool stopped;
+    bool srcDataLocked;
+  };
+
+  class ScaleWorker : public nsRunnable
+  {
+  public:
+    static ScaleWorker* Singleton();
+
+    NS_IMETHOD Run();
+
+  /* statics */
+    static nsRefPtr<ScaleWorker> sSingleton;
+
+  private: /* methods */
+    ScaleWorker()
+      : mRequestsMutex("RasterImage.ScaleWorker.mRequestsMutex")
+      , mInitialized(false)
+    {};
+
+    // Note: you MUST call RequestScale with the ScaleWorker mutex held.
+    void RequestScale(RasterImage* aImg);
+
+  private: /* members */
+
+    friend class RasterImage;
+    LinkedList<ScaleRequest> mScaleRequests;
+    Mutex mRequestsMutex;
+    bool mInitialized;
+  };
+
+  class DrawWorker : public nsRunnable
+  {
+  public:
+    static DrawWorker* Singleton();
+
+    NS_IMETHOD Run();
+
+  /* statics */
+    static nsRefPtr<DrawWorker> sSingleton;
+
+  private: /* methods */
+    DrawWorker() {};
+
+    void RequestDraw(RasterImage* aImg);
+
+  private: /* members */
+
+    friend class RasterImage;
+    LinkedList<ScaleRequest> mDrawRequests;
+  };
+
+  void DrawWithPreDownscaleIfNeeded(imgFrame *aFrame,
+                                    gfxContext *aContext,
+                                    gfxPattern::GraphicsFilter aFilter,
+                                    const gfxMatrix &aUserSpaceToImageSpace,
+                                    const gfxRect &aFill,
+                                    const nsIntRect &aSubimage);
+
   /**
    * Advances the animation. Typically, this will advance a single frame, but it
    * may advance multiple frames. This may happen if we have infrequently
@@ -675,6 +779,9 @@ private: // data
   bool     IsDecodeFinished();
   TimeStamp mDrawStartTime;
 
+  inline bool CanScale(gfxPattern::GraphicsFilter aFilter, gfxSize aScale);
+  ScaleRequest mScaleRequest;
+
   // Decoder shutdown
   enum eShutdownIntent {
     eShutdownIntent_Done        = 0,