Bug 952456 - Part 1: Implement gonk/nsClipboard for rich text and raw image. r=fabrice

Handle text/html and image MIME types on gonk/nsClipboard

widget/gonk/GonkClipboardData.cpp

@@ -0,0 +1,75 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "GonkClipboardData.h"
+#include "mozilla/gfx/DataSurfaceHelpers.h"
+
+namespace mozilla {
+
+void
+GonkClipboardData::SetText(const nsAString &aText)
+{
+  mPlain = aText;
+}
+
+bool
+GonkClipboardData::HasText() const
+{
+  return !mPlain.IsEmpty();
+}
+
+const nsAString&
+GonkClipboardData::GetText() const
+{
+  return mPlain;
+}
+
+void
+GonkClipboardData::SetHTML(const nsAString &aHTML)
+{
+  mHTML = aHTML;
+}
+
+bool
+GonkClipboardData::HasHTML() const
+{
+  return !mHTML.IsEmpty();
+}
+
+const nsAString&
+GonkClipboardData::GetHTML() const
+{
+  return mHTML;
+}
+
+void
+GonkClipboardData::SetImage(gfx::DataSourceSurface* aDataSource)
+{
+  // Clone a new DataSourceSurface and store it.
+  mImage = gfx::CreateDataSourceSurfaceByCloning(aDataSource);
+}
+
+bool
+GonkClipboardData::HasImage() const
+{
+  return static_cast<bool>(mImage);
+}
+
+already_AddRefed<gfx::DataSourceSurface>
+GonkClipboardData::GetImage() const
+{
+  // Return cloned DataSourceSurface.
+  RefPtr<gfx::DataSourceSurface> cloned = gfx::CreateDataSourceSurfaceByCloning(mImage);
+  return cloned.forget();
+}
+
+void
+GonkClipboardData::Clear()
+{
+  mPlain.Truncate(0);
+  mHTML.Truncate(0);
+  mImage = nullptr;
+}
+
+} // namespace mozilla

widget/gonk/GonkClipboardData.h

@@ -0,0 +1,49 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_GonkClipboardData
+#define mozilla_GonkClipboardData
+
+#include "mozilla/RefPtr.h"
+#include "nsString.h"
+
+namespace mozilla {
+
+namespace gfx {
+class DataSourceSurface;
+}
+
+class GonkClipboardData final
+{
+public:
+  explicit GonkClipboardData() = default;
+  ~GonkClipboardData() = default;
+
+  // For text/plain
+  void SetText(const nsAString &aText);
+  bool HasText() const;
+  const nsAString& GetText() const;
+
+  // For text/html
+  void SetHTML(const nsAString &aHTML);
+  bool HasHTML() const;
+  const nsAString& GetHTML() const;
+
+  // For images
+  void SetImage(gfx::DataSourceSurface* aDataSource);
+  bool HasImage() const;
+  already_AddRefed<gfx::DataSourceSurface> GetImage() const;
+
+  // For other APIs
+  void Clear();
+
+private:
+  nsAutoString mPlain;
+  nsAutoString mHTML;
+  RefPtr<gfx::DataSourceSurface> mImage;
+};
+
+} // namespace mozilla
+
+#endif // mozilla_GonkClipboardData

widget/gonk/moz.build

@@ -58,6 +58,7 @@ elif CONFIG['ANDROID_VERSION'] == '15':
 SOURCES += [
     'GeckoTouchDispatcher.cpp',
     'GfxInfo.cpp',
+    'GonkClipboardData.cpp',
     'GonkMemoryPressureMonitoring.cpp',
     'GonkPermission.cpp',
     'HwcComposer2D.cpp',
@@ -82,6 +83,7 @@ LOCAL_INCLUDES += [
     '/dom/system/android',
     '/gfx/skia/skia/include/config',
     '/gfx/skia/skia/include/core',
+    '/image',
     '/widget',
 ]
 

widget/gonk/nsClipboard.cpp

@@ -2,12 +2,19 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-#include "mozilla/dom/ContentChild.h"
 #include "nsClipboard.h"
+
+#include "gfxDrawable.h"
+#include "gfxUtils.h"
+#include "ImageOps.h"
+#include "imgIContainer.h"
+#include "imgTools.h"
+#include "mozilla/dom/ContentChild.h"
 #include "nsClipboardProxy.h"
 #include "nsISupportsPrimitives.h"
-#include "nsCOMPtr.h"
 #include "nsComponentManagerUtils.h"
+#include "nsCOMPtr.h"
+#include "nsStringStream.h"
 #include "nsXULAppAPI.h"
 
 using namespace mozilla;
@@ -17,15 +24,18 @@ using mozilla::dom::ContentChild;
 #define LOGI(args...) __android_log_print(ANDROID_LOG_INFO, LOG_TAG, ## args)
 #define LOGE(args...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, ## args)
 
+
 NS_IMPL_ISUPPORTS(nsClipboard, nsIClipboard)
 
 nsClipboard::nsClipboard()
+  : mClipboard(mozilla::MakeUnique<GonkClipboardData>())
 {
 }
 
 NS_IMETHODIMP
 nsClipboard::SetData(nsITransferable *aTransferable,
-                     nsIClipboardOwner *anOwner, int32_t aWhichClipboard)
+                     nsIClipboardOwner *anOwner,
+                     int32_t aWhichClipboard)
 {
   if (aWhichClipboard != kGlobalClipboard) {
     return NS_ERROR_NOT_IMPLEMENTED;
@@ -37,28 +47,108 @@ nsClipboard::SetData(nsITransferable *aTransferable,
     return clipboardProxy->SetData(aTransferable, anOwner, aWhichClipboard);
   }
 
-  nsCOMPtr<nsISupports> tmp;
-  uint32_t len;
-  nsresult rv  = aTransferable->GetTransferData(kUnicodeMime, getter_AddRefs(tmp),
-                                                &len);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-  nsCOMPtr<nsISupportsString> supportsString = do_QueryInterface(tmp);
-  // No support for non-text data
-  if (NS_WARN_IF(!supportsString)) {
-    LOGE("No support for non-text data. See bug 952456.");
-    return NS_ERROR_NOT_IMPLEMENTED;
-  }
-  nsAutoString buffer;
-  supportsString->GetData(buffer);
+  // Clear out the clipboard in order to set the new data
+  EmptyClipboard(aWhichClipboard);
+
+  // Get the types of supported flavors.
+  nsCOMPtr<nsISupportsArray> flavorList;
+  nsresult rv = aTransferable->FlavorsTransferableCanExport(getter_AddRefs(flavorList));
+  if (!flavorList || NS_FAILED(rv)) {
+    return NS_ERROR_FAILURE;
+  }
+
+  uint32_t flavorCount = 0;
+  flavorList->Count(&flavorCount);
+  bool imageAdded = false;
+  for (uint32_t i = 0; i < flavorCount; ++i) {
+    nsCOMPtr<nsISupportsCString> currentFlavor = do_QueryElementAt(flavorList, i);
+
+    if (currentFlavor) {
+      // MIME type
+      nsXPIDLCString flavorStr;
+      currentFlavor->ToString(getter_Copies(flavorStr));
+
+      // Clip is the data which will be sent to the clipboard.
+      nsCOMPtr<nsISupports> clip;
+      uint32_t len;
+
+      if (flavorStr.EqualsLiteral(kUnicodeMime)) {
+        // text/plain
+        rv = aTransferable->GetTransferData(flavorStr, getter_AddRefs(clip), &len);
+        nsCOMPtr<nsISupportsString> wideString = do_QueryInterface(clip);
+        if (!wideString || NS_FAILED(rv)) {
+          continue;
+        }
+
+        nsAutoString utf16string;
+        wideString->GetData(utf16string);
+        mClipboard->SetText(utf16string);
+      } else if (flavorStr.EqualsLiteral(kHTMLMime)) {
+        // text/html
+        rv = aTransferable->GetTransferData(flavorStr, getter_AddRefs(clip), &len);
+        nsCOMPtr<nsISupportsString> wideString = do_QueryInterface(clip);
+        if (!wideString || NS_FAILED(rv)) {
+          continue;
+        }
+
+        nsAutoString utf16string;
+        wideString->GetData(utf16string);
+        mClipboard->SetHTML(utf16string);
+      } else if (!imageAdded && // image is added only once to the clipboard.
+                 (flavorStr.EqualsLiteral(kNativeImageMime) ||
+                  flavorStr.EqualsLiteral(kPNGImageMime) ||
+                  flavorStr.EqualsLiteral(kJPEGImageMime) ||
+                  flavorStr.EqualsLiteral(kJPGImageMime) ||
+                  flavorStr.EqualsLiteral(kGIFImageMime))) {
+        // image/[png|jpeg|jpg|gif] or application/x-moz-nativeimage
+
+        // Look through our transfer data for the image.
+        static const char* const imageMimeTypes[] = {
+          kNativeImageMime, kPNGImageMime, kJPEGImageMime, kJPGImageMime, kGIFImageMime };
+
+        nsCOMPtr<nsISupportsInterfacePointer> imgPtr;
+        for (uint32_t i = 0; !imgPtr && i < ArrayLength(imageMimeTypes); ++i) {
+          aTransferable->GetTransferData(imageMimeTypes[i], getter_AddRefs(clip), &len);
+          imgPtr = do_QueryInterface(clip);
+        }
+        if (!imgPtr) {
+          continue;
+        }
+
+        nsCOMPtr<nsISupports> imageData;
+        imgPtr->GetData(getter_AddRefs(imageData));
+        nsCOMPtr<imgIContainer> image(do_QueryInterface(imageData));
+        if (!image) {
+          continue;
+        }
+
+        RefPtr<gfx::SourceSurface> surface =
+          image->GetFrame(imgIContainer::FRAME_CURRENT,
+                          imgIContainer::FLAG_SYNC_DECODE);
+        if (!surface) {
+          continue;
+        }
+
+        RefPtr<gfx::DataSourceSurface> dataSurface;
+        if (surface->GetFormat() == gfx::SurfaceFormat::B8G8R8A8) {
+          dataSurface = surface->GetDataSurface();
+        } else {
+          // Convert format to SurfaceFormat::B8G8R8A8.
+          dataSurface = gfxUtils::CopySurfaceToDataSourceSurfaceWithFormat(surface, gfx::SurfaceFormat::B8G8R8A8);
+        }
+
+        mClipboard->SetImage(dataSurface);
+        imageAdded = true;
+      }
+    }
+  }
 
-  mClipboard = buffer;
   return NS_OK;
 }
 
 NS_IMETHODIMP
-nsClipboard::GetData(nsITransferable *aTransferable, int32_t aWhichClipboard)
+nsClipboard::GetData(nsITransferable *aTransferable,
+                     int32_t aWhichClipboard)
 {
   if (aWhichClipboard != kGlobalClipboard) {
     return NS_ERROR_NOT_IMPLEMENTED;
@@ -70,29 +160,92 @@ nsClipboard::GetData(nsITransferable *aTransferable, int32_t aWhichClipboard)
     return clipboardProxy->GetData(aTransferable, aWhichClipboard);
   }
 
-  nsAutoString buffer(mClipboard);
+  // Get flavor list that includes all acceptable flavors (including
+  // ones obtained through conversion).
+  // Note: We don't need to call nsITransferable::AddDataFlavor here
+  //       because ContentParent already did.
+  nsCOMPtr<nsISupportsArray> flavorList;
+  nsresult rv = aTransferable->FlavorsTransferableCanImport(getter_AddRefs(flavorList));
 
-  nsresult rv;
-  nsCOMPtr<nsISupportsString> dataWrapper =
-    do_CreateInstance(NS_SUPPORTS_STRING_CONTRACTID, &rv);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
+  if (!flavorList || NS_FAILED(rv)) {
+    return NS_ERROR_FAILURE;
   }
 
-  rv = dataWrapper->SetData(buffer);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
+  // Walk through flavors and see which flavor matches the one being pasted.
+  uint32_t flavorCount;
+  flavorList->Count(&flavorCount);
 
-  // If our data flavor has already been added, this will fail. But we don't care
-  aTransferable->AddDataFlavor(kUnicodeMime);
+  for (uint32_t i = 0; i < flavorCount; ++i) {
+    nsCOMPtr<nsISupportsCString> currentFlavor = do_QueryElementAt(flavorList, i);
 
-  nsCOMPtr<nsISupports> nsisupportsDataWrapper =
-    do_QueryInterface(dataWrapper);
-  rv = aTransferable->SetTransferData(kUnicodeMime, nsisupportsDataWrapper,
-                                      buffer.Length() * sizeof(char16_t));
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
+    if (currentFlavor) {
+      // flavorStr is the mime type.
+      nsXPIDLCString flavorStr;
+      currentFlavor->ToString(getter_Copies(flavorStr));
+
+      // text/plain, text/Unicode
+      if (flavorStr.EqualsLiteral(kUnicodeMime) && mClipboard->HasText()) {
+        nsresult rv;
+        nsCOMPtr<nsISupportsString> dataWrapper = do_CreateInstance(NS_SUPPORTS_STRING_CONTRACTID, &rv);
+        rv = dataWrapper->SetData(mClipboard->GetText());
+        if (NS_WARN_IF(NS_FAILED(rv))) {
+          continue;
+        }
+
+        nsCOMPtr<nsISupports> genericDataWrapper = do_QueryInterface(dataWrapper);
+        uint32_t len = mClipboard->GetText().Length() * sizeof(char16_t);
+        rv = aTransferable->SetTransferData(flavorStr, genericDataWrapper, len);
+        if (NS_WARN_IF(NS_FAILED(rv))) {
+          continue;
+        }
+        break;
+      }
+
+      // text/html
+      if (flavorStr.EqualsLiteral(kHTMLMime) && mClipboard->HasHTML()) {
+        nsresult rv;
+        nsCOMPtr<nsISupportsString> dataWrapper = do_CreateInstance(NS_SUPPORTS_STRING_CONTRACTID, &rv);
+        rv = dataWrapper->SetData(mClipboard->GetHTML());
+        if (NS_WARN_IF(NS_FAILED(rv))) {
+          continue;
+        }
+
+        nsCOMPtr<nsISupports> genericDataWrapper = do_QueryInterface(dataWrapper);
+        uint32_t len = mClipboard->GetHTML().Length() * sizeof(char16_t);
+        rv = aTransferable->SetTransferData(flavorStr, genericDataWrapper, len);
+        if (NS_WARN_IF(NS_FAILED(rv))) {
+          continue;
+        }
+        break;
+      }
+
+      // image/[png|jpeg|jpg|gif]
+      if ((flavorStr.EqualsLiteral(kPNGImageMime) ||
+           flavorStr.EqualsLiteral(kJPEGImageMime) ||
+           flavorStr.EqualsLiteral(kJPGImageMime) ||
+           flavorStr.EqualsLiteral(kGIFImageMime)) &&
+          mClipboard->HasImage() ) {
+        // Get image buffer from clipboard.
+        RefPtr<gfx::DataSourceSurface> image = mClipboard->GetImage();
+
+        // Encode according to MIME type.
+        nsRefPtr<gfxDrawable> drawable = new gfxSurfaceDrawable(image, image->GetSize());
+        nsCOMPtr<imgIContainer> imageContainer(image::ImageOps::CreateFromDrawable(drawable));
+        nsCOMPtr<imgITools> imgTool = do_GetService(NS_IMGTOOLS_CID);
+
+        nsCOMPtr<nsIInputStream> byteStream;
+        imgTool->EncodeImage(imageContainer, flavorStr, EmptyString(), getter_AddRefs(byteStream));
+
+        // Set transferable.
+        nsresult rv = aTransferable->SetTransferData(flavorStr,
+                                                     byteStream,
+                                                     sizeof(nsIInputStream*));
+        if (NS_WARN_IF(NS_FAILED(rv))) {
+          continue;
+        }
+        break;
+      }
+    }
   }
 
   return NS_OK;
@@ -105,7 +258,7 @@ nsClipboard::EmptyClipboard(int32_t aWhichClipboard)
     return NS_ERROR_NOT_IMPLEMENTED;
   }
   if (XRE_IsParentProcess()) {
-    mClipboard.Truncate(0);
+    mClipboard->Clear();
   } else {
     ContentChild::GetSingleton()->SendEmptyClipboard(aWhichClipboard);
   }
@@ -123,7 +276,27 @@ nsClipboard::HasDataMatchingFlavors(const char **aFlavorList,
     return NS_ERROR_NOT_IMPLEMENTED;
   }
   if (XRE_IsParentProcess()) {
-    *aHasType = !mClipboard.IsEmpty();
+    // Retrieve the union of all aHasType in aFlavorList
+    for (uint32_t i = 0; i < aLength; ++i) {
+      const char *flavor = aFlavorList[i];
+      if (!flavor) {
+        continue;
+      }
+      if (!strcmp(flavor, kUnicodeMime) && mClipboard->HasText()) {
+        *aHasType = true;
+      } else if (!strcmp(flavor, kHTMLMime) && mClipboard->HasHTML()) {
+        *aHasType = true;
+      } else if (!strcmp(flavor, kJPEGImageMime) ||
+                 !strcmp(flavor, kJPGImageMime) ||
+                 !strcmp(flavor, kPNGImageMime) ||
+                 !strcmp(flavor, kGIFImageMime)) {
+        // We will encode the image into any format you want, so we don't
+        // need to check each specific format
+        if (mClipboard->HasImage()) {
+          *aHasType = true;
+        }
+      }
+    }
   } else {
     nsRefPtr<nsClipboardProxy> clipboardProxy = new nsClipboardProxy();
     return clipboardProxy->HasDataMatchingFlavors(aFlavorList, aLength, aWhichClipboard, aHasType);

widget/gonk/nsClipboard.h

@@ -1,16 +1,16 @@
-/* -*- Mode: c++; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*-
- * This Source Code Form is subject to the terms of the Mozilla Public
+/* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef nsClipbard_h__
 #define nsClipbard_h__
 
+#include "GonkClipboardData.h"
+#include "mozilla/UniquePtr.h"
 #include "nsIClipboard.h"
 
 class nsClipboard final : public nsIClipboard
 {
-  nsAutoString mClipboard;
 public:
   NS_DECL_ISUPPORTS
   NS_DECL_NSICLIPBOARD
@@ -19,6 +19,9 @@ public:
 
 protected:
   ~nsClipboard() {}
+
+private:
+  mozilla::UniquePtr<mozilla::GonkClipboardData> mClipboard;
 };
 
 #endif