Bug 1759504 - Return the ImageText results from recognizeCurrentImageText r=emilio,nordzilla

Differential Revision: https://phabricator.services.mozilla.com/D143423

dom/base/nsImageLoadingContent.cpp

@@ -50,6 +50,7 @@
 #include "mozilla/dom/Element.h"
 #include "mozilla/dom/PContent.h"  // For TextRecognitionResult
 #include "mozilla/dom/HTMLImageElement.h"
+#include "mozilla/dom/ImageTextBinding.h"
 #include "mozilla/dom/ImageTracker.h"
 #include "mozilla/dom/ScriptSettings.h"
 #include "mozilla/net/UrlClassifierFeatureFactory.h"
@@ -1239,13 +1240,32 @@ already_AddRefed<Promise> nsImageLoadingContent::RecognizeCurrentImageText(
           domPromise->MaybeRejectWithInvalidStateError("Request not current");
           return;
         }
+        auto& textRecognitionResult = aValue.ResolveValue();
         Element* el = ilc->AsContent()->AsElement();
         el->AttachAndSetUAShadowRoot(Element::NotifyUAWidgetSetup::Yes);
         TextRecognition::FillShadow(*el->GetShadowRoot(),
-                                    aValue.ResolveValue());
+                                    textRecognitionResult);
         el->NotifyUAWidgetSetupOrChange();
-        // TODO: Maybe resolve with the recognition results?
-        domPromise->MaybeResolveWithUndefined();
+
+        nsTArray<ImageText> imageTexts(textRecognitionResult.quads().Length());
+        nsIGlobalObject* global = el->OwnerDoc()->GetOwnerGlobal();
+
+        for (const auto& quad : textRecognitionResult.quads()) {
+          NotNull<ImageText*> imageText = imageTexts.AppendElement();
+
+          // Note: These points are not actually CSSPixels, but a DOMQuad is
+          // a conveniently similar structure that can store these values.
+          CSSPoint points[4];
+          points[0] = CSSPoint(quad.points()[0].x, quad.points()[0].y);
+          points[1] = CSSPoint(quad.points()[1].x, quad.points()[1].y);
+          points[2] = CSSPoint(quad.points()[2].x, quad.points()[2].y);
+          points[3] = CSSPoint(quad.points()[3].x, quad.points()[3].y);
+
+          imageText->mQuad = new DOMQuad(global, points);
+          imageText->mConfidence = quad.confidence();
+          imageText->mString = quad.string();
+        }
+        domPromise->MaybeResolve(std::move(imageTexts));
       });
   return domPromise.forget();
 }

dom/chrome-webidl/ImageText.webidl

@@ -0,0 +1,29 @@
+/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+/**
+ * Text that has been extracted from an image using an image recognition API.
+ */
+dictionary ImageText {
+  /**
+   * A scalar value representing how confident the text recognition model was in the
+   * produced result.
+   */
+  required float confidence;
+
+  /**
+   * The recognized text.
+   */
+  required DOMString string;
+
+  /**
+   * A DOMQuad representing the bounds of the text. This can be a free-form four pointed
+   * polygon, and not necessarily a square or parallelogram. It does not actually contain
+   * CSSPixels, but the points are ranged 0-1 and represent a ratio of the width / height
+   * of the source image.
+   */
+  required DOMQuad quad;
+};

dom/chrome-webidl/moz.build

@@ -51,6 +51,7 @@ WEBIDL_FILES = [
     "Fluent.webidl",
     "FrameLoader.webidl",
     "HeapSnapshot.webidl",
+    "ImageText.webidl",
     "InspectorUtils.webidl",
     "IteratorResult.webidl",
     "JSActor.webidl",

dom/webidl/HTMLImageElement.webidl

@@ -49,7 +49,7 @@ interface HTMLImageElement : HTMLElement {
            [NewObject]
            Promise<void> decode();
            [NewObject, ChromeOnly]
-           Promise<void> recognizeCurrentImageText();
+           Promise<sequence<ImageText>> recognizeCurrentImageText();
 };
 
 // http://www.whatwg.org/specs/web-apps/current-work/#other-elements,-attributes-and-apis