Bug 1690342 - P2: Populate NSAttributedText attributes with attributes. r=morgan DONTBUILD

A followup patch will make this work in e10s. This current implementation is non-ipc. Differential Revision: https://phabricator.services.mozilla.com/D103800
2021-02-08 23:26:31 +00:00 · 2021-02-08 23:26:31 +00:00 · e6912635e0
--- a/accessible/mac/GeckoTextMarker.h
+++ b/accessible/mac/GeckoTextMarker.h
@ -90,6 +90,11 @@ class GeckoTextMarkerRange final {
   */
  NSString* Text() const;

+  /**
+   * Return the attributed text enclosed by the range.
+   */
+  NSAttributedString* AttributedText() const;
+
  /**
   * Return length of characters enclosed by the range.
   */
--- a/accessible/mac/GeckoTextMarker.mm
+++ b/accessible/mac/GeckoTextMarker.mm
@ -402,6 +402,123 @@ NSString* GeckoTextMarkerRange::Text() const {
  return nsCocoaUtils::ToNSString(text);
 }

+static NSColor* ColorFromString(const nsString& aColorStr) {
+  uint32_t r, g, b;
+  if (sscanf(NS_ConvertUTF16toUTF8(aColorStr).get(), "rgb(%u, %u, %u)", &r, &g,
+             &b) > 0) {
+    return [NSColor colorWithCalibratedRed:(CGFloat)r / 0xff
+                                     green:(CGFloat)g / 0xff
+                                      blue:(CGFloat)b / 0xff
+                                     alpha:1.0];
+  }
+
+  return nil;
+}
+
+static NSDictionary* StringAttributesFromAttributes(
+    nsTArray<Attribute>& aAttributes, const AccessibleOrProxy& aContainer) {
+  NSMutableDictionary* attrDict =
+      [NSMutableDictionary dictionaryWithCapacity:aAttributes.Length()];
+  NSMutableDictionary* fontAttrDict = [[NSMutableDictionary alloc] init];
+  [attrDict setObject:fontAttrDict forKey:@"AXFont"];
+  for (size_t ii = 0; ii < aAttributes.Length(); ii++) {
+    RefPtr<nsAtom> attrName = NS_Atomize(aAttributes.ElementAt(ii).Name());
+    if (attrName == nsGkAtoms::backgroundColor) {
+      if (NSColor* color = ColorFromString(aAttributes.ElementAt(ii).Value())) {
+        [attrDict setObject:(__bridge id)color.CGColor
+                     forKey:@"AXBackgroundColor"];
+      }
+    } else if (attrName == nsGkAtoms::color) {
+      if (NSColor* color = ColorFromString(aAttributes.ElementAt(ii).Value())) {
+        [attrDict setObject:(__bridge id)color.CGColor
+                     forKey:@"AXForegroundColor"];
+      }
+    } else if (attrName == nsGkAtoms::font_size) {
+      float fontPointSize = 0;
+      if (sscanf(NS_ConvertUTF16toUTF8(aAttributes.ElementAt(ii).Value()).get(),
+                 "%fpt", &fontPointSize) > 0) {
+        int32_t fontPixelSize = static_cast<int32_t>(fontPointSize * 4 / 3);
+        [fontAttrDict setObject:@(fontPixelSize) forKey:@"AXFontSize"];
+      }
+    } else if (attrName == nsGkAtoms::font_family) {
+      [fontAttrDict
+          setObject:nsCocoaUtils::ToNSString(aAttributes.ElementAt(ii).Value())
+             forKey:@"AXFontFamily"];
+    } else if (attrName == nsGkAtoms::textUnderlineColor) {
+      [attrDict setObject:@1 forKey:@"AXUnderline"];
+      if (NSColor* color = ColorFromString(aAttributes.ElementAt(ii).Value())) {
+        [attrDict setObject:(__bridge id)color.CGColor
+                     forKey:@"AXUnderlineColor"];
+      }
+    } else if (attrName == nsGkAtoms::invalid) {
+      // XXX: There is currently no attribute for grammar
+      if (aAttributes.ElementAt(ii).Value().EqualsLiteral("spelling")) {
+        [attrDict setObject:@YES
+                     forKey:NSAccessibilityMarkedMisspelledTextAttribute];
+      }
+    } else {
+      [attrDict
+          setObject:nsCocoaUtils::ToNSString(aAttributes.ElementAt(ii).Value())
+             forKey:nsCocoaUtils::ToNSString(NS_ConvertUTF8toUTF16(
+                        aAttributes.ElementAt(ii).Name()))];
+    }
+  }
+
+  mozAccessible* container = GetNativeFromGeckoAccessible(aContainer);
+  id<MOXAccessible> link =
+      [container moxFindAncestor:^BOOL(id<MOXAccessible> moxAcc, BOOL* stop) {
+        return [[moxAcc moxRole] isEqualToString:NSAccessibilityLinkRole];
+      }];
+  if (link) {
+    [attrDict setObject:link forKey:@"AXLink"];
+  }
+
+  id<MOXAccessible> heading =
+      [container moxFindAncestor:^BOOL(id<MOXAccessible> moxAcc, BOOL* stop) {
+        return [[moxAcc moxRole] isEqualToString:@"AXHeading"];
+      }];
+  if (heading) {
+    [attrDict setObject:[heading moxValue] forKey:@"AXHeadingLevel"];
+  }
+
+  return attrDict;
+}
+
+NSAttributedString* GeckoTextMarkerRange::AttributedText() const {
+  NSMutableAttributedString* str =
+      [[[NSMutableAttributedString alloc] init] autorelease];
+
+  if (mStart.mContainer.IsProxy() && mEnd.mContainer.IsProxy()) {
+    NSAttributedString* substr =
+        [[[NSAttributedString alloc] initWithString:Text()] autorelease];
+    [str appendAttributedString:substr];
+  } else if (auto htWrap = mStart.ContainerAsHyperTextWrap()) {
+    nsTArray<nsString> texts;
+    nsTArray<Accessible*> containers;
+    nsTArray<nsCOMPtr<nsIPersistentProperties>> props;
+
+    htWrap->AttributedTextForRange(texts, props, containers, mStart.mOffset,
+                                   mEnd.ContainerAsHyperTextWrap(),
+                                   mEnd.mOffset);
+
+    MOZ_ASSERT(texts.Length() == props.Length() &&
+               texts.Length() == containers.Length());
+
+    for (size_t i = 0; i < texts.Length(); i++) {
+      nsTArray<Attribute> attributes;
+      nsAccUtils::PersistentPropertiesToArray(props.ElementAt(i), &attributes);
+
+      NSAttributedString* substr = [[[NSAttributedString alloc]
+          initWithString:nsCocoaUtils::ToNSString(texts.ElementAt(i))
+              attributes:StringAttributesFromAttributes(
+                             attributes, containers.ElementAt(i))] autorelease];
+      [str appendAttributedString:substr];
+    }
+  }
+
+  return str;
+}
+
 int32_t GeckoTextMarkerRange::Length() const {
  int32_t length = 0;
  if (mStart.mContainer.IsProxy() && mEnd.mContainer.IsProxy()) {
--- a/accessible/mac/HyperTextAccessibleWrap.h
+++ b/accessible/mac/HyperTextAccessibleWrap.h
@ -22,6 +22,12 @@ class HyperTextAccessibleWrap : public HyperTextAccessible {
  void TextForRange(nsAString& aText, int32_t aStartOffset,
                    HyperTextAccessible* aEndContainer, int32_t aEndOffset);

+  void AttributedTextForRange(
+      nsTArray<nsString>& aStrings,
+      nsTArray<nsCOMPtr<nsIPersistentProperties>>& aProperties,
+      nsTArray<Accessible*>& aContainers, int32_t aStartOffset,
+      HyperTextAccessible* aEndContainer, int32_t aEndOffset);
+
  nsIntRect BoundsForRange(int32_t aStartOffset,
                           HyperTextAccessible* aEndContainer,
                           int32_t aEndOffset);
--- a/accessible/mac/HyperTextAccessibleWrap.mm
+++ b/accessible/mac/HyperTextAccessibleWrap.mm
@ -13,6 +13,7 @@
 #include "nsFrameSelection.h"
 #include "TextRange.h"
 #include "TreeWalker.h"
+#include "nsPersistentProperties.h"

 using namespace mozilla;
 using namespace mozilla::a11y;
@ -253,6 +254,57 @@ void HyperTextAccessibleWrap::TextForRange(nsAString& aText,
  }
 }

+void HyperTextAccessibleWrap::AttributedTextForRange(
+    nsTArray<nsString>& aStrings,
+    nsTArray<nsCOMPtr<nsIPersistentProperties>>& aProperties,
+    nsTArray<Accessible*>& aContainers, int32_t aStartOffset,
+    HyperTextAccessible* aEndContainer, int32_t aEndOffset) {
+  if (IsHTMLListItem()) {
+    Accessible* maybeBullet = GetChildAtOffset(aStartOffset - 1);
+    if (maybeBullet) {
+      Accessible* bullet = AsHTMLListItem()->Bullet();
+      if (maybeBullet == bullet) {
+        nsAutoString text;
+        TextSubstring(0, nsAccUtils::TextLength(bullet), text);
+
+        int32_t unusedAttrStartOffset, unusedAttrEndOffset;
+        nsCOMPtr<nsIPersistentProperties> props =
+            TextAttributes(true, aStartOffset - 1, &unusedAttrStartOffset,
+                           &unusedAttrEndOffset);
+        nsTArray<Attribute> textAttrArray;
+        nsAccUtils::PersistentPropertiesToArray(props, &textAttrArray);
+
+        aStrings.AppendElement(text);
+        aProperties.AppendElement(props);
+        aContainers.AppendElement(this);
+      }
+    }
+  }
+
+  HyperTextIterator iter(this, aStartOffset, aEndContainer, aEndOffset);
+  while (iter.Next()) {
+    int32_t attrStartOffset = 0;
+    int32_t attrEndOffset = iter.mCurrentStartOffset;
+    do {
+      nsCOMPtr<nsIPersistentProperties> props =
+          iter.mCurrentContainer->TextAttributes(
+              true, attrEndOffset, &attrStartOffset, &attrEndOffset);
+
+      nsAutoString text;
+      iter.mCurrentContainer->TextSubstring(
+          attrStartOffset < iter.mCurrentStartOffset ? iter.mCurrentStartOffset
+                                                     : attrStartOffset,
+          attrEndOffset < iter.mCurrentEndOffset ? attrEndOffset
+                                                 : iter.mCurrentEndOffset,
+          text);
+
+      aStrings.AppendElement(text);
+      aProperties.AppendElement(props);
+      aContainers.AppendElement(iter.mCurrentContainer);
+    } while (attrEndOffset < iter.mCurrentEndOffset);
+  }
+}
+
 nsIntRect HyperTextAccessibleWrap::BoundsForRange(
    int32_t aStartOffset, HyperTextAccessible* aEndContainer,
    int32_t aEndOffset) {
--- a/accessible/mac/MOXAccessibleProtocol.h
+++ b/accessible/mac/MOXAccessibleProtocol.h
@ -388,7 +388,7 @@
 // AXStyleRangeForIndex
 - (NSValue* _Nullable)moxStyleRangeForIndex:(NSNumber* _Nonnull)index;

-// AttributedStringForRange
+// AXAttributedStringForRange
 - (NSAttributedString* _Nullable)moxAttributedStringForRange:
    (NSValue* _Nonnull)range;

--- a/accessible/mac/MOXTextMarkerDelegate.mm
+++ b/accessible/mac/MOXTextMarkerDelegate.mm
@ -337,9 +337,13 @@ static nsDataHashtable<nsUint64HashKey, MOXTextMarkerDelegate*> sDelegates;

 - (NSAttributedString*)moxAttributedStringForTextMarkerRange:
    (id)textMarkerRange {
-  return [[[NSAttributedString alloc]
-      initWithString:[self moxStringForTextMarkerRange:textMarkerRange]]
-      autorelease];
+  mozilla::a11y::GeckoTextMarkerRange range(mGeckoDocAccessible,
+                                            textMarkerRange);
+  if (!range.IsValid()) {
+    return nil;
+  }
+
+  return range.AttributedText();
 }

 - (NSValue*)moxBoundsForTextMarkerRange:(id)textMarkerRange {
--- a/accessible/mac/mozTextAccessible.mm
+++ b/accessible/mac/mozTextAccessible.mm
@ -283,8 +283,13 @@ inline NSString* ToNSString(id aValue) {
 }

 - (NSAttributedString*)moxAttributedStringForRange:(NSValue*)range {
-  return [[[NSAttributedString alloc]
-      initWithString:[self moxStringForRange:range]] autorelease];
+  GeckoTextMarkerRange markerRange = [self textMarkerRangeFromRange:range];
+
+  if (!markerRange.IsValid()) {
+    return nil;
+  }
+
+  return markerRange.AttributedText();
 }

 - (NSValue*)moxRangeForLine:(NSNumber*)line {
@ -429,8 +434,15 @@ inline NSString* ToNSString(id aValue) {
 }

 - (NSAttributedString*)moxAttributedStringForRange:(NSValue*)range {
-  return [[[NSAttributedString alloc]
-      initWithString:[self moxStringForRange:range]] autorelease];
+  MOZ_ASSERT(!mGeckoAccessible.IsNull());
+
+  NSRange r = [range rangeValue];
+  GeckoTextMarkerRange textMarkerRange(mGeckoAccessible);
+  textMarkerRange.mStart.mOffset += r.location;
+  textMarkerRange.mEnd.mOffset =
+      textMarkerRange.mStart.mOffset + r.location + r.length;
+
+  return textMarkerRange.AttributedText();
 }

 - (NSValue*)moxBoundsForRange:(NSValue*)range {
--- a/accessible/tests/browser/mac/browser.ini
+++ b/accessible/tests/browser/mac/browser.ini
@ -47,4 +47,6 @@ skip-if = os == 'mac' && debug # Bug 1664577
 [browser_rich_listbox.js]
 [browser_live_regions.js]
 [browser_aria_busy.js]
-[browser_aria_controls_flowto.js]
+[browser_aria_controls_flowto.js]
+[browser_attributed_text.js]
+skip-if = os != 'mac' || e10s
--- a/accessible/tests/browser/mac/browser_attributed_text.js
+++ b/accessible/tests/browser/mac/browser_attributed_text.js
@ -0,0 +1,86 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+// Test read-only attributed strings
+addAccessibleTask(
+  `<h1>hello <a href="#" id="a1">world</a></h1>
+   <p>this <b style="color: red; background-color: yellow;" aria-invalid="spelling">is</b> <span style="text-decoration: underline dotted green;">a</span> <a href="#" id="a2">test</a></p>`,
+  async (browser, accDoc) => {
+    let macDoc = accDoc.nativeInterface.QueryInterface(
+      Ci.nsIAccessibleMacInterface
+    );
+
+    let range = macDoc.getParameterizedAttributeValue(
+      "AXTextMarkerRangeForUnorderedTextMarkers",
+      [
+        macDoc.getAttributeValue("AXStartTextMarker"),
+        macDoc.getAttributeValue("AXEndTextMarker"),
+      ]
+    );
+
+    let attributedText = macDoc.getParameterizedAttributeValue(
+      "AXAttributedStringForTextMarkerRange",
+      range
+    );
+
+    let attributesList = attributedText.map(
+      ({
+        string,
+        AXForegroundColor,
+        AXBackgroundColor,
+        AXUnderline,
+        AXUnderlineColor,
+        AXHeadingLevel,
+        AXFont,
+        AXLink,
+        AXMarkedMisspelled,
+      }) => [
+        string,
+        AXForegroundColor,
+        AXBackgroundColor,
+        AXUnderline,
+        AXUnderlineColor,
+        AXHeadingLevel,
+        AXFont.AXFontSize,
+        AXLink ? AXLink.getAttributeValue("AXDOMIdentifier") : null,
+        AXMarkedMisspelled,
+      ]
+    );
+
+    Assert.deepEqual(attributesList, [
+      // string, fg color, bg color, underline, underline color, heading level, font size, link id, misspelled
+      ["hello ", "#000000", "#ffffff", null, null, 1, 32, null, null],
+      ["world", "#0000ee", "#ffffff", 1, "#0000ee", 1, 32, "a1", null],
+      ["this ", "#000000", "#ffffff", null, null, null, 16, null, null],
+      ["is", "#ff0000", "#ffff00", null, null, null, 16, null, 1],
+      [" ", "#000000", "#ffffff", null, null, null, 16, null, null],
+      ["a", "#000000", "#ffffff", 1, "#008000", null, 16, null, null],
+      [" ", "#000000", "#ffffff", null, null, null, 16, null, null],
+      ["test", "#0000ee", "#ffffff", 1, "#0000ee", null, 16, "a2", null],
+    ]);
+  }
+);
+
+// Test misspelling in text area
+addAccessibleTask(
+  `<textarea id="t">hello worlf</textarea>`,
+  async (browser, accDoc) => {
+    let textArea = getNativeInterface(accDoc, "t");
+    let spellDone = waitForEvent(EVENT_TEXT_ATTRIBUTE_CHANGED, "t");
+    textArea.setAttributeValue("AXFocused", true);
+    await spellDone;
+
+    let range = textArea.getAttributeValue("AXVisibleCharacterRange");
+
+    let attributedText = textArea.getParameterizedAttributeValue(
+      "AXAttributedStringForRange",
+      NSRange(...range)
+    );
+
+    is(attributedText.length, 2);
+    ok(attributedText[1].AXMarkedMisspelled);
+  }
+);
--- a/accessible/tests/browser/mac/head.js
+++ b/accessible/tests/browser/mac/head.js
@ -107,8 +107,23 @@ function stringForRange(macDoc, range) {
    return "";
  }

-  return macDoc.getParameterizedAttributeValue(
+  let str = macDoc.getParameterizedAttributeValue(
    "AXStringForTextMarkerRange",
    range
  );
+
+  let attrStr = macDoc.getParameterizedAttributeValue(
+    "AXAttributedStringForTextMarkerRange",
+    range
+  );
+
+  // This is a fly-by test to make sure our attributed strings
+  // always match our flat strings.
+  is(
+    attrStr.map(({ string }) => string).join(""),
+    str,
+    "attributed text matches non-attributed text"
+  );
+
+  return str;
 }
--- a/accessible/xpcom/xpcAccessibleMacInterface.mm
+++ b/accessible/xpcom/xpcAccessibleMacInterface.mm
@ -1,4 +1,6 @@
-/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* clang-format off */
+/* -*- Mode: Objective-C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* clang-format on */
 /* vim: set ts=2 et sw=2 tw=80: */
 /* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
@ -256,6 +258,35 @@ nsresult xpcAccessibleMacInterface::NSObjectToJsValue(id aObj, JSContext* aCx,
      JS_SetUCProperty(aCx, obj, strKey.get(), strKey.Length(), value);
    }
    aResult.setObject(*obj);
+  } else if ([aObj isKindOfClass:[NSAttributedString class]]) {
+    NSAttributedString* attrStr = (NSAttributedString*)aObj;
+    __block NSMutableArray* attrRunArray = [[NSMutableArray alloc] init];
+
+    [attrStr
+        enumerateAttributesInRange:NSMakeRange(0, [attrStr length])
+                           options:NSAttributedStringEnumerationLongestEffectiveRangeNotRequired
+                        usingBlock:^(NSDictionary* attributes, NSRange range, BOOL* stop) {
+                          NSString* str = [[attrStr string] substringWithRange:range];
+                          if (!str || !attributes) {
+                            return;
+                          }
+
+                          NSMutableDictionary* attrRun = [attributes mutableCopy];
+                          attrRun[@"string"] = str;
+
+                          [attrRunArray addObject:attrRun];
+                        }];
+
+    // The attributed string is represented in js as an array of objects.
+    // Each object represents a run of text where the "string" property is the
+    // string value and all the AX* properties are the attributes.
+    return NSObjectToJsValue(attrRunArray, aCx, aResult);
+  } else if (CFGetTypeID(aObj) == CGColorGetTypeID()) {
+    const CGFloat* components = CGColorGetComponents((CGColorRef)aObj);
+    NSString* hexString =
+        [NSString stringWithFormat:@"#%02x%02x%02x", (int)(components[0] * 0xff),
+                                   (int)(components[1] * 0xff), (int)(components[2] * 0xff)];
+    return NSObjectToJsValue(hexString, aCx, aResult);
  } else if ([aObj respondsToSelector:@selector(isAccessibilityElement)]) {
    // We expect all of our accessibility objects to implement isAccessibilityElement
    // at the very least. If it is implemented we will assume its an accessibility object.