Bug 1848282 - Part 3. a11y's word segmenter should reference layout.word_select preferences. r=morgan

We need to consider `layout.word_select.stop_at_punctuation preference` even if using new segmenter. Differential Revision: https://phabricator.services.mozilla.com/D188786
2023-10-03 10:43:18 +00:00 · 2023-10-03 10:43:18 +00:00 · bef691023f
--- a/accessible/base/TextLeafRange.cpp
+++ b/accessible/base/TextLeafRange.cpp
@ -36,6 +36,7 @@
 #include "TextAttrs.h"

 using mozilla::intl::WordBreaker;
+using FindWordOptions = mozilla::intl::WordBreaker::FindWordOptions;

 namespace mozilla::a11y {

@ -826,11 +827,23 @@ TextLeafPoint TextLeafPoint::FindPrevWordStartSameAcc(
  if (mOffset == 0) {
    word.mBegin = 0;
  } else if (mOffset == static_cast<int32_t>(text.Length())) {
-    word = WordBreaker::FindWord(text, mOffset - 1);
+    word = WordBreaker::FindWord(
+        text, mOffset - 1,
+        StaticPrefs::layout_word_select_stop_at_punctuation()
+            ? FindWordOptions::StopAtPunctuation
+            : FindWordOptions::None);
  } else {
-    word = WordBreaker::FindWord(text, mOffset);
+    word = WordBreaker::FindWord(
+        text, mOffset,
+        StaticPrefs::layout_word_select_stop_at_punctuation()
+            ? FindWordOptions::StopAtPunctuation
+            : FindWordOptions::None);
  }
-  for (;; word = WordBreaker::FindWord(text, word.mBegin - 1)) {
+  for (;; word = WordBreaker::FindWord(
+              text, word.mBegin - 1,
+              StaticPrefs::layout_word_select_stop_at_punctuation()
+                  ? FindWordOptions::StopAtPunctuation
+                  : FindWordOptions::None)) {
    if (!aIncludeOrigin && static_cast<int32_t>(word.mBegin) == mOffset) {
      // A word possibly starts at the origin, but the caller doesn't want this
      // included.
@ -887,6 +900,7 @@ TextLeafPoint TextLeafPoint::FindNextWordStartSameAcc(
  }
  // Keep walking forward until we find an acceptable word start.
  intl::WordBreakIteratorUtf16 wordBreakIter(text);
+  int32_t previousPos = wordStart;
  Maybe<uint32_t> nextBreak = wordBreakIter.Seek(wordStart);
  for (;;) {
    if (!nextBreak || *nextBreak == text.Length()) {
@ -894,6 +908,16 @@ TextLeafPoint TextLeafPoint::FindNextWordStartSameAcc(
        // A line start always starts a new word.
        return lineStart;
      }
+      if (StaticPrefs::layout_word_select_stop_at_punctuation()) {
+        // If layout.word_select.stop_at_punctuation is true, we have to look
+        // for punctuation class since it may not break state in UAX#29.
+        for (int32_t i = previousPos + 1;
+             i < static_cast<int32_t>(text.Length()); i++) {
+          if (IsAcceptableWordStart(mAcc, text, i)) {
+            return TextLeafPoint(mAcc, i);
+          }
+        }
+      }
      return TextLeafPoint();
    }
    wordStart = AssertedCast<int32_t>(*nextBreak);
@ -904,6 +928,17 @@ TextLeafPoint TextLeafPoint::FindNextWordStartSameAcc(
    if (IsAcceptableWordStart(mAcc, text, wordStart)) {
      break;
    }
+
+    if (StaticPrefs::layout_word_select_stop_at_punctuation()) {
+      // If layout.word_select.stop_at_punctuation is true, we have to look
+      // for punctuation class since it may not break state in UAX#29.
+      for (int32_t i = previousPos + 1; i < wordStart; i++) {
+        if (IsAcceptableWordStart(mAcc, text, i)) {
+          return TextLeafPoint(mAcc, i);
+        }
+      }
+    }
+    previousPos = wordStart;
    nextBreak = wordBreakIter.Next();
  }
  return TextLeafPoint(mAcc, wordStart);
--- a/accessible/tests/browser/mac/browser_text_basics.js
+++ b/accessible/tests/browser/mac/browser_text_basics.js
@ -248,11 +248,14 @@ function testMarkerIntegrity(accDoc, expectedMarkerValues) {
 // Run tests with old word segmenter
 addAccessibleTask("mac/doc_textmarker_test.html", async (browser, accDoc) => {
  await SpecialPowers.pushPrefEnv({
-    set: [["intl.icu4x.segmenter.enabled", false]],
+    set: [
+      ["intl.icu4x.segmenter.enabled", false],
+      ["layout.word_select.stop_at_punctuation", true], // This is default
+    ],
  });

  const expectedValues = await SpecialPowers.spawn(browser, [], async () => {
-    return content.wrappedJSObject.getExpected(false);
+    return content.wrappedJSObject.getExpected(false, true);
  });

  testMarkerIntegrity(accDoc, expectedValues);
@ -260,13 +263,35 @@ addAccessibleTask("mac/doc_textmarker_test.html", async (browser, accDoc) => {
  await SpecialPowers.popPrefEnv();
 });

+// new UAX#14 segmenter without stop_at_punctuation.
 addAccessibleTask("mac/doc_textmarker_test.html", async (browser, accDoc) => {
  await SpecialPowers.pushPrefEnv({
-    set: [["intl.icu4x.segmenter.enabled", true]],
+    set: [
+      ["intl.icu4x.segmenter.enabled", true],
+      ["layout.word_select.stop_at_punctuation", false],
+    ],
  });

  const expectedValues = await SpecialPowers.spawn(browser, [], async () => {
-    return content.wrappedJSObject.getExpected(true);
+    return content.wrappedJSObject.getExpected(true, false);
+  });
+
+  testMarkerIntegrity(accDoc, expectedValues);
+
+  await SpecialPowers.popPrefEnv();
+});
+
+// new UAX#14 segmenter with stop_at_punctuation
+addAccessibleTask("mac/doc_textmarker_test.html", async (browser, accDoc) => {
+  await SpecialPowers.pushPrefEnv({
+    set: [
+      ["intl.icu4x.segmenter.enabled", true],
+      ["layout.word_select.stop_at_punctuation", true], // this is default
+    ],
+  });
+
+  const expectedValues = await SpecialPowers.spawn(browser, [], async () => {
+    return content.wrappedJSObject.getExpected(true, true);
  });

  testMarkerIntegrity(accDoc, expectedValues);
--- a/accessible/tests/browser/mac/doc_textmarker_test.html
+++ b/accessible/tests/browser/mac/doc_textmarker_test.html
@ -19,7 +19,7 @@
    <script>
      "use strict";
      // eslint-disable-next-line no-unused-vars
-      function getExpected(useNewSegmenter) {
+      function getExpected(useNewSegmenter, stopAtPunctuation) {
        return [
        { style: "Bob Loblaw Lobs Law Bomb",
          paragraph: "Bob Loblaw Lobs Law Bomb",
@ -1909,7 +1909,7 @@
          lines: ["Do not order the Skip's Scramble",
                  "Do not order the Skip's Scramble",
                  "Do not order the Skip's Scramble"],
-          words: useNewSegmenter ? [" ", "Skip's"] : [" ", "Skip'"],
+          words: useNewSegmenter && !stopAtPunctuation ? [" ", "Skip's"] : [" ", "Skip'"],
          element: ["AXStaticText",
                    "Do not order the Skip's Scramble",
                    "Do not order the Skip's Scramble"] },
@ -1918,7 +1918,7 @@
          lines: ["Do not order the Skip's Scramble",
                  "Do not order the Skip's Scramble",
                  "Do not order the Skip's Scramble"],
-          words: useNewSegmenter ? ["Skip's", "Skip's"] : ["Skip'", "Skip'"],
+          words: useNewSegmenter && !stopAtPunctuation ? ["Skip's", "Skip's"] : ["Skip'", "Skip'"],
          element: ["AXStaticText",
                    "Do not order the Skip's Scramble",
                    "Do not order the Skip's Scramble"] },
@ -1927,7 +1927,7 @@
          lines: ["Do not order the Skip's Scramble",
                  "Do not order the Skip's Scramble",
                  "Do not order the Skip's Scramble"],
-          words: useNewSegmenter ? ["Skip's", "Skip's"] : ["Skip'", "Skip'"],
+          words: useNewSegmenter && !stopAtPunctuation ? ["Skip's", "Skip's"] : ["Skip'", "Skip'"],
          element: ["AXStaticText",
                    "Do not order the Skip's Scramble",
                    "Do not order the Skip's Scramble"] },
@ -1936,7 +1936,7 @@
          lines: ["Do not order the Skip's Scramble",
                  "Do not order the Skip's Scramble",
                  "Do not order the Skip's Scramble"],
-          words: useNewSegmenter ? ["Skip's", "Skip's"] : ["Skip'", "Skip'"],
+          words: useNewSegmenter && !stopAtPunctuation ? ["Skip's", "Skip's"] : ["Skip'", "Skip'"],
          element: ["AXStaticText",
                    "Do not order the Skip's Scramble",
                    "Do not order the Skip's Scramble"] },
@ -1945,7 +1945,7 @@
          lines: ["Do not order the Skip's Scramble",
                  "Do not order the Skip's Scramble",
                  "Do not order the Skip's Scramble"],
-          words: useNewSegmenter ? ["Skip's", "Skip's"] : ["Skip'", "Skip'"],
+          words: useNewSegmenter && !stopAtPunctuation ? ["Skip's", "Skip's"] : ["Skip'", "Skip'"],
          element: ["AXStaticText",
                    "Do not order the Skip's Scramble",
                    "Do not order the Skip's Scramble"] },
@ -1954,7 +1954,7 @@
          lines: ["Do not order the Skip's Scramble",
                  "Do not order the Skip's Scramble",
                  "Do not order the Skip's Scramble"],
-          words: useNewSegmenter ? ["Skip's", "Skip's"] : ["Skip'", "s"],
+          words: useNewSegmenter && !stopAtPunctuation ? ["Skip's", "Skip's"] : ["Skip'", "s"],
          element: ["AXStaticText",
                    "Do not order the Skip's Scramble",
                    "Do not order the Skip's Scramble"] },
@ -1963,7 +1963,7 @@
          lines: ["Do not order the Skip's Scramble",
                  "Do not order the Skip's Scramble",
                  "Do not order the Skip's Scramble"],
-          words: useNewSegmenter ? ["Skip's", " "] : ["s", " "],
+          words: useNewSegmenter && !stopAtPunctuation ? ["Skip's", " "] : ["s", " "],
          element: ["AXStaticText",
                    "Do not order the Skip's Scramble",
                    "Do not order the Skip's Scramble"] },
--- a/accessible/tests/browser/text/browser_textleafpoint.js
+++ b/accessible/tests/browser/text/browser_textleafpoint.js
@ -483,3 +483,42 @@ addAccessibleTask(
  },
  { chrome: true, topLevel: true, iframe: false, remoteIframe: false }
 );
+
+/*
+ * Test the word boundary with punctuation character
+ */
+addAccessibleTask(
+  `
+<p>ab'cd</p>
+  `,
+  async function (browser, docAcc) {
+    const firstPoint = createTextLeafPoint(docAcc, 0);
+    const lastPoint = createTextLeafPoint(docAcc, kTextEndOffset);
+
+    const expectedWordStartSequence = [
+      ["ab'cd", 0],
+      ["ab'cd", 3],
+      ["ab'cd", 5],
+    ];
+    testBoundarySequence(
+      firstPoint,
+      BOUNDARY_WORD_START,
+      DIRECTION_NEXT,
+      expectedWordStartSequence,
+      "Forward BOUNDARY_WORD_START sequence is correct"
+    );
+    const expectedWordEndSequence = [
+      ["ab'cd", 5],
+      ["ab'cd", 3],
+      ["ab'cd", 0],
+    ];
+    testBoundarySequence(
+      lastPoint,
+      BOUNDARY_WORD_END,
+      DIRECTION_PREVIOUS,
+      expectedWordEndSequence,
+      "Backward BOUNDARY_WORD_END sequence is correct"
+    );
+  },
+  { chrome: true, topLevel: true, iframe: false, remoteIframe: false }
+);
--- a/accessible/tests/mochitest/text/test_words.html
+++ b/accessible/tests/mochitest/text/test_words.html
@ -19,8 +19,11 @@

    async function doTest() {
      test_common();
-      await test_per_segmenter(false);
-      await test_per_segmenter(true);
+      for (let newSegmenter of [true, false]) {
+        for (let stopAtPunctuation of [true, false]) {
+          await test_per_segmenter(newSegmenter, stopAtPunctuation);
+	}
+      }
      SimpleTest.finish();
    }

@ -83,14 +86,15 @@
      testWords("input_1", ["foo", "bar"]);
    }

-    async function test_per_segmenter(aNewSegmenter) {
+    async function test_per_segmenter(aNewSegmenter, aStopAtPunctuation) {
      // If aNewSegmenter is true, use UAX#14/#29 compatible segmenter.
      await SpecialPowers.pushPrefEnv({"set": [
        ["intl.icu4x.segmenter.enabled", aNewSegmenter],
+        ["layout.word_select.stop_at_punctuation", aStopAtPunctuation],
      ]});

      // "one.two"
-      if (aNewSegmenter) {
+      if (!aStopAtPunctuation) {
        testWordCount("div8", 1, kOk);
        testWordAt("div8", 0, "one.two", kOk);
      } else {
@ -100,16 +104,16 @@
      }

      // "3.1416"
-      testWords("div11", ["3.1416"], aNewSegmenter ? kOk : kTodo);
+      testWords("div11", ["3.1416"], !aStopAtPunctuation ? kOk : kTodo);

      // "4,261.01"
-      testWords("div12", ["4,261.01"], aNewSegmenter ? kOk: kTodo);
+      testWords("div12", ["4,261.01"], !aStopAtPunctuation ? kOk: kTodo);

      // "Peter's car"
-      testWords("div14", ["Peter's", "car"], aNewSegmenter ? kOk : kTodo);
+      testWords("div14", ["Peter's", "car"], !aStopAtPunctuation ? kOk : kTodo);

      // "N.A.T.O."
-      testWords("div15", ["N.A.T.O."], aNewSegmenter ? kOk : kTodo);
+      testWords("div15", ["N.A.T.O."], !aStopAtPunctuation ? kOk : kTodo);

      await SpecialPowers.popPrefEnv();
    }