Bug 450088 Line breaking regression (in Chinese and other languages) p=Jonathan Kew, r=masayuki, sr=roc

intl/lwbrk/src/jisx4501class.h

@@ -60,7 +60,7 @@ static const PRUint32 gLBClass00[32] = {
 0x77777777, // U+0090 - U+0097
 0x77777777, // U+0098 - U+009F
 0xAA9A9AAB, // U+00A0 - U+00A7
-0x77A9A77A, // U+00A8 - U+00AF
+0x77A9777A, // U+00A8 - U+00AF
 0xAAAAAAAA, // U+00B0 - U+00B7
 0xAAAAAAAA, // U+00B8 - U+00BF
 0x77777777, // U+00C0 - U+00C7
@@ -77,7 +77,7 @@ static const PRUint32 gLBClass20[32] = {
 0xB5555555, // U+2000 - U+2007
 0x77775555, // U+2008 - U+200F
 0x777277B7, // U+2010 - U+2017
-0x77777777, // U+2018 - U+201F
+0x77A777A7, // U+2018 - U+201F
 0xAAAA7777, // U+2020 - U+2027
 0xB7777777, // U+2028 - U+202F
 0x77744444, // U+2030 - U+2037

intl/lwbrk/src/nsJISx4501LineBreaker.cpp

@@ -307,7 +307,7 @@ static const PRUint16 gPair[MAX_CLASSES] = {
       18  0000 1111 1101 1111  = 0x0FDF
  COMPLEX  0000 1111 1100 0010  = 0x0FC2
       [c] 0000 1111 1111 1111  = 0x0FFF
-      [d] 0000 1111 1101 1111  = 0x0EDF
+      [d] 0000 1111 1101 1111  = 0x0FDF
       [e] 0000 1111 1111 1111  = 0x0FFF
 */
 
@@ -322,7 +322,7 @@ static const PRUint16 gPairConservative[MAX_CLASSES] = {
   0x0FDF,
   0x0FC2,
   0x0FFF,
-  0x0EDF,
+  0x0FDF,
   0x0FFF
 };
 
@@ -384,13 +384,19 @@ static const PRUint16 gPairConservative[MAX_CLASSES] = {
 #define U_AMPERSAND PRUnichar('&')
 #define U_SEMICOLON PRUnichar(';')
 #define U_BACKSLASH PRUnichar('\\')
+#define U_OPEN_SINGLE_QUOTE PRUnichar(0x2018)
+#define U_OPEN_DOUBLE_QUOTE PRUnichar(0x201C)
+#define U_OPEN_GUILLEMET    PRUnichar(0x00AB)
 
 #define NEED_CONTEXTUAL_ANALYSIS(c) (IS_HYPHEN(c) || \
                                      (c) == U_SLASH || \
                                      (c) == U_PERCENT || \
                                      (c) == U_AMPERSAND || \
                                      (c) == U_SEMICOLON || \
-                                     (c) == U_BACKSLASH)
+                                     (c) == U_BACKSLASH || \
+                                     (c) == U_OPEN_SINGLE_QUOTE || \
+                                     (c) == U_OPEN_DOUBLE_QUOTE || \
+                                     (c) == U_OPEN_GUILLEMET)
 
 #define IS_ASCII_DIGIT(u) (0x0030 <= (u) && (u) <= 0x0039)
 
@@ -719,6 +725,14 @@ ContextualAnalysis(PRUnichar prev, PRUnichar cur, PRUnichar next,
     if (!aState.UseConservativeBreaking(1) &&
         aState.HasCharacterAlready(U_EQUAL))
       return CLASS_CLOSE;
+  } else if (cur == U_OPEN_SINGLE_QUOTE ||
+             cur == U_OPEN_DOUBLE_QUOTE ||
+             cur == U_OPEN_GUILLEMET) {
+    // for CJK usage, we treat these as openers to allow a break before them,
+    // but otherwise treat them as normal characters because quote mark usage
+    // in various Western languages varies too much; see bug #450088 discussion.
+    if (!aState.UseConservativeBreaking() && IS_CJK_CHAR(next))
+      return CLASS_OPEN;
   } else {
     NS_ERROR("Forgot to handle the current character!");
   }

intl/lwbrk/tools/anzx4501.html

@@ -326,10 +326,10 @@ Analysis of JIS X 4051 to Unicode General Category Mapping
 <TD>157</TD>
 <TD></TD>
 <TD>33</TD>
-<TD>57</TD>
+<TD>56</TD>
 <TD>125</TD>
 <TD>3</TD>
-<TD BGCOLOR=white>394</TD>
+<TD BGCOLOR=white>393</TD>
 <TD></TD>
 <TD>19</TD>
 <TD></TD>
@@ -348,8 +348,8 @@ Analysis of JIS X 4051 to Unicode General Category Mapping
 <TD>4</TD>
 <TD>5</TD>
 <TD>2</TD>
-<TD>2</TD>
+<TD></TD>
-<TD>4</TD>
+<TD>5</TD>
 <TD>36</TD>
 <TD>4</TD>
 <TD></TD>
@@ -443,10 +443,10 @@ Analysis of JIS X 4051 to Unicode General Category Mapping
 <TD>2</TD>
 <TD></TD>
 <TD>6</TD>
-<TD>25</TD>
+<TD>26</TD>
 <TD>16</TD>
 <TD></TD>
-<TD BGCOLOR=white>50</TD>
+<TD BGCOLOR=white>51</TD>
 <TD></TD>
 <TD>1</TD>
 <TD></TD>
@@ -465,8 +465,8 @@ Analysis of JIS X 4051 to Unicode General Category Mapping
 <TD></TD>
 <TD></TD>
 <TD>3</TD>
-<TD>1</TD>
+<TD>3</TD>
-<TD>1</TD>
+<TD></TD>
 <TD>20</TD>
 <TD></TD>
 <TD>2</TD>
@@ -580,10 +580,10 @@ Analysis of JIS X 4051 to Unicode General Category Mapping
 <TD></TD>
 <TD>33</TD>
 <TD>10</TD>
-<TD>126</TD>
+<TD>127</TD>
 <TD></TD>
 <TD>7</TD>
-<TD>45</TD>
+<TD>44</TD>
 <TD>2</TD>
 <TD></TD>
 </TR>
@@ -610,10 +610,10 @@ Analysis of JIS X 4051 to Unicode General Category Mapping
 <TD>5</TD>
 <TD>12</TD>
 <TD></TD>
-<TD>104</TD>
+<TD>102</TD>
 <TD></TD>
 <TD></TD>
-<TD>5</TD>
+<TD>7</TD>
 <TD>3</TD>
 <TD></TD>
 </TR>

intl/lwbrk/tools/jisx4501class.txt

@@ -24,6 +24,7 @@
 00A5;;22
 00A9;;18
 00AA;;18
+00AB;;18
 00AC;;22
 00AE;;18
 00AF;;18
@@ -49,6 +50,8 @@
 2014;;7
 2015;;18
 2016;2017;18
+2019;;23
+201D;;23
 2018;201F;18
 2020;2023;18
 2024;2026;23

layout/reftests/line-breaking/quotationmarks-cjk-1-ref.html

@@ -0,0 +1,18 @@
+<html>
+<head>
+<style type="text/css"> p { margin: 5px 1em; width: 0; white-space: nowrap; } </style>
+</head>
+<body>
+
+<p>&#x5B57;<br>&#x2018;&#x5B57;&#x2019;<br>&#x5B57;<br>&#x201C;&#x5B57;&#x201D;<br>&#x5B57;</p>
+<p>&#x5B57;&#xFF0C;<br>&#x2018;&#x5B57;&#xFF0C;&#x2019;<br>&#x5B57;&#xFF0C;<br>&#x201C;&#x5B57;&#xFF0C;&#x201D;<br>&#x5B57;</p>
+<p>&#x5B57;&#xFF0E;<br>&#x2018;&#x5B57;&#xFF0E;&#x2019;<br>&#x5B57;&#xFF0E;<br>&#x201C;&#x5B57;&#xFF0E;&#x201D;<br>&#x5B57;</p>
+<p>&#x5B57;&#xFF1A;<br>&#x2018;&#x5B57;&#xFF1A;&#x2019;<br>&#x5B57;&#xFF1A;<br>&#x201C;&#x5B57;&#xFF1A;&#x201D;<br>&#x5B57;</p>
+<p>&#x5B57;&#xFF1B;<br>&#x2018;&#x5B57;&#xFF1B;&#x2019;<br>&#x5B57;&#xFF1B;<br>&#x201C;&#x5B57;&#xFF1B;&#x201D;<br>&#x5B57;</p>
+<p>&#x5B57;&#xFF01;<br>&#x2018;&#x5B57;&#xFF01;&#x2019;<br>&#x5B57;&#xFF01;<br>&#x201C;&#x5B57;&#xFF01;&#x201D;<br>&#x5B57;</p>
+<p>&#x5B57;&#xFF1F;<br>&#x2018;&#x5B57;&#xFF1F;&#x2019;<br>&#x5B57;&#xFF1F;<br>&#x201C;&#x5B57;&#xFF1F;&#x201D;<br>&#x5B57;</p>
+<p>&#x5B57;<br>&#x2018;&#xFF08;&#x5B57;&#xFF09;&#x2019;<br>&#x5B57;<br>&#x201C;&#xFF08;&#x5B57;&#xFF09;&#x201D;<br>&#x5B57;</p>
+<p>&#x5B57;<br>&#xFF08;&#x2018;&#x5B57;&#x2019;&#xFF09;<br>&#x5B57;<br>&#xFF08;&#x201C;&#x5B57;&#x201D;&#xFF09;<br>&#x5B57;</p>
+
+</body>
+</html>

layout/reftests/line-breaking/quotationmarks-cjk-1.html

@@ -0,0 +1,19 @@
+<html>
+<head>
+<meta content-type="text/html" charset="utf8">
+<style type="text/css"> p { margin: 5px 1em; width: 0; } </style>
+</head>
+<body>
+
+<p>&#x5B57;&#x2018;&#x5B57;&#x2019;&#x5B57;&#x201C;&#x5B57;&#x201D;&#x5B57;</p>
+<p>&#x5B57;&#xFF0C;&#x2018;&#x5B57;&#xFF0C;&#x2019;&#x5B57;&#xFF0C;&#x201C;&#x5B57;&#xFF0C;&#x201D;&#x5B57;</p>
+<p>&#x5B57;&#xFF0E;&#x2018;&#x5B57;&#xFF0E;&#x2019;&#x5B57;&#xFF0E;&#x201C;&#x5B57;&#xFF0E;&#x201D;&#x5B57;</p>
+<p>&#x5B57;&#xFF1A;&#x2018;&#x5B57;&#xFF1A;&#x2019;&#x5B57;&#xFF1A;&#x201C;&#x5B57;&#xFF1A;&#x201D;&#x5B57;</p>
+<p>&#x5B57;&#xFF1B;&#x2018;&#x5B57;&#xFF1B;&#x2019;&#x5B57;&#xFF1B;&#x201C;&#x5B57;&#xFF1B;&#x201D;&#x5B57;</p>
+<p>&#x5B57;&#xFF01;&#x2018;&#x5B57;&#xFF01;&#x2019;&#x5B57;&#xFF01;&#x201C;&#x5B57;&#xFF01;&#x201D;&#x5B57;</p>
+<p>&#x5B57;&#xFF1F;&#x2018;&#x5B57;&#xFF1F;&#x2019;&#x5B57;&#xFF1F;&#x201C;&#x5B57;&#xFF1F;&#x201D;&#x5B57;</p>
+<p>&#x5B57;&#x2018;&#xFF08;&#x5B57;&#xFF09;&#x2019;&#x5B57;&#x201C;&#xFF08;&#x5B57;&#xFF09;&#x201D;&#x5B57;</p>
+<p>&#x5B57;&#xFF08;&#x2018;&#x5B57;&#x2019;&#xFF09;&#x5B57;&#xFF08;&#x201C;&#x5B57;&#x201D;&#xFF09;&#x5B57;</p>
+
+</body>
+</html>

layout/reftests/line-breaking/reftest.list

@@ -12,6 +12,7 @@
 == numerics-1.html numerics-1-ref.html
 == parentheses-1.html parentheses-1-ref.html
 == quotationmarks-1.html quotationmarks-1-ref.html
+== quotationmarks-cjk-1.html quotationmarks-cjk-1-ref.html
 == smileys-1.html smileys-1-ref.html
 == smileys-2.html smileys-2-ref.html
 == url-1.html url-1-ref.html