Update transliterate.properties to Unicode 4.1.0 database. Bug 288137, r=jshin, sr=rbs

content/base/src/nsHTMLContentSerializer.cpp

@@ -935,6 +935,8 @@ nsHTMLContentSerializer::AppendToString(const nsAString& aStr,
            iter != done_reading; 
            iter.advance(PRInt32(advanceLength))) {
         PRUint32 fragmentLength = iter.size_forward();
+        PRUint32 lengthReplaced = 0; // the number of UTF-16 codepoints
+                                     //  replaced by a particular entity
         const PRUnichar* c = iter.get();
         const PRUnichar* fragmentStart = c;
         const PRUnichar* fragmentEnd = c + fragmentLength;
@@ -962,16 +964,33 @@ nsHTMLContentSerializer::AppendToString(const nsAString& aStr,
 
             if (!entityReplacement.IsEmpty()) {
               entityText = entityReplacement.get();
+              lengthReplaced = 1;
               break;
             }
           }
           else if (val > 127 && 
                    mFlags & nsIDocumentEncoder::OutputEncodeW3CEntities &&
-                   mEntityConverter &&
+                   mEntityConverter) {
-                   NS_SUCCEEDED(mEntityConverter->ConvertToEntity(val,
+            if (IS_HIGH_SURROGATE(val) &&
+                c + 1 < fragmentEnd &&
+                IS_LOW_SURROGATE(*(c + 1))) {
+              PRUint32 valUTF32 = SURROGATE_TO_UCS4(val, *(++c));
+              if (NS_SUCCEEDED(mEntityConverter->ConvertUTF32ToEntity(valUTF32,
                                nsIEntityConverter::entityW3C, &fullEntityText))) {
+                lengthReplaced = 2;
                 break;
               }
+              else {
+                advanceLength++;
+              }
+            }
+            else if (NS_SUCCEEDED(mEntityConverter->ConvertToEntity(val,
+                                  nsIEntityConverter::entityW3C, 
+                                  &fullEntityText))) {
+              lengthReplaced = 1;
+              break;
+            }
+          }
         }
 
         aOutputStr.Append(fragmentStart, advanceLength);
@@ -979,13 +998,13 @@ nsHTMLContentSerializer::AppendToString(const nsAString& aStr,
           aOutputStr.Append(PRUnichar('&'));
           AppendASCIItoUTF16(entityText, aOutputStr);
           aOutputStr.Append(PRUnichar(';'));
-          advanceLength++;
+          advanceLength += lengthReplaced;
         }
         // if it comes from nsIEntityConverter, it already has '&' and ';'
         else if (fullEntityText) {
           AppendASCIItoUTF16(fullEntityText, aOutputStr);
           nsMemory::Free(fullEntityText);
-          advanceLength++;
+          advanceLength += lengthReplaced;
         }
       }
     } else {

intl/unicharutil/idl/nsIEntityConverter.idl

@@ -38,8 +38,8 @@
 #include "nsISupports.idl"
 
 %{C++
-// {8C4506A1-55E6-11d3-91D9-00105AA3F7DC}
+// {9E9B565A-8E52-4C1A-8805-B2B91655F874}
-#define NS_ENTITYCONVERTER_CID { 0x8c4506a1, 0x55e6, 0x11d3, {0x91, 0xd9, 0x0, 0x10, 0x5a, 0xa3, 0xf7, 0xdc}}
+#define NS_ENTITYCONVERTER_CID { 0x9e9b565a, 0x8e52, 0x4c1a, {0x88, 0x05, 0xb2, 0xb9, 0x16, 0x55, 0xf8, 0x74}}
 #define NS_ENTITYCONVERTER_CONTRACTID "@mozilla.org/intl/entityconverter;1"
 %}
 
@@ -56,6 +56,7 @@ interface nsIEntityConverter : nsISupports
   const unsigned long html40 = html40Latin1+html40Symbols+html40Special;
   const unsigned long entityW3C = html40+mathml20;
 
+  string ConvertUTF32ToEntity(in unsigned long character, in unsigned long entityVersion);
   string ConvertToEntity(in wchar character, in unsigned long entityVersion);
 
   wstring ConvertToEntities(in wstring inString, in unsigned long entityVersion);

intl/unicharutil/src/nsEntityConverter.cpp

@@ -44,6 +44,7 @@
 #include "nsLiteralString.h"
 #include "nsXPIDLString.h"
 #include "nsString.h"
+#include "nsUnicharUtils.h"
 
 //
 // implementation methods
@@ -179,6 +180,12 @@ NS_IMPL_ISUPPORTS1(nsEntityConverter,nsIEntityConverter)
 //
 NS_IMETHODIMP
 nsEntityConverter::ConvertToEntity(PRUnichar character, PRUint32 entityVersion, char **_retval)
+{ 
+  return ConvertUTF32ToEntity((PRUint32)character, entityVersion, _retval);
+}
+
+NS_IMETHODIMP
+nsEntityConverter::ConvertUTF32ToEntity(PRUint32 character, PRUint32 entityVersion, char **_retval)
 {
   NS_ASSERTION(_retval, "null ptr- _retval");
   if(nsnull == _retval)
@@ -226,7 +233,14 @@ nsEntityConverter::ConvertToEntities(const PRUnichar *inString, PRUint32 entityV
   PRUint32 len = nsCRT::strlen(inString);
   for (PRUint32 i = 0; i < len; i++) {
     nsAutoString key(NS_LITERAL_STRING("entity."));
+    if (IS_HIGH_SURROGATE(inString[i]) &&
+        i + 2 < len &&
+        IS_LOW_SURROGATE(inString[i + 1])) {
+      key.AppendInt(SURROGATE_TO_UCS4(inString[i], inString[++i]), 10);
+    }
+    else {
       key.AppendInt(inString[i],10);
+    }
     
     nsXPIDLString value;
     

intl/unicharutil/src/nsEntityConverter.h

@@ -72,6 +72,7 @@ public:
 	//
 	// nsIEntityConverter
 	//
+	NS_IMETHOD ConvertUTF32ToEntity(PRUint32 character, PRUint32 entityVersion, char **_retval);
 	NS_IMETHOD ConvertToEntity(PRUnichar character, PRUint32 entityVersion, char **_retval);
 
 	NS_IMETHOD ConvertToEntities(const PRUnichar *inString, PRUint32 entityVersion, PRUnichar **_retval);

intl/unicharutil/src/nsSaveAsCharset.cpp

@@ -315,10 +315,9 @@ nsSaveAsCharset::DoConversionFallBack(PRUint32 inUCS4, char *outString, PRInt32
   if (ATTR_NO_FALLBACK(mAttribute)) {
     return NS_OK;
   }
-  if (attr_EntityAfterCharsetConv == MASK_ENTITY(mAttribute) && 
+  if (attr_EntityAfterCharsetConv == MASK_ENTITY(mAttribute)) {
-      !(inUCS4 & 0xff0000) ) {
     char *entity = NULL;
-    rv = mEntityConverter->ConvertToEntity((PRUnichar)inUCS4, mEntityVersion, &entity);
+    rv = mEntityConverter->ConvertUTF32ToEntity(inUCS4, mEntityVersion, &entity);
     if (NS_SUCCEEDED(rv)) {
       if (NULL == entity || (PRInt32)strlen(entity) > bufferLength) {
         return NS_ERROR_OUT_OF_MEMORY;

intl/unicharutil/tools/gentransliterate.pl

@@ -114,9 +114,9 @@ entity.8221="
 # BULLET
 entity.8226=.
 # EN DASH
-entity.8211=-
+entity.8211=--
 # EM DASH
-entity.8212=--
+entity.8212=---
 # SMALL TILDE
 entity.732=~
 # SINGLE RIGHT-POINTING ANGLE QUOTATION MARK
@@ -124,9 +124,9 @@ entity.8250=>
 # LATIN SMALL LIGATURE OE
 entity.339=oe
 # U+2000 EN QUAD
-entity.8192=\u0020
+entity.8192=\\u0020
 # U+2001 EM QUAD
-entity.8193=\u0020
+entity.8193=\\u0020
 # U+2010 HYPHEN
 entity.8208=-
 # U+2011 NON-BREAKING HYPHEN
@@ -167,7 +167,7 @@ sub FromLatinComment
 {
   my ($cmt) = (@_);
   $char = "";
-  if($cmt =~ /PRESEDED BY APOSTROPHE/) {
+  if($cmt =~ /PRECEDED BY APOSTROPHE/) {
       $char = "\'";
   }
   if($cmt =~ /CAPITAL LETTER ([A-Z]*)/) {