bug 221024 : add a new option for turning invisible chars. to 'nothingness' to the transliterator (r=smontagu, sr=rbs)

gfx/src/gtk/nsFontMetricsGTK.cpp

@@ -2828,7 +2828,8 @@ nsFontGTKSubstitute::Convert(const PRUnichar* aSrc, PRUint32 aSrcLen,
     if (gFontSubConverter) {
       res = gFontSubConverter->Init("ISO-8859-1",
                              nsISaveAsCharset::attr_FallbackQuestionMark +
-                               nsISaveAsCharset::attr_EntityAfterCharsetConv,
+                               nsISaveAsCharset::attr_EntityAfterCharsetConv +
+                               nsISaveAsCharset::attr_IgnoreIgnorables, 
                              nsIEntityConverter::transliterate);
       if (NS_FAILED(res)) {
         NS_RELEASE(gFontSubConverter);

gfx/src/mac/nsUnicodeRenderingToolkit.cpp

@@ -699,7 +699,8 @@ PRBool nsUnicodeRenderingToolkit :: LoadTransliterator()
     {
        res = mTrans->Init("x-mac-roman",
                nsISaveAsCharset::attr_FallbackQuestionMark +
-               nsISaveAsCharset::attr_EntityBeforeCharsetConv,
+               nsISaveAsCharset::attr_EntityBeforeCharsetConv +
+               nsISaveAsCharset::attr_IgnoreIgnorables,
                nsIEntityConverter::transliterate);
       NS_ASSERTION(NS_SUCCEEDED(res), "cannot init the converter");
       if (NS_FAILED(res)) 

gfx/src/windows/nsFontMetricsWin.cpp

@@ -4405,7 +4405,7 @@ SubstituteChars(PRBool              aDisplayUnicode,
       res = gFontSubstituteConverter->Init("ISO-8859-1",
               aDisplayUnicode
               ? nsISaveAsCharset::attr_FallbackHexNCR
-              : nsISaveAsCharset::attr_EntityAfterCharsetConv + nsISaveAsCharset::attr_FallbackQuestionMark,
+              : nsISaveAsCharset::attr_EntityAfterCharsetConv + nsISaveAsCharset::attr_FallbackQuestionMark + nsISaveAsCharset::attr_IgnoreIgnorables,
               nsIEntityConverter::transliterate);
       if (NS_FAILED(res)) {
         NS_RELEASE(gFontSubstituteConverter);

gfx/src/xlib/nsFontMetricsXlib.cpp

@@ -3222,7 +3222,8 @@ nsFontXlibSubstitute::Convert(const PRUnichar* aSrc, PRUint32 aSrcLen,
     if (mFontMetricsContext->mFontSubConverter) {
       res = mFontMetricsContext->mFontSubConverter->Init("ISO-8859-1",
                              nsISaveAsCharset::attr_FallbackQuestionMark +
-                               nsISaveAsCharset::attr_EntityAfterCharsetConv,
+                               nsISaveAsCharset::attr_EntityAfterCharsetConv +
+                               nsISaveAsCharset::attr_IgnoreIgnorables,
                              nsIEntityConverter::transliterate);
       if (NS_FAILED(res))
         mFontMetricsContext->mFontSubConverter = nsnull; // destroy converter

intl/unicharutil/idl/nsISaveAsCharset.idl

@@ -51,6 +51,7 @@ interface nsISaveAsCharset : nsISupports
   const unsigned long mask_Fallback                = 0x000000FF; // mask for fallback (8bits)
   const unsigned long mask_Entity                  = 0x00000300; // mask for entity (2bits)
   const unsigned long mask_CharsetFallback         = 0x00000400; // mask for charset fallback (1bit)
+  const unsigned long mask_IgnorableFallback       = 0x00000800; // mask for ignorable code points (1bit)
 
   const unsigned long attr_FallbackNone = 0;                   // no fall back for unconverted chars (skipped)
   const unsigned long attr_FallbackQuestionMark = 1;           // unconverted chars are replaced by '?'
@@ -64,6 +65,8 @@ interface nsISaveAsCharset : nsISupports
 
   const unsigned long attr_CharsetFallback        =  0x00000400; // fallback to other charset and restart the convesion
 
+  const unsigned long attr_IgnoreIgnorables       =  0x00000800; // turn default ignorable code points (and Hebrew/Arabic/Syriac Vowel signs to nothingness
+
                                                                // default attribute for plain text
   const unsigned long attr_plainTextDefault = attr_FallbackNone + attr_EntityNone;               
   

intl/unicharutil/src/nsSaveAsCharset.cpp

@@ -46,12 +46,20 @@
 #include "nsSaveAsCharset.h"
 #include "nsCRT.h"
 #include "nsUnicharUtils.h"
+#include "nsCompressedCharMap.h"
 
 //
 // nsISupports methods
 //
 NS_IMPL_ISUPPORTS1(nsSaveAsCharset, nsISaveAsCharset)
 
+const static PRUint16 gIgnorableCCMapExtRaw[] =
+{
+#include "ignorables_abjadpoints.x-ccmap"
+};
+
+// 1st Two elements of ext. ccmap need to have negative indices.
+const static PRUint16 *gIgnorableCCMapExt = gIgnorableCCMapExtRaw + 2; 
 
 //
 // nsSaveAsCharset
@@ -258,6 +266,11 @@ nsSaveAsCharset::DoCharsetConversion(const PRUnichar *inString, char **outString
         unMappedChar = inString[pos1-1];
       }
 
+      // if we're asked to ignore default ignorable code points, skip them.
+      if (MASK_IGNORABLE_FALLBACK(mAttribute) &&
+          CCMAP_HAS_CHAR_EXT(gIgnorableCCMapExt, unMappedChar)) 
+				continue;
+
       rv = mEncoder->GetMaxLength(inString+pos1, inStringLength-pos1, &dstLength);
       if (NS_FAILED(rv)) 
         break;

intl/unicharutil/src/nsSaveAsCharset.h

@@ -46,6 +46,7 @@
 #define MASK_FALLBACK(a) (nsISaveAsCharset::mask_Fallback & (a))
 #define MASK_ENTITY(a) (nsISaveAsCharset::mask_Entity & (a))
 #define MASK_CHARSET_FALLBACK(a) (nsISaveAsCharset::mask_CharsetFallback & (a))
+#define MASK_IGNORABLE_FALLBACK(a) (nsISaveAsCharset::mask_IgnorableFallback & (a))
 #define ATTR_NO_FALLBACK(a) (nsISaveAsCharset::attr_FallbackNone == MASK_FALLBACK(a) && \
                              nsISaveAsCharset::attr_EntityAfterCharsetConv != MASK_ENTITY(a))
 
@@ -73,12 +74,13 @@ public:
   NS_IMETHOD Init(const char *charset, PRUint32 attr, PRUint32 entityVersion);
 
   NS_IMETHOD Convert(const PRUnichar *inString, char **_retval);
+  NS_IMETHOD Convert2(const PRUnichar *inString, PRUint32* outLen, char **_retval);
 
   NS_IMETHODIMP GetCharset(char * *aCharset);
 
 protected:
 
-  NS_IMETHOD DoCharsetConversion(const PRUnichar *inString, char **outString);
+  NS_IMETHOD DoCharsetConversion(const PRUnichar *inString, PRUint32* outLen, char **outString);
 
   NS_IMETHOD DoConversionFallBack(PRUint32 inUCS4, char *outString, PRInt32 bufferLength);