Added a new param to the HTML->Text sink stream to take the charset so that we can convert to plain text with the correct charset for the platform. Enhanced the XIFFormatConverter to take advantage of this, fixing bug 21414. r=akkana, nhotta.

content/base/src/nsDocumentEncoder.cpp

@@ -195,7 +195,7 @@ nsTextEncoder::EncodeToString(nsString& aOutputString)
         rv = NS_New_HTML_ContentSinkStream(&sink, &aOutputString, mFlags);
 
       else  // default to text/plain
-        rv = NS_New_HTMLToTXT_SinkStream(&sink, &aOutputString,
+        rv = NS_New_HTMLToTXT_SinkStream(&sink, &aOutputString, nsnull,
                                          mWrapColumn, mFlags);
 
       if (sink && NS_SUCCEEDED(rv))

htmlparser/src/nsHTMLToTXTSinkStream.h

@@ -224,6 +224,7 @@ NS_New_HTMLToTXT_SinkStream(nsIHTMLContentSink** aInstancePtrResult,
 inline nsresult
 NS_New_HTMLToTXT_SinkStream(nsIHTMLContentSink** aInstancePtrResult, 
                             nsString* aOutString,
+                            const nsString* aCharsetOverride=nsnull,
                             PRUint32 aWrapColumn=0, PRUint32 aFlags=0)
 {
   nsCOMPtr<nsIHTMLToTXTSinkStream> it;
@@ -238,9 +239,12 @@ NS_New_HTMLToTXT_SinkStream(nsIHTMLContentSink** aInstancePtrResult,
 
     if (NS_SUCCEEDED(rv)) {
       it->SetWrapColumn(aWrapColumn);
+      if ( aCharsetOverride )
+        it->SetCharsetOverride(aCharsetOverride);
+      else {
         nsAutoString ucs2("ucs2");
         it->SetCharsetOverride(&ucs2);
-
+      }
       rv = it->QueryInterface(nsIHTMLContentSink::GetIID(),
                               (void**)aInstancePtrResult);
     }

layout/base/src/nsDocumentEncoder.cpp

@@ -195,7 +195,7 @@ nsTextEncoder::EncodeToString(nsString& aOutputString)
         rv = NS_New_HTML_ContentSinkStream(&sink, &aOutputString, mFlags);
 
       else  // default to text/plain
-        rv = NS_New_HTMLToTXT_SinkStream(&sink, &aOutputString,
+        rv = NS_New_HTMLToTXT_SinkStream(&sink, &aOutputString, nsnull,
                                          mWrapColumn, mFlags);
 
       if (sink && NS_SUCCEEDED(rv))

mailnews/compose/src/nsMsgCompUtils.cpp

@@ -1936,7 +1936,8 @@ ConvertBufToPlainText(nsString &aConBuf, const char *charSet)
         converterFlags |= nsIDocumentEncoder::OutputFormatFlowed;
     }    
     
-    rv = NS_New_HTMLToTXT_SinkStream((nsIHTMLContentSink **)&sink, &convertedText, wrapWidth, converterFlags);
+    nsAutoString charSetStr(charSet);
+    rv = NS_New_HTMLToTXT_SinkStream((nsIHTMLContentSink **)&sink, &convertedText, &charSetStr, wrapWidth, converterFlags);
     if (sink && NS_SUCCEEDED(rv)) 
     {  
         sink->DoFragment(PR_TRUE);

parser/htmlparser/src/nsHTMLToTXTSinkStream.h

@@ -224,6 +224,7 @@ NS_New_HTMLToTXT_SinkStream(nsIHTMLContentSink** aInstancePtrResult,
 inline nsresult
 NS_New_HTMLToTXT_SinkStream(nsIHTMLContentSink** aInstancePtrResult, 
                             nsString* aOutString,
+                            const nsString* aCharsetOverride=nsnull,
                             PRUint32 aWrapColumn=0, PRUint32 aFlags=0)
 {
   nsCOMPtr<nsIHTMLToTXTSinkStream> it;
@@ -238,9 +239,12 @@ NS_New_HTMLToTXT_SinkStream(nsIHTMLContentSink** aInstancePtrResult,
 
     if (NS_SUCCEEDED(rv)) {
       it->SetWrapColumn(aWrapColumn);
+      if ( aCharsetOverride )
+        it->SetCharsetOverride(aCharsetOverride);
+      else {
         nsAutoString ucs2("ucs2");
         it->SetCharsetOverride(&ucs2);
-
+      }
       rv = it->QueryInterface(nsIHTMLContentSink::GetIID(),
                               (void**)aInstancePtrResult);
     }

widget/src/xpwidgets/nsXIFFormatConverter.cpp

@@ -50,6 +50,12 @@
 #include "nsWidgetsCID.h"
 #include "nsXIFFormatConverter.h"
 
+// unicode conversion
+#define NS_IMPL_IDS
+#include "nsIPlatformCharset.h"
+#undef NS_IMPL_IDS
+#include "nsISaveAsCharset.h"
+  
 
 static NS_DEFINE_IID(kCParserCID, NS_PARSER_IID);
 
@@ -303,9 +309,20 @@ nsXIFFormatConverter::Convert(const char *aFromDataFlavor, nsISupports *aFromDat
 NS_IMETHODIMP
 nsXIFFormatConverter::ConvertFromXIFToText(const nsAutoString & aFromStr, nsAutoString & aToStr)
 {
+  // we may cache it since the platform charset will not change through application life
+  nsAutoString aCharset;
+  nsCOMPtr <nsIPlatformCharset> platformCharset;
+  nsresult rv = nsComponentManager::CreateInstance(NS_PLATFORMCHARSET_PROGID, nsnull, 
+                                                     NS_GET_IID(nsIPlatformCharset), 
+                                                     getter_AddRefs(platformCharset));
+  if (NS_SUCCEEDED(rv))
+    rv = platformCharset->GetCharset(kPlatformCharsetSel_PlainTextInClipboard, aCharset);
+  if (NS_FAILED(rv))
+    aCharset.SetString("ISO-8859-1");
+ 
   aToStr = "";
   nsCOMPtr<nsIParser> parser;
-  nsresult rv = nsComponentManager::CreateInstance(kCParserCID, 
+  rv = nsComponentManager::CreateInstance(kCParserCID, 
                                            nsnull, 
                                            nsIParser::GetIID(), 
                                            getter_AddRefs(parser));
@@ -313,7 +330,7 @@ nsXIFFormatConverter::ConvertFromXIFToText(const nsAutoString & aFromStr, nsAuto
     return rv;
 
   nsCOMPtr<nsIHTMLContentSink> sink;
-  rv = NS_New_HTMLToTXT_SinkStream(getter_AddRefs(sink),&aToStr,0);
+  rv = NS_New_HTMLToTXT_SinkStream(getter_AddRefs(sink),&aToStr,&aCharset);
   if ( sink ) {
     parser->SetContentSink(sink);