Bug 1054252 - wrong values in From and Recipient columns in message list due to improper RFC 2047 decoding / parsing order r=kent

--HG--
extra : amend_source : 2eb22afbe1197e20c16d449dd5b6063826cba264
extra : histedit_source : 3aeba052d346275a3a619dc2375a33f7a06ff2d1

mailnews/base/public/nsIMsgHdr.idl

@@ -10,7 +10,7 @@
 interface nsIMsgFolder;
 interface nsIUTF8StringEnumerator;
 
-[scriptable, uuid(ec1053a9-1a3d-4572-8f47-c103f2c014fb)]
+[scriptable, uuid(3c11ddbe-c805-40c5-b9c9-d065fad5d0be)]
 interface nsIMsgDBHdr : nsISupports
 {
     /* general property routines - I think this can retrieve any
@@ -86,6 +86,13 @@ interface nsIMsgDBHdr : nsISupports
                                   [array, size_is(aCount)] out octet aKey);
 
     attribute string Charset;
+    /**
+     * Returns the effective character set for the message (@ref Charset).
+     * If there is no specific set defined for the message or the
+     * characterSetOverride option is turned on for the folder it will return
+     * the effective character set of the folder instead.
+     */
+    readonly attribute ACString effectiveCharset;
     attribute nsMsgLabelValue label;
     attribute string accountKey;
     readonly attribute nsIMsgFolder folder;

mailnews/base/src/nsMsgDBView.cpp

@@ -394,12 +394,16 @@ nsresult nsMsgDBView::FetchAuthor(nsIMsgDBHdr * aHdr, nsAString &aSenderString)
     }
   }
 
-  nsString author;
-  nsresult rv = aHdr->GetMime2DecodedAuthor(author);
+  nsCString author;
+  nsresult rv = aHdr->GetAuthor(getter_Copies(author));
+
+  nsCString headerCharset;
+  aHdr->GetEffectiveCharset(headerCharset);
 
   nsCString emailAddress;
   nsString name;
-  ExtractFirstAddress(DecodedHeader(author), name, emailAddress);
+  ExtractFirstAddress(EncodedHeader(author, headerCharset.get()), name,
+    emailAddress);
 
   if (showCondensedAddresses)
     GetDisplayNameInAddressBook(emailAddress, aSenderString);
@@ -454,7 +458,6 @@ nsresult nsMsgDBView::FetchAccount(nsIMsgDBHdr * aHdr, nsAString& aAccount)
 
 nsresult nsMsgDBView::FetchRecipients(nsIMsgDBHdr * aHdr, nsAString &aRecipientsString)
 {
-  nsString unparsedRecipients;
   nsCString recipients;
   int32_t currentDisplayNameVersion = 0;
   bool showCondensedAddresses = false;
@@ -480,11 +483,16 @@ nsresult nsMsgDBView::FetchRecipients(nsIMsgDBHdr * aHdr, nsAString &aRecipients
     }
   }
 
-  nsresult rv = aHdr->GetMime2DecodedRecipients(unparsedRecipients);
+  nsCString unparsedRecipients;
+  nsresult rv = aHdr->GetRecipients(getter_Copies(unparsedRecipients));
+
+  nsCString headerCharset;
+  aHdr->GetEffectiveCharset(headerCharset);
+
   nsTArray<nsString> names;
   nsTArray<nsCString> emails;
-  ExtractAllAddresses(DecodedHeader(unparsedRecipients), names,
-    UTF16ArrayAdapter<>(emails));
+  ExtractAllAddresses(EncodedHeader(unparsedRecipients, headerCharset.get()),
+    names, UTF16ArrayAdapter<>(emails));
 
   uint32_t numAddresses = names.Length();
 

mailnews/base/test/unit/test_nsMsgDBView_headerValues.js

@@ -0,0 +1,71 @@
+/* Test that nsMsgDBView properly reports the values of messages in the display.
+ */
+load("../../../resources/logHelper.js");
+load("../../../resources/asyncTestUtils.js");
+load("../../../resources/abSetup.js");
+
+load("../../../resources/messageGenerator.js");
+load("../../../resources/messageModifier.js");
+load("../../../resources/messageInjection.js");
+
+var ViewType = Components.interfaces.nsMsgViewType;
+var SortType = Components.interfaces.nsMsgViewSortType;
+var SortOrder = Components.interfaces.nsMsgViewSortOrder;
+
+// This is an array of the actual test data. Each test datum is an array of two
+// elements: the first element is the argument into a simple message generator,
+// and the second element is a map of column names to expected values when
+// requesting the cell text for a given column name.
+let tests = [
+  [{from: "John Doe <db@tinderbox.invalid>"}, {sender: "John Doe"}],
+  [{from: "\"Doe, John\" <db@tinderbox.invalid>"}, {sender: "Doe, John"}],
+  [{from: "John Doe <db@tinderbox.invalid>, Sally Ann <db@null.invalid>"},
+    {sender: "John Doe"}],
+  [{from: "=?UTF-8?Q?David_H=C3=A5s=C3=A4ther?= <db@null.invalid>"},
+    {sender: "David Håsäther"}],
+  [{from: "=?UTF-8?Q?H=C3=A5s=C3=A4ther=2C_David?= <db@null.invalid>"},
+    {sender: "Håsäther, David"}],
+  [{from: "John Doe \xF5  <db@null.invalid>",
+     clobberHeaders: { "Content-type" : "text/plain; charset=ISO-8859-1" }},
+	{sender: "John Doe õ"}],
+  [{from: "John Doe \xF5 <db@null.invalid>",
+     clobberHeaders: { "Content-type" : "text/plain; charset=ISO-8859-2" }},
+	{sender: "John Doe ő"}],
+  [{from: "=?UTF-8?Q?H=C3=A5s=C3=A4ther=2C_David?= <db@null.invalid>",
+     clobberHeaders: { "Content-type" : "text/plain; charset=ISO-8859-2" }},
+    {sender: "Håsäther, David"}],
+];
+
+function real_test() {
+  // Add the messages to the folder
+  let msgGenerator = new MessageGenerator();
+  let genMessages = tests.map(data => msgGenerator.makeMessage(data[0]));
+  let folder = make_empty_folder();
+  yield add_sets_to_folder(folder, [new SyntheticMessageSet(genMessages)]);
+
+  // Make the DB view
+  let dbviewContractId = "@mozilla.org/messenger/msgdbview;1?type=threaded";
+  let dbView = Components.classes[dbviewContractId]
+                         .createInstance(Components.interfaces.nsIMsgDBView);
+  dbView.init(null, null, null);
+  var outCount = {};
+  dbView.open(folder, SortType.byDate, SortOrder.ascending, 0, outCount);
+
+  // Did we add all the messages properly?
+  let treeView = dbView.QueryInterface(Components.interfaces.nsITreeView);
+  do_check_eq(treeView.rowCount, tests.length);
+
+  // For each test, make sure that the display is correct.
+  tests.forEach(function (data, i) {
+    do_print("Checking data for " + uneval(data));
+    let expected = data[1];
+    for (let column in expected) {
+      do_check_eq(dbView.cellTextForColumn(i, column), expected[column]);
+    }
+  });
+}
+
+function run_test() {
+  configure_message_injection({mode: "local"});
+  async_run_tests([real_test]);
+}

mailnews/base/test/unit/xpcshell.ini

@@ -49,6 +49,7 @@ skip-if = os != 'mac'
 [test_nsIMsgTagService.js]
 [test_nsMailDirProvider.js]
 [test_nsMsgDBView.js]
+[test_nsMsgDBView_headerValues.js]
 [test_nsMsgMailSession_Alerts.js]
 [test_nsMsgMailSession_Listeners.js]
 [test_nsMsgTraitService.js]

mailnews/db/msgdb/public/nsMsgDatabase.h

@@ -194,6 +194,8 @@ public:
   nsresult RowCellColumnToConstCharPtr(nsIMdbRow *row, mdb_token columnToken, const char **ptr);
   nsresult RowCellColumnToAddressCollationKey(nsIMdbRow *row, mdb_token colToken, uint8_t **result, uint32_t *len);
 
+  nsresult GetEffectiveCharset(nsIMdbRow *row, nsACString &resultCharset);
+
   // these methods take the property name as a string, not a token.
   // they should be used when the properties aren't accessed a lot
   nsresult        GetProperty(nsIMdbRow *row, const char *propertyName, char **result);

mailnews/db/msgdb/src/nsMsgDatabase.cpp

@@ -3596,6 +3596,20 @@ nsIMimeConverter *nsMsgDatabase::GetMimeConverter()
   return m_mimeConverter;
 }
 
+nsresult nsMsgDatabase::GetEffectiveCharset(nsIMdbRow *row, nsACString &resultCharset)
+{
+  resultCharset.Truncate();
+  bool characterSetOverride;
+  m_dbFolderInfo->GetCharacterSetOverride(&characterSetOverride);
+  nsresult rv = RowCellColumnToCharPtr(row, m_messageCharSetColumnToken, getter_Copies(resultCharset));
+  if (NS_FAILED(rv) || resultCharset.IsEmpty() ||
+      resultCharset.Equals("us-ascii") || characterSetOverride)
+  {
+    rv = m_dbFolderInfo->GetEffectiveCharacterSet(resultCharset);
+  }
+  return rv;
+}
+
 nsresult nsMsgDatabase::RowCellColumnToMime2DecodedString(nsIMdbRow *row, mdb_token columnToken, nsAString &resultStr)
 {
   nsresult err = NS_OK;
@@ -3608,17 +3622,10 @@ nsresult nsMsgDatabase::RowCellColumnToMime2DecodedString(nsIMdbRow *row, mdb_to
     {
       nsAutoString decodedStr;
       nsCString charSet;
-      bool characterSetOverride;
-      m_dbFolderInfo->GetCharacterSetOverride(&characterSetOverride);
-      err = RowCellColumnToCharPtr(row, m_messageCharSetColumnToken, getter_Copies(charSet));
-      if (NS_FAILED(err) || charSet.IsEmpty() || charSet.Equals("us-ascii") ||
-          characterSetOverride)
-      {
-        m_dbFolderInfo->GetEffectiveCharacterSet(charSet);
-      }
+      GetEffectiveCharset(row, charSet);
 
       err = m_mimeConverter->DecodeMimeHeader(nakedString, charSet.get(),
-        characterSetOverride, true, resultStr);
+        false, true, resultStr);
     }
   }
   return err;
@@ -3682,17 +3689,10 @@ nsresult nsMsgDatabase::RowCellColumnToCollationKey(nsIMdbRow *row, mdb_token co
     {
       nsCString decodedStr;
       nsCString charSet;
-      bool characterSetOverride;
-      m_dbFolderInfo->GetCharacterSetOverride(&characterSetOverride);
-      err = RowCellColumnToCharPtr(row, m_messageCharSetColumnToken, getter_Copies(charSet));
-      if (NS_FAILED(err) || charSet.IsEmpty() || charSet.Equals("us-ascii") ||
-          characterSetOverride)
-      {
-        m_dbFolderInfo->GetEffectiveCharacterSet(charSet);
-      }
+      GetEffectiveCharset(row, charSet);
 
       err = m_mimeConverter->DecodeMimeHeaderToUTF8(
-        nsDependentCString(nakedString), charSet.get(), characterSetOverride,
+        nsDependentCString(nakedString), charSet.get(), false,
         true, decodedStr);
       if (NS_SUCCEEDED(err))
         err = CreateCollationKey(NS_ConvertUTF8toUTF16(decodedStr), len, result);

mailnews/db/msgdb/src/nsMsgHdr.cpp

@@ -626,6 +626,11 @@ NS_IMETHODIMP nsMsgHdr::SetCharset(const char *aCharset)
   return SetStringColumn(aCharset, m_mdb->m_messageCharSetColumnToken);
 }
 
+NS_IMETHODIMP nsMsgHdr::GetEffectiveCharset(nsACString &resultCharset)
+{
+  return m_mdb->GetEffectiveCharset(m_mdbRow, resultCharset);
+}
+
 NS_IMETHODIMP nsMsgHdr::SetThreadParent(nsMsgKey inKey)
 {
   m_threadParent = inKey;