Bug 999881, part 2: Replace charsetOverlay.xul in the compose window, r=mconley, a=jcranmer.

--HG--
extra : rebase_source : 1145a0b70cc552b8228dc47859ef9d4d71c64871
extra : amend_source : e4cef33f4c44f5661e322e517f925534023b9e8b

mail/components/compose/content/MsgComposeCommands.js

@@ -1753,6 +1753,17 @@ function GetArgs(originalData)
 
 function ComposeFieldsReady()
 {
+  // Limit the charsets to those we think are safe to encode (i.e., they are in
+  // the charset menu). Easiest way to normalize this is to use the TextDecoder
+  // to get the canonical alias and default if it isn't valid.
+  let charset;
+  try {
+    charset = new TextDecoder(gMsgCompose.compFields.characterSet).encoding;
+  } catch (e) {
+    charset = gMsgCompose.compFields.defaultCharacterSet;
+  }
+  SetDocumentCharacterSet(charset);
+
   //If we are in plain text, we need to set the wrap column
   if (! gMsgCompose.composeHTML) {
     try {
@@ -2427,44 +2438,6 @@ function SetDocumentCharacterSet(aCharset)
     dump("Compose has not been created!\n");
 }
 
-function UpdateMailEditCharset()
-{
-  var send_default_charset = gMsgCompose.compFields.defaultCharacterSet;
-//  dump("send_default_charset is " + send_default_charset + "\n");
-
-  let compFieldsCharset = gMsgCompose.compFields.characterSet ||
-                          "ISO-8859-1";
-//  dump("gMsgCompose.compFields is " + compFieldsCharset + "\n");
-
-  if (gCharsetConvertManager) {
-    var charsetAlias = gCharsetConvertManager.getCharsetAlias(compFieldsCharset);
-    if (charsetAlias == "us-ascii")
-      compFieldsCharset = "ISO-8859-1";   // no menu item for "us-ascii"
-  }
-
-  // charset may have been set implicitly in case of reply/forward
-  // or use pref default otherwise
-  var menuitem = document.getElementById(send_default_charset == compFieldsCharset ?
-                                         send_default_charset : compFieldsCharset);
-  if (menuitem)
-    menuitem.setAttribute('checked', 'true');
-
-  // Set a document charset to a default mail send charset.
-  if (send_default_charset == compFieldsCharset)
-    SetDocumentCharacterSet(send_default_charset);
-}
-
-function InitCharsetMenuCheckMark()
-{
-  // Check the menu
-  UpdateMailEditCharset();
-  // use setTimeout workaround to delay checkmark the menu
-  // when onmenucomplete is ready then use it instead of oncreate
-  // see bug #78290 for the details
-  setTimeout(UpdateMailEditCharset, 0);
-
-}
-
 function GetCharsetUIString()
 {
   var charset = gMsgCompose.compFields.characterSet;

mail/components/compose/content/messengercompose.xul

@@ -11,13 +11,14 @@
 <?xul-overlay href="chrome://messenger/content/macMenuOverlay.xul"?>
 #endif
 <?xul-overlay href="chrome://messenger/content/baseMenuOverlay.xul"?>
+<?xul-overlay href="chrome://communicator/content/charsetOverlay.xul"?>
 
 <!DOCTYPE window [
   <!ENTITY % messengercomposeDTD SYSTEM "chrome://messenger/locale/messengercompose/messengercompose.dtd" >
   %messengercomposeDTD;
   <!ENTITY % brandDTD SYSTEM "chrome://branding/locale/brand.dtd" >
   %brandDTD;
-  <!ENTITY % charsetDTD SYSTEM "chrome://global/locale/charsetOverlay.dtd" >
+  <!ENTITY % charsetDTD SYSTEM "chrome://global/locale/charsetMenu.dtd" >
   %charsetDTD;
   <!ENTITY % customizeToolbarDTD SYSTEM "chrome://global/locale/customizeToolbar.dtd">
   %customizeToolbarDTD;
@@ -640,19 +641,8 @@
           <menuitem id="returnReceiptMenu" type="checkbox" label="&returnReceiptMenu.label;" accesskey="&returnReceiptMenu.accesskey;" checked="false" oncommand="ToggleReturnReceipt(event.target)"/>
           <menuitem id="dsnMenu" type="checkbox" label="&dsnMenu.label;" accesskey="&dsnMenu.accesskey;" oncommand="ToggleDSN(event.target)"/>
           <menuseparator/>
-          <menu id="maileditCharsetMenu" label="&charsetMenu.label;" accesskey="&charsetMenu.accesskey;" datasources="rdf:charset-menu" ref="NC:MaileditCharsetMenuRoot" oncommand="SetDocumentCharacterSet(event.target.getAttribute('id'));" onpopupshowing="CreateMenu('mailedit');InitCharsetMenuCheckMark();" onpopupshown="CreateMenu('more-menu');">
-            <template>
-              <rule>
-                <menupopup>
-                  <menuitem type="radio" name="charsetGroup" checked="rdf:http://home.netscape.com/NC-rdf#Checked" uri="..." label="rdf:http://home.netscape.com/NC-rdf#Name"/>
-                </menupopup>
-              </rule>
-            </template>
-
-            <menupopup id="maileditCharsetMenuPopup">
-              <menuitem name="charsetCustomize" accesskey="&charsetCustomize.accesskey;" label="&charsetCustomize.label;" oncommand="window.openDialog('chrome://global/content/customizeCharset.xul','PrefWindow', 'chrome,modal=yes,resizable=yes', 'mailedit')"/>
-              <menuseparator />
-            </menupopup>
+          <menu id="charsetMenu" oncommand="SetDocumentCharacterSet(event.target.getAttribute('charset'));" onpopupshowing="UpdateCharsetMenu(gMsgCompose.compFields.characterSet, this);">
+            <menupopup id="charsetPopup" detectors="false"/>
           </menu>
           <menu id="outputFormatMenu" label="&deliveryFormatMenu.label;" accesskey="&deliveryFormatMenu.accesskey;" oncommand="OutputFormatMenuSelect(event.target)">
             <menupopup id="outputFormatMenuPopup">

mail/test/mozmill/composition/test-charset-edit.js

@@ -0,0 +1,173 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Tests that we do the right thing wrt. message encoding when editing or
+ * replying to messages.
+ */
+
+// make SOLO_TEST=composition/test-charset-edit.js mozmill-one
+
+const MODULE_NAME = "test-charset-upgrade";
+
+const RELATIVE_ROOT = "../shared-modules";
+const MODULE_REQUIRES = ["folder-display-helpers", "window-helpers", "compose-helpers"];
+
+var os = {};
+Cu.import("resource://mozmill/stdlib/os.js", os);
+Cu.import('resource://gre/modules/Services.jsm');
+Cu.import("resource:///modules/mailServices.js");
+Cu.import("resource:///modules/mimeParser.jsm");
+var elib = {};
+Cu.import("resource://mozmill/modules/elementslib.js", elib);
+var utils = {};
+Cu.import("resource://mozmill/modules/utils.js", utils);
+
+var draftsFolder;
+
+function setupModule(module) {
+  for (let req of MODULE_REQUIRES) {
+    collector.getModule(req).installInto(module);
+  }
+  let rootFolder = MailServices.accounts.localFoldersServer.rootFolder;
+  if (!rootFolder.containsChildNamed("Drafts")) {
+     create_folder("Drafts", [Ci.nsMsgFolderFlags.Drafts]);
+  }
+  draftsFolder = rootFolder.getChildNamed("Drafts");
+  if (!draftsFolder)
+    throw new Error("draftsFolder not found");
+
+  // Ensure reply charset isn't UTF-8, otherwise there's no need to upgrade,
+  // which is what this test tests.
+  let str = Components.classes["@mozilla.org/pref-localizedstring;1"]
+                      .createInstance(Components.interfaces.nsIPrefLocalizedString);
+  str.data = "windows-1252";
+  Services.prefs.setComplexValue("mailnews.send_default_charset",
+                                 Components.interfaces.nsIPrefLocalizedString, str);
+}
+
+/**
+ * Helper to get the full message content.
+ *
+ * @param aMsgHdr: nsIMsgDBHdr object whose text body will be read
+ * @param aGetText: if true, return header objects. if false, return body data.
+ * @return Map(partnum -> message headers)
+ */
+function getMsgHeaders(aMsgHdr, aGetText=false) {
+  let msgFolder = aMsgHdr.folder;
+  let msgUri = msgFolder.getUriForMsg(aMsgHdr);
+
+  let messenger = Cc["@mozilla.org/messenger;1"]
+                    .createInstance(Ci.nsIMessenger);
+  let handler = {
+    _done: false,
+    _data: new Map(),
+    _text: new Map(),
+    endMessage: function () { this._done = true; },
+    deliverPartData: function (num, text) {
+      this._text.set(num, this._text.get(num) + text);
+    },
+    startPart: function (num, headers) {
+      this._data.set(num, headers);
+      this._text.set(num, "");
+    },
+  };
+  let streamListener = MimeParser.makeStreamListenerParser(handler,
+    {strformat: "unicode"});
+  messenger.messageServiceFromURI(msgUri).streamMessage(msgUri,
+                                                        streamListener,
+                                                        null,
+                                                        null,
+                                                        false,
+                                                        "",
+                                                        false);
+  utils.waitFor(() => handler._done);
+  return aGetText ? handler._text : handler._data;
+}
+
+/**
+ * Test that if we reply to a message in x-mac-croatian, we don't try to compose
+ * in x-mac-croatian. Instead, we should be using the default charset (set to
+ * not be UTF-8 in this test).
+ */
+function test_wrong_reply_charset() {
+  let folder = draftsFolder;
+  let msg0 = create_message({
+    bodyPart: new SyntheticPartLeaf("Some text",
+      {charset: "x-mac-croatian"})
+  });
+  add_message_to_folder(folder, msg0);
+  be_in_folder(folder);
+  let msg = select_click_row(0);
+  assert_selected_and_displayed(mc, msg);
+  assert_equals(getMsgHeaders(msg).get("").charset, "x-mac-croatian");
+
+  let rwc = open_compose_with_reply();
+  // Ctrl+S = save as draft.
+  rwc.keypress(null, "s", {shiftKey: false, accelKey: true});
+  close_compose_window(rwc);
+
+  let draftMsg = select_click_row(1);
+  assert_equals(getMsgHeaders(draftMsg).get("").charset, "windows-1252");
+  press_delete(mc); // Delete message
+
+  // Edit the original message. Charset should be windows-1252 now.
+  msg = select_click_row(0);
+  plan_for_new_window("msgcompose");
+  mc.click(mc.eid("menu_editMsgAsNew"));
+  let rwc = wait_for_compose_window();
+  rwc.keypress(null, "s", {shiftKey: false, accelKey: true});
+  close_compose_window(rwc);
+  msg = select_click_row(0);
+  assert_equals(getMsgHeaders(msg).get("").charset, "windows-1252");
+  press_delete(mc); // Delete message
+}
+
+/**
+ * Test that replying to bad charsets don't screw up the existing text.
+ */
+function test_no_mojibake() {
+  let folder = draftsFolder;
+  let nonASCII = "ケツァルコアトル";
+  let UTF7 = "+MLEwxDChMOswszCiMMgw6w-";
+  let msg0 = create_message({
+    bodyPart: new SyntheticPartLeaf(UTF7, {charset: "utf-7"})
+  });
+  add_message_to_folder(folder, msg0);
+  be_in_folder(folder);
+  let msg = select_click_row(0);
+  assert_selected_and_displayed(mc, msg);
+  assert_equals(getMsgHeaders(msg).get("").charset, "utf-7");
+  assert_equals(getMsgHeaders(msg, true).get("").trim(), nonASCII);
+
+  let rwc = open_compose_with_reply();
+  // Ctrl+S = save as draft.
+  rwc.keypress(null, "s", {shiftKey: false, accelKey: true});
+  close_compose_window(rwc);
+
+  let draftMsg = select_click_row(1);
+  assert_equals(getMsgHeaders(draftMsg).get("").charset, "UTF-8");
+  let text = getMsgHeaders(draftMsg, true).get("");
+  if (!text.contains(nonASCII))
+    throw new Error("Expected to find " + nonASCII + " in " + text);
+  press_delete(mc); // Delete message
+
+  // Edit the original message. Charset should be UTF-8 now.
+  msg = select_click_row(0);
+  plan_for_new_window("msgcompose");
+  mc.click(mc.eid("menu_editMsgAsNew"));
+  let rwc = wait_for_compose_window();
+  rwc.keypress(null, "s", {shiftKey: false, accelKey: true});
+  close_compose_window(rwc);
+  msg = select_click_row(0);
+  assert_equals(getMsgHeaders(msg).get("").charset, "UTF-8");
+  assert_equals(getMsgHeaders(msg, true).get("").trim(), nonASCII);
+  press_delete(mc); // Delete message
+}
+
+function teardownModule(module) {
+  Services.prefs.clearUserPref("mailnews.send_default_charset");
+  MailServices.accounts.localFoldersServer.rootFolder
+              .propagateDelete(draftsFolder, true, null);
+}

mail/test/mozmill/composition/test-charset-upgrade.js

@@ -58,7 +58,7 @@ function setupModule(module) {
   //  which is what this test tests.
   let str = Components.classes["@mozilla.org/pref-localizedstring;1"]
                       .createInstance(Components.interfaces.nsIPrefLocalizedString);
-  str.data = "ISO-8859-1";
+  str.data = "windows-1252";
   Services.prefs.setComplexValue("mailnews.send_default_charset",
                                  Components.interfaces.nsIPrefLocalizedString, str);
 }
@@ -113,10 +113,10 @@ function test_encoding_upgrade_html_compose() {
   let draftMsg = select_click_row(0);
 
   // Charset should still be the default.
-  assert_equals(draftMsg.Charset, "ISO-8859-1");
+  assert_equals(draftMsg.Charset, "windows-1252");
 
   let draftMsgContent = getMsgSource(draftMsg);
-  if (!draftMsgContent.contains('content="text/html; charset=ISO-8859-1"'))
+  if (!draftMsgContent.contains('content="text/html; charset=windows-1252"'))
     throw new Error("Expected content type not in msg; draftMsgContent=" +
                     draftMsgContent);
 
@@ -195,7 +195,7 @@ function test_encoding_upgrade_plaintext_compose() {
   let draftMsg = select_click_row(0);
 
   // Charset should still be the default.
-  assert_equals(draftMsg.Charset, "ISO-8859-1");
+  assert_equals(draftMsg.Charset, "windows-1252");
 
   const CHINESE = "漢皇重色思傾國漢皇重色思傾國";
   compWin.type(compWin.eid("content-frame"),