Store the charset we had when we were written into when document.open()happens. Then restore that as the document charset when loading from a wyciwygchannel, instead of just claiming to be UTF-16. Bug 255820, r+sr=jst

content/base/public/nsIDocument.h

@@ -234,6 +234,8 @@ public:
     return mCharacterSetSource;
   }
 
+  // This method MUST be called before SetDocumentCharacterSet if
+  // you're planning to call both.
   void SetDocumentCharacterSetSource(PRInt32 aCharsetSource)
   {
     mCharacterSetSource = aCharsetSource;

content/html/document/public/nsIWyciwygChannel.idl

@@ -45,7 +45,7 @@
  * help layout save dynamic pages in cache for future retrievals.
  */
 
-[scriptable, uuid (280da566-6f19-4487-a8ca-70c5ba1602c1)]
+[scriptable, uuid (89b0674a-9e2f-4124-9b59-481444be841d)]
 interface nsIWyciwygChannel : nsIChannel
 {
   /**
@@ -62,4 +62,17 @@ interface nsIWyciwygChannel : nsIChannel
    * Set the wyciwyg channels security info
    */
   void setSecurityInfo(in nsISupports aSecurityInfo);
+
+  /**
+   * Store and read a charset and charset source on the wyciwyg channel.  These
+   * are opaque values to the channel; consumers who set them should know what
+   * they mean.
+   */
+  void setCharsetAndSource(in long aSource, in ACString aCharset);
+  /**
+   * The return value is the charset.  Throws if either the charset or the
+   * source cannot be retrieved.  This is guaranteed to return a nonzero source
+   * and a nonempty charset if it does not throw.
+   */
+  ACString getCharsetAndSource(out long aSource);
 };

content/html/document/src/nsHTMLDocument.cpp

@@ -775,6 +775,19 @@ nsHTMLDocument::StartAutodetection(nsIDocShell *aDocShell, nsACString& aCharset,
   }
 }
 
+void
+nsHTMLDocument::SetDocumentCharacterSet(const nsACString& aCharSetID)
+{
+  nsDocument::SetDocumentCharacterSet(aCharSetID);
+  // Make sure to stash this charset on our channel as needed if it's a wyciwyg
+  // channel.
+  nsCOMPtr<nsIWyciwygChannel> wyciwygChannel = do_QueryInterface(mChannel);
+  if (wyciwygChannel) {
+    wyciwygChannel->SetCharsetAndSource(GetDocumentCharacterSetSource(),
+                                        aCharSetID);
+  }
+}
+
 nsresult
 nsHTMLDocument::StartDocumentLoad(const char* aCommand,
                                   nsIChannel* aChannel,
@@ -900,15 +913,26 @@ nsHTMLDocument::StartDocumentLoad(const char* aCommand,
   printf("Determining charset for %s\n", urlSpec.get());
 #endif
 
+  // These are the charset source and charset for our document
   PRInt32 charsetSource;
   nsCAutoString charset;
 
+  // These are the charset source and charset for the parser.  This can differ
+  // from that for the document if the channel is a wyciwyg channel.
+  PRInt32 parserCharsetSource;
+  nsCAutoString parserCharset;
+
+  nsCOMPtr<nsIWyciwygChannel> wyciwygChannel;
+  
   if (IsXHTML()) {
     charsetSource = kCharsetFromDocTypeDefault;
     charset.AssignLiteral("UTF-8");
     TryChannelCharset(aChannel, charsetSource, charset);
+    parserCharsetSource = charsetSource;
+    parserCharset = charset;
   } else {
     charsetSource = kCharsetUninitialized;
+    wyciwygChannel = do_QueryInterface(aChannel);
 
     // The following charset resolving calls has implied knowledge
     // about charset source priority order. Each try will return true
@@ -919,7 +943,11 @@ nsHTMLDocument::StartDocumentLoad(const char* aCommand,
     if (!TryUserForcedCharset(muCV, dcInfo, charsetSource, charset)) {
       TryHintCharset(muCV, charsetSource, charset);
       TryParentCharset(dcInfo, parentDocument, charsetSource, charset);
-      if (TryChannelCharset(aChannel, charsetSource, charset)) {
+
+      // Don't actually get the charset from the channel if this is a
+      // wyciwyg channel; it'll always be UTF-16
+      if (!wyciwygChannel &&
+          TryChannelCharset(aChannel, charsetSource, charset)) {
         // Use the channel's charset (e.g., charset from HTTP
         // "Content-Type" header).
       }
@@ -960,20 +988,45 @@ nsHTMLDocument::StartDocumentLoad(const char* aCommand,
       }
     }
 
+    if (wyciwygChannel) {
+      // We know for sure that the parser needs to be using UTF16.
+      parserCharset = "UTF-16";
+      parserCharsetSource = charsetSource < kCharsetFromChannel ?
+        kCharsetFromChannel : charsetSource;
+        
+      nsCAutoString cachedCharset;
+      PRInt32 cachedSource;
+      rv = wyciwygChannel->GetCharsetAndSource(&cachedSource, cachedCharset);
+      if (NS_SUCCEEDED(rv)) {
+        if (cachedSource > charsetSource) {
+          charsetSource = cachedSource;
+          charset = cachedCharset;
+        }
+      } else {
+        // Don't propagate this error.
+        rv = NS_OK;
+      }
+      
+    } else {
+      parserCharset = charset;
+      parserCharsetSource = charsetSource;
+    }
+
     if(kCharsetFromAutoDetection > charsetSource && !isPostPage) {
       StartAutodetection(docShell, charset, aCommand);
     }
 
     // ahmed
     // Check if 864 but in Implicit mode !
+    // XXXbz why is this happening after StartAutodetection ?
     if ((textType == IBMBIDI_TEXTTYPE_LOGICAL) &&
         (charset.LowerCaseEqualsLiteral("ibm864"))) {
       charset.AssignLiteral("IBM864i");
     }
   }
 
-  SetDocumentCharacterSet(charset);
   SetDocumentCharacterSetSource(charsetSource);
+  SetDocumentCharacterSet(charset);
 
   // set doc charset to muCV for next document.
   // Don't propagate this back up to the parent document if we have one.
@@ -981,6 +1034,9 @@ nsHTMLDocument::StartDocumentLoad(const char* aCommand,
     muCV->SetPrevDocCharacterSet(charset);
 
   if(cacheDescriptor) {
+    NS_ASSERTION(charset == parserCharset,
+                 "How did those end up different here?  wyciwyg channels are "
+                 "not nsICachingChannel");
     rv = cacheDescriptor->SetMetaDataElement("charset",
                                              charset.get());
     NS_ASSERTION(NS_SUCCEEDED(rv),"cannot SetMetaDataElement");
@@ -997,7 +1053,7 @@ nsHTMLDocument::StartDocumentLoad(const char* aCommand,
     printf(" charset = %s source %d\n",
           charset.get(), charsetSource);
 #endif
-    mParser->SetDocumentCharset(charset, charsetSource);
+    mParser->SetDocumentCharset(parserCharset, parserCharsetSource);
     mParser->SetCommand(aCommand);
 
     // create the content sink
@@ -3626,6 +3682,11 @@ nsHTMLDocument::CreateAndAddWyciwygChannel(void)
 
   mWyciwygChannel->SetSecurityInfo(mSecurityInfo);
 
+  // Note: we want to treat this like a "previous document" hint so that,
+  // e.g. a <meta> tag in the document.write content can override it.
+  mWyciwygChannel->SetCharsetAndSource(kCharsetFromHintPrevDoc,
+                                       GetDocumentCharacterSet());
+
   // Use our new principal
   channel->SetOwner(NodePrincipal());
 

content/html/document/src/nsHTMLDocument.h

@@ -297,6 +297,9 @@ protected:
   void StartAutodetection(nsIDocShell *aDocShell, nsACString& aCharset,
                           const char* aCommand);
 
+  // Override so we can munge the charset on our wyciwyg channel as needed.
+  virtual void SetDocumentCharacterSet(const nsACString& aCharSetID);
+
   // mWriteState tracks the status of this document if the document is being
   // entirely created by script. In the normal load case, mWriteState will be
   // eNotWriting. Once document.open has been called (either implicitly or

content/html/document/src/nsWyciwygChannel.cpp

@@ -389,6 +389,59 @@ nsWyciwygChannel::SetSecurityInfo(nsISupports *aSecurityInfo)
   return NS_OK;
 }
 
+NS_IMETHODIMP
+nsWyciwygChannel::SetCharsetAndSource(PRInt32 aSource,
+                                      const nsACString& aCharset)
+{
+  NS_ENSURE_ARG(!aCharset.IsEmpty());
+
+  if (!mCacheEntry) {
+    nsCAutoString spec;
+    nsresult rv = mURI->GetAsciiSpec(spec);
+    if (NS_FAILED(rv)) return rv;
+    rv = OpenCacheEntry(spec, nsICache::ACCESS_WRITE);
+    if (NS_FAILED(rv)) return rv;
+  }
+
+  mCacheEntry->SetMetaDataElement("charset",
+                                  PromiseFlatCString(aCharset).get());
+  nsCAutoString source;
+  source.AppendInt(aSource);
+  mCacheEntry->SetMetaDataElement("charset-source", source.get());
+  
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsWyciwygChannel::GetCharsetAndSource(PRInt32* aSource, nsACString& aCharset)
+{
+  if (!mCacheEntry) {
+    return NS_ERROR_NOT_AVAILABLE;
+  }
+
+  nsXPIDLCString data;
+  mCacheEntry->GetMetaDataElement("charset", getter_Copies(data));
+
+  if (data.IsEmpty()) {
+    return NS_ERROR_NOT_AVAILABLE;
+  }
+
+  nsXPIDLCString sourceStr;
+  mCacheEntry->GetMetaDataElement("charset-source", getter_Copies(sourceStr));
+
+  PRInt32 source;
+  // XXXbz ToInteger takes an PRInt32* but outputs an nsresult in it... :(
+  PRInt32 err;
+  source = sourceStr.ToInteger(&err);
+  if (NS_FAILED(err) || source == 0) {
+    return NS_ERROR_NOT_AVAILABLE;
+  }
+
+  *aSource = source;
+  aCharset = data;
+  return NS_OK;
+}
+
 //////////////////////////////////////////////////////////////////////////////
 // nsICachelistener
 //////////////////////////////////////////////////////////////////////////////

content/html/document/test/Makefile.in

@@ -47,6 +47,7 @@ include $(topsrcdir)/config/rules.mk
 _TEST_FILES = 	test_bug1682.html \
 		test_bug1823.html \
 		test_bug172261.html \
+		test_bug255820.html \
 		test_bug311681.html \
 		test_bug311681.xhtml \
 		test_bug332848.xhtml \

content/html/document/test/test_bug255820.html

@@ -0,0 +1,120 @@
+<!DOCTYPE HTML>
+<html>
+<!--
+https://bugzilla.mozilla.org/show_bug.cgi?id=255820
+-->
+<head>
+  <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+  <title>Test for Bug 255820</title>
+  <script type="text/javascript" src="/MochiKit/MochiKit.js"></script>
+  <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>        
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" />
+</head>
+<body>
+<a target="_blank" href="https://bugzilla.mozilla.org/show_bug.cgi?id=255820">Mozilla Bug 255820</a>
+<p id="display">
+  <iframe id="f1"></iframe>
+  <iframe id="f2"></iframe>
+  <iframe id="f3"></iframe>
+</p>
+<div id="content" style="display: none">
+  
+</div>
+<pre id="test">
+<script class="testbody" type="text/javascript">
+
+/** Test for Bug 255820 **/
+SimpleTest.waitForExplicitFinish();
+
+is(document.characterSet, "UTF-8",
+   "Unexpected character set for our document");
+
+var testsLeft = 4;
+
+function testFinished() {
+  --testsLeft;
+  if (testsLeft == 0) {
+    SimpleTest.finish();
+  }
+}
+
+function charsetTestFinished(id, doc, charsetTarget) {
+  is(doc.characterSet, charsetTarget, "Unexpected charset for subframe " + id);
+  testFinished();
+}
+
+function f2Continue() {
+  $("f2").
+    setAttribute("onload",
+                 "charsetTestFinished('f2 reloaded', this.contentDocument, 'us-ascii');");
+  $("f2").contentWindow.location.reload();
+}
+
+function f3Continue() {
+  var doc = $("f3").contentDocument;
+  is(doc.defaultView.getComputedStyle(doc.body, "").color, "rgb(0, 180, 0)",
+     "Wrong color before reload");
+  $("f3").
+    setAttribute("onload",
+                 'var doc = this.contentDocument; ' + 
+                 'is(doc.defaultView.getComputedStyle(doc.body, "").color, ' +
+                 '   "rgb(0, 180, 0)",' +
+                 '   "Wrong color after reload");' +
+                 "charsetTestFinished('f1', this.contentDocument, 'UTF-8')");
+  $("f3").contentWindow.location.reload();
+}
+
+function runTest() {
+  var doc = $("f1").contentDocument;
+  is(doc.characterSet, "UTF-8",
+     "Unexpected initial character set for first frame");
+  doc.open();
+  doc.write('<html></html>');
+  doc.close();
+  is(doc.characterSet, "UTF-8",
+     "Unexpected character set for first frame after write");
+  $("f1").
+    setAttribute("onload",
+                 "charsetTestFinished('f1', this.contentDocument, 'UTF-8')");
+  $("f1").contentWindow.location.reload();
+
+  doc = $("f2").contentDocument;
+  is(doc.characterSet, "UTF-8",
+     "Unexpected initial character set for second frame");
+  doc.open();
+  var str = '<html><head>';
+  str += '<script src="data:application/javascript,"><'+'/script>';
+  str += '<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">';
+  str += '</head><body>';
+  str += '</body></html>';
+  doc.write(str);
+  doc.close();
+  is(doc.characterSet, "UTF-8",
+     "Unexpected character set for second frame after write");
+  $("f2").
+    setAttribute("onload",
+                 "charsetTestFinished('f2', this.contentDocument, 'us-ascii');" +
+                 "f2Continue()");
+
+  doc = $("f3").contentDocument;
+  is(doc.characterSet, "UTF-8",
+     "Unexpected initial character set for first frame");
+  doc.open();
+  var str = '<html><head>';
+  str += '<style>body { color: rgb(255, 0, 0) }</style>';
+  str += '<link type="text/css" rel="stylesheet" href="data:text/css, body { color: rgb(0, 180, 0) }">';
+  str += '</head><body>';
+  str += '</body></html>';
+  doc.write(str);
+  doc.close();
+  is(doc.characterSet, "UTF-8",
+     "Unexpected character set for first frame after write");
+  $("f3").setAttribute("onload", "f3Continue()");
+}
+
+addLoadEvent(runTest);
+</script>
+</pre>
+</body>
+</html>
+

content/xslt/src/base/txURIUtils.cpp

@@ -241,9 +241,9 @@ URIUtils::ResetWithSource(nsIDocument *aNewDoc, nsIDOMNode *aSourceNode)
     aNewDoc->SetBaseURI(sourceDoc->GetBaseURI());
 
     // Copy charset
-    aNewDoc->SetDocumentCharacterSet(sourceDoc->GetDocumentCharacterSet());
     aNewDoc->SetDocumentCharacterSetSource(
           sourceDoc->GetDocumentCharacterSetSource());
+    aNewDoc->SetDocumentCharacterSet(sourceDoc->GetDocumentCharacterSet());
 }
 
 #endif /* TX_EXE */

content/xslt/src/xslt/txMozillaTextOutput.cpp

@@ -197,8 +197,8 @@ txMozillaTextOutput::createResultDocument(nsIDOMDocument* aSourceDocument,
 
         if (calias &&
             NS_SUCCEEDED(calias->GetPreferred(charset, canonicalCharset))) {
-            mDocument->SetDocumentCharacterSet(canonicalCharset);
             mDocument->SetDocumentCharacterSetSource(kCharsetFromOtherComponent);
+            mDocument->SetDocumentCharacterSet(canonicalCharset);
         }
     }
 

content/xslt/src/xslt/txMozillaXMLOutput.cpp

@@ -883,8 +883,8 @@ txMozillaXMLOutput::createResultDocument(const nsSubstring& aName, PRInt32 aNsID
 
         if (calias &&
             NS_SUCCEEDED(calias->GetPreferred(charset, canonicalCharset))) {
-            mDocument->SetDocumentCharacterSet(canonicalCharset);
             mDocument->SetDocumentCharacterSetSource(kCharsetFromOtherComponent);
+            mDocument->SetDocumentCharacterSet(canonicalCharset);
         }
     }