Bug 1070763 - Ensure that XHRs sniff the BOM for non-JSON responseTypes, and flush the decoder upon end-of-stream; r=hsivonen

MozReview-Commit-ID: ICHbs2BQcbR

--HG--
extra : rebase_source : 91fac79b191330c5f771386f9977450a569157b1

dom/xhr/XMLHttpRequestMainThread.cpp

@@ -521,15 +521,25 @@ XMLHttpRequestMainThread::DetectCharset()
   }
 
   mResponseCharset = encoding;
-  mDecoder = encoding->NewDecoderWithBOMRemoval();
+
+  // Only sniff the BOM for non-JSON responseTypes
+  if (mResponseType == XMLHttpRequestResponseType::Json) {
+    mDecoder = encoding->NewDecoderWithBOMRemoval();
+  } else {
+    mDecoder = encoding->NewDecoder();
+  }
 
   return NS_OK;
 }
 
 nsresult
 XMLHttpRequestMainThread::AppendToResponseText(const char * aSrcBuffer,
-                                               uint32_t aSrcBufferLen)
+                                               uint32_t aSrcBufferLen,
+                                               bool aLast)
 {
+  // Call this with an empty buffer to send the decoder the signal
+  // that we have hit the end of the stream.
+
   NS_ENSURE_STATE(mDecoder);
 
   CheckedInt<size_t> destBufferLen =
@@ -550,7 +560,6 @@ XMLHttpRequestMainThread::AppendToResponseText(const char * aSrcBuffer,
     return NS_ERROR_OUT_OF_MEMORY;
   }
 
-  // XXX there's no handling for incomplete byte sequences on EOF!
   uint32_t result;
   size_t read;
   size_t written;
@@ -558,7 +567,7 @@ XMLHttpRequestMainThread::AppendToResponseText(const char * aSrcBuffer,
   Tie(result, read, written, hadErrors) = mDecoder->DecodeToUTF16(
     AsBytes(MakeSpan(aSrcBuffer, aSrcBufferLen)),
     MakeSpan(helper.EndOfExistingData(), destBufferLen.value()),
-    false);
+    aLast);
   MOZ_ASSERT(result == kInputEmpty);
   MOZ_ASSERT(read == aSrcBufferLen);
   MOZ_ASSERT(written <= destBufferLen.value());
@@ -2189,6 +2198,12 @@ XMLHttpRequestMainThread::OnStopRequest(nsIRequest *request, nsISupports *ctxt,
     return NS_OK;
   }
 
+  // send the decoder the signal that we've hit the end of the stream,
+  // but only when parsing text (not XML, which does this already).
+  if (mDecoder && !mFlagParseBody) {
+    AppendToResponseText(nullptr, 0, true);
+  }
+
   mWaitingForOnStopRequest = false;
 
   if (mRequestObserver) {
@@ -2364,7 +2379,7 @@ XMLHttpRequestMainThread::MatchCharsetAndDecoderToResponseDocument()
     mResponseCharset = mResponseXML->GetDocumentCharacterSet();
     TruncateResponseText();
     mResponseBodyDecodedPos = 0;
-    mDecoder = mResponseCharset->NewDecoderWithBOMRemoval();
+    mDecoder = mResponseCharset->NewDecoder();
   }
 }
 

dom/xhr/XMLHttpRequestMainThread.h

@@ -520,7 +520,8 @@ protected:
   };
 
   nsresult DetectCharset();
-  nsresult AppendToResponseText(const char * aBuffer, uint32_t aBufferLen);
+  nsresult AppendToResponseText(const char* aBuffer, uint32_t aBufferLen,
+                                bool aLast = false);
   static nsresult StreamReaderFunc(nsIInputStream* in,
                                    void* closure,
                                    const char* fromRawSegment,

testing/web-platform/meta/XMLHttpRequest/responsetext-decoding.htm.ini

@@ -1,11 +0,0 @@
-[responsetext-decoding.htm]
-  type: testharness
-  [XMLHttpRequest: responseText decoding (text/plain %FE%FF)]
-    expected: FAIL
-
-  [XMLHttpRequest: responseText decoding (text/plain %FE%FF%FE%FF)]
-    expected: FAIL
-
-  [XMLHttpRequest: responseText decoding (text/plain %C2)]
-    expected: FAIL
-

testing/web-platform/meta/XMLHttpRequest/send-receive-utf16.htm.ini

@@ -1,5 +0,0 @@
-[send-receive-utf16.htm]
-  type: testharness
-  [UTF-16 with BOM, no encoding in content-type]
-    expected: FAIL
-

testing/web-platform/meta/encoding/unsupported-encodings.html.ini

@@ -1,17 +1,5 @@
 [unsupported-encodings.html]
   type: testharness
-  [UTF-32 with BOM should decode as UTF-16LE]
-    expected: FAIL
-
-  [utf-32 with BOM should decode as UTF-16LE]
-    expected: FAIL
-
-  [UTF-32LE with BOM should decode as UTF-16LE]
-    expected: FAIL
-
-  [utf-32le with BOM should decode as UTF-16LE]
-    expected: FAIL
-
   [UTF-32be with BOM should decode as windows-1252]
     expected: FAIL