bug 752756. Read UTF-16le and UTF-8 BOMs in nsINIParser on Windows. Add parameter to nsIINIParserWriter::writeFile() specifying the charset of the file (BOM will be written). r=bsmedberg

toolkit/crashreporter/nsExceptionHandler.cpp

@@ -1589,7 +1589,7 @@ static nsresult PrefSubmitReports(bool* aSubmitReports, bool writePref)
                               *aSubmitReports ?  NS_LITERAL_CSTRING("1") :
                                                  NS_LITERAL_CSTRING("0"));
     NS_ENSURE_SUCCESS(rv, rv);
-    rv = iniWriter->WriteFile(NULL);
+    rv = iniWriter->WriteFile(NULL, 0);
     return rv;
   }
   

xpcom/ds/nsIINIParser.idl

@@ -26,9 +26,16 @@ interface nsIINIParser : nsISupports
   AUTF8String getString(in AUTF8String aSection, in AUTF8String aKey);
 };
 
-[scriptable, uuid(712dc5da-8d09-45d0-ba2e-de27eb384c4c)]
+[scriptable, uuid(b67bb24b-31a3-4a6a-a5d9-0485c9af5a04)]
 interface nsIINIParserWriter : nsISupports
 {
+  /**
+   * Windows and the NSIS installer code sometimes expect INI files to be in
+   * UTF-16 encoding. On Windows only, this flag to writeFile can be used to
+   * change the encoding from its default UTF-8.
+   */
+  const unsigned long WRITE_UTF16 = 0x1;
+
   /**
    * Set the value of a string for a particular section and key.
    */
@@ -37,7 +44,8 @@ interface nsIINIParserWriter : nsISupports
   /**
    * Write to the INI file.
    */
-  void writeFile([optional] in nsILocalFile aINIFile);
+  void writeFile([optional] in nsILocalFile aINIFile,
+                 [optional] in unsigned long aFlags);
 };
 
 [scriptable, uuid(ccae7ea5-1218-4b51-aecb-c2d8ecd46af9)]

xpcom/ds/nsINIProcessor.js

@@ -37,18 +37,29 @@ function INIProcessor(aFile) {
 INIProcessor.prototype = {
     QueryInterface : XPCOMUtils.generateQI([Ci.nsIINIParser, Ci.nsIINIParserWriter]),
 
-    __utfConverter : null, // UCS2 <--> UTF8 string conversion
+    __utf8Converter : null, // UCS2 <--> UTF8 string conversion
-    get _utfConverter() {
+    get _utf8Converter() {
-        if (!this.__utfConverter) {
+        if (!this.__utf8Converter) {
-            this.__utfConverter = Cc["@mozilla.org/intl/scriptableunicodeconverter"].
+            this.__utf8Converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"].
                                   createInstance(Ci.nsIScriptableUnicodeConverter);
-            this.__utfConverter.charset = "UTF-8";
+            this.__utf8Converter.charset = "UTF-8";
         }
-        return this.__utfConverter;
+        return this.__utf8Converter;
+    },
+
+    __utf16leConverter : null, // UCS2 <--> UTF16LE string conversion
+    get _utf16leConverter() {
+        if (!this.__utf16leConverter) {
+            this.__utf16leConverter = Cc["@mozilla.org/intl/scriptableunicodeconverter"].
+                                  createInstance(Ci.nsIScriptableUnicodeConverter);
+            this.__utf16leConverter.charset = "UTF-16LE";
+        }
+        return this.__utf16leConverter;
     },
 
     _utfConverterReset : function() {
-        this.__utfConverter = null;
+        this.__utf8Converter = null;
+        this.__utf16leConverter = null;
     },
 
     _iniFile : null,
@@ -117,13 +128,13 @@ INIProcessor.prototype = {
         this._iniData[aSection][aKey] = aValue;
     },
 
-    writeFile : function(aFile) {
+    writeFile : function(aFile, aFlags) {
 
-        let converter = this._utfConverter;
+        let converter;
         function writeLine(data) {
+            data += "\n";
             data = converter.ConvertFromUnicode(data);
             data += converter.Finish();
-            data += "\n";
             outputStream.write(data, data.length);
         }
 
@@ -140,6 +151,14 @@ INIProcessor.prototype = {
         outputStream.init(safeStream, 8192);
         outputStream.QueryInterface(Ci.nsISafeOutputStream); // for .finish()
 
+        if (Ci.nsIINIParserWriter.WRITE_UTF16 == aFlags
+         && 'nsIWindowsRegKey' in Ci) {
+            outputStream.write("\xFF\xFE", 2);
+            converter = this._utf16leConverter;
+        } else {
+            converter = this._utf8Converter;
+        }
+
         for (let section in this._iniData) {
             writeLine("[" + section + "]");
             for (let key in this._iniData[section]) {

xpcom/glue/nsINIParser.cpp

@@ -3,13 +3,16 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-#include "nsINIParser.h"
+// Moz headers (alphabetical)
+#include "nsCRTGlue.h"
 #include "nsError.h"
 #include "nsILocalFile.h"
-#include "nsCRTGlue.h"
+#include "nsINIParser.h"
+#include "mozilla/FileUtils.h" // AutoFILE
 
-#include <stdlib.h>
+// System headers (alphabetical)
 #include <stdio.h>
+#include <stdlib.h>
 #ifdef XP_WIN
 #include <windows.h>
 #endif
@@ -109,7 +112,7 @@ nsINIParser::InitFromFILE(FILE *fd)
         return NS_ERROR_FAILURE;
 
     /* malloc an internal buf the size of the file */
-    mFileContents = new char[flen + 1];
+    mFileContents = new char[flen + 2];
     if (!mFileContents)
         return NS_ERROR_OUT_OF_MEMORY;
 
@@ -121,9 +124,56 @@ nsINIParser::InitFromFILE(FILE *fd)
     if (rd != flen)
         return NS_BASE_STREAM_OSERROR;
 
-    mFileContents[flen] = '\0';
+    // We write a UTF16 null so that the file is easier to convert to UTF8
+    mFileContents[flen] = mFileContents[flen + 1] = '\0';
+
+    char *buffer = &mFileContents[0];
+
+    if (flen >= 3
+    && mFileContents[0] == static_cast<char>(0xEF)
+    && mFileContents[1] == static_cast<char>(0xBB)
+    && mFileContents[2] == static_cast<char>(0xBF)) {
+      // Someone set us up the Utf-8 BOM
+      // This case is easy, since we assume that BOM-less
+      // files are Utf-8 anyway.  Just skip the BOM and process as usual.
+      buffer = &mFileContents[3];
+    }
+
+#ifdef XP_WIN
+    if (flen >= 2
+     && mFileContents[0] == static_cast<char>(0xFF)
+     && mFileContents[1] == static_cast<char>(0xFE)) {
+        // Someone set us up the Utf-16LE BOM
+        buffer = &mFileContents[2];
+        // Get the size required for our Utf8 buffer
+        flen = WideCharToMultiByte(CP_UTF8,
+                                   0,
+                                   reinterpret_cast<LPWSTR>(buffer),
+                                   -1,
+                                   NULL,
+                                   0,
+                                   NULL,
+                                   NULL);
+        if (0 == flen) {
+            return NS_ERROR_FAILURE;
+        }
+
+        nsAutoArrayPtr<char> utf8Buffer = new char[flen];
+        if (0 == WideCharToMultiByte(CP_UTF8,
+                                     0,
+                                     reinterpret_cast<LPWSTR>(buffer),
+                                     -1,
+                                     utf8Buffer,
+                                     flen,
+                                     NULL,
+                                     NULL)) {
+            return NS_ERROR_FAILURE;
+        }
+        mFileContents = utf8Buffer.forget();
+        buffer = mFileContents;
+    }
+#endif
 
-    char *buffer = mFileContents;
     char *currSection = nsnull;
 
     // outer loop tokenizes into lines

xpcom/tests/unit/data/iniparser01-utf16leBOM.ini

@@ -0,0 +1 @@
+

xpcom/tests/unit/data/iniparser01-utf8BOM.ini

@@ -0,0 +1 @@
+

xpcom/tests/unit/data/iniparser02-utf8BOM.ini

@@ -0,0 +1 @@
+

xpcom/tests/unit/data/iniparser03-utf8BOM.ini

@@ -0,0 +1 @@
+[]

xpcom/tests/unit/data/iniparser04-utf8BOM.ini

@@ -0,0 +1 @@
+[section1]

xpcom/tests/unit/data/iniparser05-utf8BOM.ini

@@ -0,0 +1 @@
+[section1]junk

xpcom/tests/unit/data/iniparser06-utf8BOM.ini

@@ -0,0 +1,2 @@
+[section1]
+

xpcom/tests/unit/data/iniparser07-utf8BOM.ini

@@ -0,0 +1,2 @@
+[section1]
+name1

xpcom/tests/unit/data/iniparser08-utf8BOM.ini

@@ -0,0 +1,2 @@
+[section1]
+name1=

xpcom/tests/unit/data/iniparser09-utf8BOM.ini

@@ -0,0 +1,2 @@
+[section1]
+name1=value1

xpcom/tests/unit/data/iniparser10-utf8BOM.ini

@@ -0,0 +1,3 @@
+
+[section1]
+name1=value1

xpcom/tests/unit/data/iniparser11-utf8BOM.ini

@@ -0,0 +1,3 @@
+# comment
+[section1]
+name1=value1

xpcom/tests/unit/data/iniparser12-utf8BOM.ini

@@ -0,0 +1,3 @@
+[section1]
+# [sectionBAD]
+name1=value1

xpcom/tests/unit/data/iniparser13-utf8BOM.ini

@@ -0,0 +1,3 @@
+[section1]
+name1=value1
+# nameBAD=valueBAD

xpcom/tests/unit/data/iniparser14-utf8BOM.ini

@@ -0,0 +1,6 @@
+[section1]
+name1=value1
+name2=value2
+[section2]
+name1=value1
+name2=foopy

xpcom/tests/unit/data/iniparser15-utf8BOM.ini

@@ -0,0 +1,6 @@
+[section1]
+name1=value1
+[section2]
+name1=foopy
+[section1]
+name1=newValue1

xpcom/tests/unit/data/iniparser16-utf8BOM.ini

@@ -0,0 +1,13 @@
+#Ώṍҳϖ·̐˄ȡǨŅ©&
+[☺♫]
+#ѼΏṍҳϖ
+♫=☻
+#·̐˄ȡǨŅ©
+♪=♥
+#‽ἧᵿΏṍҳ
+#ϖ·̐˄ȡǨŅ©&
+[☼]
+♣=♠
+♦=♥
+#‽ἧᵿΏṍҳ
+#·̐˄ȡǨŅ©

xpcom/tests/unit/data/iniparser16.ini

@@ -0,0 +1,13 @@
+#Ώṍҳϖ·̐˄ȡǨŅ©&
+[☺♫]
+#ѼΏṍҳϖ
+♫=☻
+#·̐˄ȡǨŅ©
+♪=♥
+#‽ἧᵿΏṍҳ
+#ϖ·̐˄ȡǨŅ©&
+[☼]
+♣=♠
+♦=♥
+#‽ἧᵿΏṍҳ
+#·̐˄ȡǨŅ©

xpcom/tests/unit/test_iniProcessor.js

@@ -90,20 +90,93 @@ let testdata = [
     { filename: "data/iniparser15.ini", reference: 
                     { section1: { name1: "newValue1" },
                       section2: { name1: "foopy"     }} },
+    { filename: "data/iniparser16.ini", reference: 
+                    { "☺♫": { "♫": "☻", "♪": "♥"  },
+                       "☼": { "♣": "♠", "♦": "♥"  }} },
+
     ];
 
+    testdata.push( { filename: "data/iniparser01-utf8BOM.ini",
+                     reference: testdata[0].reference } );
+    testdata.push( { filename: "data/iniparser02-utf8BOM.ini",
+                     reference: testdata[1].reference } );
+    testdata.push( { filename: "data/iniparser03-utf8BOM.ini",
+                     reference: testdata[2].reference } );
+    testdata.push( { filename: "data/iniparser04-utf8BOM.ini",
+                     reference: testdata[3].reference } );
+    testdata.push( { filename: "data/iniparser05-utf8BOM.ini",
+                     reference: testdata[4].reference } );
+    testdata.push( { filename: "data/iniparser06-utf8BOM.ini",
+                     reference: testdata[5].reference } );
+    testdata.push( { filename: "data/iniparser07-utf8BOM.ini",
+                     reference: testdata[6].reference } );
+    testdata.push( { filename: "data/iniparser08-utf8BOM.ini",
+                     reference: testdata[7].reference } );
+    testdata.push( { filename: "data/iniparser09-utf8BOM.ini",
+                     reference: testdata[8].reference } );
+    testdata.push( { filename: "data/iniparser10-utf8BOM.ini",
+                     reference: testdata[9].reference } );
+    testdata.push( { filename: "data/iniparser11-utf8BOM.ini",
+                     reference: testdata[10].reference } );
+    testdata.push( { filename: "data/iniparser12-utf8BOM.ini",
+                     reference: testdata[11].reference } );
+    testdata.push( { filename: "data/iniparser13-utf8BOM.ini",
+                     reference: testdata[12].reference } );
+    testdata.push( { filename: "data/iniparser14-utf8BOM.ini",
+                     reference: testdata[13].reference } );
+    testdata.push( { filename: "data/iniparser15-utf8BOM.ini",
+                     reference: testdata[14].reference } );
+    testdata.push( { filename: "data/iniparser16-utf8BOM.ini",
+                     reference: testdata[15].reference } );
+
+    let os = Cc["@mozilla.org/xre/app-info;1"]
+             .getService(Ci.nsIXULRuntime).OS;
+    if("WINNT" === os) {
+        testdata.push( { filename: "data/iniparser01-utf16leBOM.ini",
+                         reference: testdata[0].reference } );
+        testdata.push( { filename: "data/iniparser02-utf16leBOM.ini",
+                         reference: testdata[1].reference } );
+        testdata.push( { filename: "data/iniparser03-utf16leBOM.ini",
+                         reference: testdata[2].reference } );
+        testdata.push( { filename: "data/iniparser04-utf16leBOM.ini",
+                         reference: testdata[3].reference } );
+        testdata.push( { filename: "data/iniparser05-utf16leBOM.ini",
+                         reference: testdata[4].reference } );
+        testdata.push( { filename: "data/iniparser06-utf16leBOM.ini",
+                         reference: testdata[5].reference } );
+        testdata.push( { filename: "data/iniparser07-utf16leBOM.ini",
+                         reference: testdata[6].reference } );
+        testdata.push( { filename: "data/iniparser08-utf16leBOM.ini",
+                         reference: testdata[7].reference } );
+        testdata.push( { filename: "data/iniparser09-utf16leBOM.ini",
+                         reference: testdata[8].reference } );
+        testdata.push( { filename: "data/iniparser10-utf16leBOM.ini",
+                         reference: testdata[9].reference } );
+        testdata.push( { filename: "data/iniparser11-utf16leBOM.ini",
+                         reference: testdata[10].reference } );
+        testdata.push( { filename: "data/iniparser12-utf16leBOM.ini",
+                         reference: testdata[11].reference } );
+        testdata.push( { filename: "data/iniparser13-utf16leBOM.ini",
+                         reference: testdata[12].reference } );
+        testdata.push( { filename: "data/iniparser14-utf16leBOM.ini",
+                         reference: testdata[13].reference } );
+        testdata.push( { filename: "data/iniparser15-utf16leBOM.ini",
+                         reference: testdata[14].reference } );
+        testdata.push( { filename: "data/iniparser16-utf16leBOM.ini",
+                         reference: testdata[15].reference } );
+    }
+
 /* ========== 0 ========== */
 factory = Cc["@mozilla.org/xpcom/ini-processor-factory;1"].
           getService(Ci.nsIINIParserFactory);
 do_check_true(!!factory);
 
-/* ========== 1 - 15 ========== */
-
 // Test reading from a variety of files. While we're at it, write out each one
 // and read it back to ensure that nothing changed.
-for (testnum = 1; testnum <= 15; testnum++) {
+while (testnum < testdata.length) {
+    dump("\nINFO | test #" + ++testnum);
     let filename = testdata[testnum -1].filename;
-    dump("INFO | test #" + testnum + ", filename " + filename + "\n");
+    dump(", filename " + filename + "\n");
     let parser = parserForFile(filename);
     checkParserOutput(parser, testdata[testnum - 1].reference);
     if (!parser)
@@ -121,7 +194,7 @@ for (testnum = 1; testnum <= 15; testnum++) {
     newfile.remove(false);
 }
 
-/* ========== 16 ========== */
+dump("INFO | test #" + ++testnum + "\n");
 
 // test writing to a new file.
 let newfile = do_get_file("data/");
@@ -148,7 +221,7 @@ checkParserOutput(parser, {section: {key: "value"} });
 // cleanup after the test
 newfile.remove(false);
 
-/* ========== 17 ========== */
+dump("INFO | test #" + ++testnum + "\n");
 
 // test modifying a existing key's value (in an existing section)
 parser = parserForFile("data/iniparser09.ini");
@@ -158,7 +231,7 @@ do_check_true(parser instanceof Ci.nsIINIParserWriter);
 parser.setString("section1", "name1", "value2");
 checkParserOutput(parser, {section1: {name1: "value2"} });
 
-/* ========== 18 ========== */
+dump("INFO | test #" + ++testnum + "\n");
 
 // test trying to set illegal characters
 let caughtError;