зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1396286 - Support UTF-16 in JSON Viewer. r=tromey
MozReview-Commit-ID: Dy7474tyVyc --HG-- rename : devtools/client/jsonview/test/browser_jsonview_utf8.js => devtools/client/jsonview/test/browser_jsonview_encoding.js extra : rebase_source : 1f543beaabc92fda63726485076f05f78283771c
This commit is contained in:
Oriol Brufau
Родитель
34d66eceb2
Коммит
6a2f7418ce
|
|
@ -6,7 +6,7 @@
|
|||
|
||||
"use strict";
|
||||
|
||||
const {Cc, Ci, Cu} = require("chrome");
|
||||
const {Cc, Ci, Cu, CC} = require("chrome");
|
||||
const { XPCOMUtils } = Cu.import("resource://gre/modules/XPCOMUtils.jsm", {});
|
||||
const Services = require("Services");
|
||||
|
||||
|
|
@ -20,6 +20,12 @@ loader.lazyGetter(this, "debug", function () {
|
|||
const childProcessMessageManager =
|
||||
Cc["@mozilla.org/childprocessmessagemanager;1"]
|
||||
.getService(Ci.nsISyncMessageSender);
|
||||
const BinaryInput = CC("@mozilla.org/binaryinputstream;1",
|
||||
"nsIBinaryInputStream", "setInputStream");
|
||||
const BufferStream = CC("@mozilla.org/io/arraybuffer-input-stream;1",
|
||||
"nsIArrayBufferInputStream", "setData");
|
||||
const encodingLength = 2;
|
||||
const encoder = new TextEncoder();
|
||||
|
||||
// Localization
|
||||
loader.lazyGetter(this, "jsonViewStrings", () => {
|
||||
|
|
@ -52,8 +58,8 @@ Converter.prototype = {
|
|||
* 1. asyncConvertData captures the listener
|
||||
* 2. onStartRequest fires, initializes stuff, modifies the listener
|
||||
* to match our output type
|
||||
* 3. onDataAvailable spits it back to the listener
|
||||
* 4. onStopRequest spits it back to the listener
|
||||
* 3. onDataAvailable converts to UTF-8 and spits back to the listener
|
||||
* 4. onStopRequest flushes data and spits back to the listener
|
||||
* 5. convert does nothing, it's just the synchronous version
|
||||
* of asyncConvertData
|
||||
*/
|
||||
|
|
@ -66,7 +72,29 @@ Converter.prototype = {
|
|||
},
|
||||
|
||||
onDataAvailable: function (request, context, inputStream, offset, count) {
|
||||
this.listener.onDataAvailable(...arguments);
|
||||
// If the encoding is not known, store data in an array until we have enough bytes.
|
||||
if (this.encodingArray) {
|
||||
let desired = encodingLength - this.encodingArray.length;
|
||||
let n = Math.min(desired, count);
|
||||
let bytes = new BinaryInput(inputStream).readByteArray(n);
|
||||
offset += n;
|
||||
count -= n;
|
||||
this.encodingArray.push(...bytes);
|
||||
if (n < desired) {
|
||||
// Wait until there is more data.
|
||||
return;
|
||||
}
|
||||
this.determineEncoding(request, context);
|
||||
}
|
||||
|
||||
// Spit back the data if the encoding is UTF-8, otherwise convert it first.
|
||||
if (!this.decoder) {
|
||||
this.listener.onDataAvailable(request, context, inputStream, offset, count);
|
||||
} else {
|
||||
let buffer = new ArrayBuffer(count);
|
||||
new BinaryInput(inputStream).readArrayBuffer(count, buffer);
|
||||
this.convertAndSendBuffer(request, context, buffer);
|
||||
}
|
||||
},
|
||||
|
||||
onStartRequest: function (request, context) {
|
||||
|
|
@ -76,7 +104,7 @@ Converter.prototype = {
|
|||
request.QueryInterface(Ci.nsIChannel);
|
||||
request.contentType = "text/html";
|
||||
|
||||
// JSON enforces UTF-8 charset (see bug 741776).
|
||||
// Don't honor the charset parameter and use UTF-8 (see bug 741776).
|
||||
request.contentCharset = "UTF-8";
|
||||
|
||||
// Changing the content type breaks saving functionality. Fix it.
|
||||
|
|
@ -92,22 +120,79 @@ Converter.prototype = {
|
|||
|
||||
// Initialize stuff.
|
||||
let win = NetworkHelper.getWindowForRequest(request);
|
||||
exportData(win, request);
|
||||
this.data = exportData(win, request);
|
||||
win.addEventListener("DOMContentLoaded", event => {
|
||||
win.addEventListener("contentMessage", onContentMessage, false, true);
|
||||
}, {once: true});
|
||||
|
||||
// Insert the initial HTML code.
|
||||
let converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"]
|
||||
.createInstance(Ci.nsIScriptableUnicodeConverter);
|
||||
converter.charset = "UTF-8";
|
||||
let stream = converter.convertToInputStream(initialHTML(win.document));
|
||||
this.listener.onDataAvailable(request, context, stream, 0, stream.available());
|
||||
// Send the initial HTML code.
|
||||
let bytes = encoder.encode(initialHTML(win.document));
|
||||
this.convertAndSendBuffer(request, context, bytes.buffer);
|
||||
|
||||
// Create an array to store data until the encoding is determined.
|
||||
this.encodingArray = [];
|
||||
},
|
||||
|
||||
onStopRequest: function (request, context, statusCode) {
|
||||
// Flush data.
|
||||
if (this.encodingArray) {
|
||||
this.determineEncoding(request, context, true);
|
||||
} else {
|
||||
this.convertAndSendBuffer(request, context, new ArrayBuffer(0), true);
|
||||
}
|
||||
|
||||
// Stop the request.
|
||||
this.listener.onStopRequest(request, context, statusCode);
|
||||
this.listener = null;
|
||||
this.decoder = null;
|
||||
this.data = null;
|
||||
},
|
||||
|
||||
// Determines the encoding of the response.
|
||||
determineEncoding: function (request, context, flush = false) {
|
||||
// Determine the encoding using the bytes in encodingArray, defaulting to UTF-8.
|
||||
// An initial byte order mark character (U+FEFF) does the trick.
|
||||
// If there is no BOM, since the first character of valid JSON will be ASCII,
|
||||
// the pattern of nulls in the first two bytes can be used instead.
|
||||
// - UTF-16BE: 00 xx or FE FF
|
||||
// - UTF-16LE: xx 00 or FF FE
|
||||
// - UTF-8: anything else.
|
||||
let encoding = "UTF-8";
|
||||
let bytes = this.encodingArray;
|
||||
if (bytes.length >= 2) {
|
||||
if (!bytes[0] && bytes[1] || bytes[0] == 0xFE && bytes[1] == 0xFF) {
|
||||
encoding = "UTF-16BE";
|
||||
} else if (bytes[0] && !bytes[1] || bytes[0] == 0xFF && bytes[1] == 0xFE) {
|
||||
encoding = "UTF-16LE";
|
||||
}
|
||||
}
|
||||
|
||||
// Create a decoder unless the data is already in UTF-8.
|
||||
if (encoding !== "UTF-8") {
|
||||
this.decoder = new TextDecoder(encoding, {ignoreBOM: true});
|
||||
}
|
||||
|
||||
this.data.encoding = encoding;
|
||||
|
||||
// Send the bytes in encodingArray, and remove it.
|
||||
let buffer = new Uint8Array(bytes).buffer;
|
||||
this.convertAndSendBuffer(request, context, buffer, flush);
|
||||
this.encodingArray = null;
|
||||
},
|
||||
|
||||
// Converts an ArrayBuffer to UTF-8 and sends it.
|
||||
convertAndSendBuffer: function (request, context, buffer, flush = false) {
|
||||
// If the encoding is not UTF-8, decode the buffer and encode into UTF-8.
|
||||
if (this.decoder) {
|
||||
let data = this.decoder.decode(buffer, {stream: !flush});
|
||||
buffer = encoder.encode(data).buffer;
|
||||
}
|
||||
|
||||
// Create an input stream that contains the bytes in the buffer.
|
||||
let stream = new BufferStream(buffer, 0, buffer.byteLength);
|
||||
|
||||
// Send the input stream.
|
||||
this.listener.onDataAvailable(request, context, stream, 0, stream.available());
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -177,6 +262,8 @@ function exportData(win, request) {
|
|||
});
|
||||
}
|
||||
data.headers = Cu.cloneInto(headers, win);
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
// Serializes a qualifiedName and an optional set of attributes into an HTML
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ support-files =
|
|||
!/devtools/client/framework/test/shared-head.js
|
||||
|
||||
[browser_jsonview_bug_1380828.js]
|
||||
[browser_jsonview_ignore_charset.js]
|
||||
[browser_jsonview_copy_headers.js]
|
||||
subsuite = clipboard
|
||||
skip-if = (os == 'linux' && bits == 32 && debug) # bug 1328915, disable linux32 debug devtools for timeouts
|
||||
|
|
@ -32,6 +33,7 @@ subsuite = clipboard
|
|||
skip-if = (os == 'linux' && bits == 32 && debug) # bug 1328915, disable linux32 debug devtools for timeouts
|
||||
[browser_jsonview_csp_json.js]
|
||||
[browser_jsonview_empty_object.js]
|
||||
[browser_jsonview_encoding.js]
|
||||
[browser_jsonview_filter.js]
|
||||
[browser_jsonview_invalid_json.js]
|
||||
[browser_jsonview_manifest.js]
|
||||
|
|
@ -42,6 +44,5 @@ skip-if = (os == 'linux' && bits == 32 && debug) # bug 1328915, disable linux32
|
|||
support-files =
|
||||
!/toolkit/content/tests/browser/common/mockTransfer.js
|
||||
[browser_jsonview_slash.js]
|
||||
[browser_jsonview_utf8.js]
|
||||
[browser_jsonview_valid_json.js]
|
||||
[browser_json_refresh.js]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,70 @@
|
|||
/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
|
||||
/* vim: set ts=2 et sw=2 tw=80: */
|
||||
/* Any copyright is dedicated to the Public Domain.
|
||||
* http://creativecommons.org/publicdomain/zero/1.0/ */
|
||||
|
||||
"use strict";
|
||||
|
||||
add_task(function* () {
|
||||
info("Test JSON encoding started");
|
||||
|
||||
const text = Symbol("text");
|
||||
|
||||
const tests = [
|
||||
{
|
||||
"UTF-8 with BOM": "",
|
||||
"UTF-16BE with BOM": "",
|
||||
"UTF-16LE with BOM": "",
|
||||
[text]: ""
|
||||
}, {
|
||||
"UTF-8": "%30",
|
||||
"UTF-16BE": "%00%30",
|
||||
"UTF-16LE": "%30%00",
|
||||
[text]: "0"
|
||||
}, {
|
||||
"UTF-8": "%30%FF",
|
||||
"UTF-16BE": "%00%30%00",
|
||||
"UTF-16LE": "%30%00%00",
|
||||
[text]: "0\uFFFD" // 0<>
|
||||
}, {
|
||||
"UTF-8": "%C3%A0",
|
||||
"UTF-16BE": "%00%E0",
|
||||
"UTF-16LE": "%E0%00",
|
||||
[text]: "\u00E0" // à
|
||||
}, {
|
||||
"UTF-8 with BOM": "%E2%9D%A4",
|
||||
"UTF-16BE with BOM": "%27%64",
|
||||
"UTF-16LE with BOM": "%64%27",
|
||||
[text]: "\u2764" // ❤
|
||||
}, {
|
||||
"UTF-8": "%30%F0%9F%9A%80",
|
||||
"UTF-16BE": "%00%30%D8%3D%DE%80",
|
||||
"UTF-16LE": "%30%00%3D%D8%80%DE",
|
||||
[text]: "0\uD83D\uDE80" // 0🚀
|
||||
}
|
||||
];
|
||||
|
||||
const bom = {
|
||||
"UTF-8": "%EF%BB%BF",
|
||||
"UTF-16BE": "%FE%FF",
|
||||
"UTF-16LE": "%FF%FE"
|
||||
};
|
||||
|
||||
for (let test of tests) {
|
||||
let result = test[text];
|
||||
for (let [encoding, data] of Object.entries(test)) {
|
||||
info("Testing " + JSON.stringify(result) + " encoded in " + encoding + ".");
|
||||
|
||||
if (encoding.endsWith("BOM")) {
|
||||
data = bom[encoding.split(" ")[0]] + data;
|
||||
}
|
||||
|
||||
yield addJsonViewTab("data:application/json," + data);
|
||||
yield selectJsonViewContentTab("rawdata");
|
||||
|
||||
// Check displayed data.
|
||||
let output = yield getElementText(".textPanelBox .data");
|
||||
is(output, result, "The right data has been received.");
|
||||
}
|
||||
}
|
||||
});
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
|
||||
/* vim: set ts=2 et sw=2 tw=80: */
|
||||
/* Any copyright is dedicated to the Public Domain.
|
||||
* http://creativecommons.org/publicdomain/zero/1.0/ */
|
||||
|
||||
"use strict";
|
||||
|
||||
add_task(function* () {
|
||||
info("Test ignored charset parameter started");
|
||||
|
||||
const encodedChar = "%E2%9D%A4"; // In UTF-8 this is a heavy black heart
|
||||
const result = "\u2764"; // ❤
|
||||
const TEST_JSON_URL = "data:application/json;charset=ANSI," + encodedChar;
|
||||
|
||||
yield addJsonViewTab(TEST_JSON_URL);
|
||||
yield selectJsonViewContentTab("rawdata");
|
||||
|
||||
let text = yield getElementText(".textPanelBox .data");
|
||||
is(text, result, "The charset parameter is ignored and UTF-8 is used.");
|
||||
});
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
|
||||
/* vim: set ts=2 et sw=2 tw=80: */
|
||||
/* Any copyright is dedicated to the Public Domain.
|
||||
* http://creativecommons.org/publicdomain/zero/1.0/ */
|
||||
|
||||
"use strict";
|
||||
|
||||
// In UTF-8 this is a heavy black heart.
|
||||
const encodedChar = "%E2%9D%A4";
|
||||
|
||||
add_task(function* () {
|
||||
info("Test UTF-8 JSON started");
|
||||
|
||||
info("Test 1: UTF-8 is used by default");
|
||||
yield testUrl("data:application/json,[\"" + encodedChar + "\"]");
|
||||
|
||||
info("Test 2: The charset parameter is ignored");
|
||||
yield testUrl("data:application/json;charset=ANSI,[\"" + encodedChar + "\"]");
|
||||
|
||||
info("Test 3: The UTF-8 BOM is tolerated.");
|
||||
const bom = "%EF%BB%BF";
|
||||
yield testUrl("data:application/json," + bom + "[\"" + encodedChar + "\"]");
|
||||
});
|
||||
|
||||
function* testUrl(TEST_JSON_URL) {
|
||||
yield addJsonViewTab(TEST_JSON_URL);
|
||||
|
||||
let countBefore = yield getElementCount(".jsonPanelBox .treeTable .treeRow");
|
||||
is(countBefore, 1, "There must be one row.");
|
||||
|
||||
let objectCellCount = yield getElementCount(
|
||||
".jsonPanelBox .treeTable .stringCell");
|
||||
is(objectCellCount, 1, "There must be one string cell.");
|
||||
|
||||
let objectCellText = yield getElementText(
|
||||
".jsonPanelBox .treeTable .stringCell");
|
||||
is(objectCellText, JSON.stringify(decodeURIComponent(encodedChar)),
|
||||
"The source has been parsed as UTF-8, ignoring the charset parameter.");
|
||||
}
|
||||
Загрузка…
Ссылка в новой задаче