mozilla
/
gecko-dev
зеркало из https://github.com/mozilla/gecko-dev.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 1077354 - Transfer buffers to OS.File methods rather than passing a pointer r=froydnj

This commit is contained in:
Jon Coppeard 2015-01-08 11:31:21 +00:00
Родитель 37e0662101
Коммит 0e23adfc34
11 изменённых файлов: 119 добавлений и 200 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

8
toolkit/components/osfile/modules/osfile_async_front.jsm
Просмотреть файл

@ -647,10 +647,6 @@ File.prototype = {
options = clone(options, ["outExecutionDuration"]);
options.bytes = buffer.byteLength;
}
// Note: Type.void_t.out_ptr.toMsg ensures that
// - the buffer is effectively shared (not neutered) between both
// threads;
// - we take care of any |byteOffset|.
return Scheduler.post("File_prototype_write",
[this._fdmsg,
Type.void_t.in_ptr.toMsg(buffer),
@ -1162,10 +1158,6 @@ File.writeAtomic = function writeAtomic(path, buffer, options = {}) {
if (isTypedArray(buffer) && (!("bytes" in options))) {
options.bytes = buffer.byteLength;
};
// Note: Type.void_t.out_ptr.toMsg ensures that
// - the buffer is effectively shared (not neutered) between both
// threads;
// - we take care of any |byteOffset|.
let refObj = {};
TelemetryStopwatch.start("OSFILE_WRITEATOMIC_JANK_MS", refObj);
let promise = Scheduler.post("writeAtomic",

159
toolkit/components/osfile/modules/osfile_shared_allthreads.jsm
Просмотреть файл

@ -25,6 +25,15 @@
const Cu = typeof Components != "undefined" ? Components.utils : undefined;
const Ci = typeof Components != "undefined" ? Components.interfaces : undefined;
const Cc = typeof Components != "undefined" ? Components.classes : undefined;
/**
* A constructor for messages that require transfers instead of copies.
*
* See BasePromiseWorker.Meta.
*
* @constructor
*/
let Meta;
if (typeof Components != "undefined") {
// Global definition of |exports|, to keep everybody happy.
// In non-main thread, |exports| is provided by the module
@ -32,6 +41,10 @@ if (typeof Components != "undefined") {
this.exports = {};
Cu.import("resource://gre/modules/Services.jsm", this);
Meta = Cu.import("resource://gre/modules/PromiseWorker.jsm", {}).BasePromiseWorker.Meta;
} else {
importScripts("resource://gre/modules/workers/require.js");
Meta = require("resource://gre/modules/workers/PromiseWorker.js").Meta;
}
let EXPORTED_SYMBOLS = [
@ -46,11 +59,11 @@ let EXPORTED_SYMBOLS = [
"declareFFI",
"declareLazy",
"declareLazyFFI",
"normalizeToPointer",
"normalizeBufferArgs",
"projectValue",
"isArrayBuffer",
"isTypedArray",
"defineLazyGetter",
"offsetBy",
"OS" // Warning: this exported symbol will disappear
];
@ -141,7 +154,15 @@ let stringifyArg = function stringifyArg(arg) {
* compartment.
*/
if (argToString === "[object Object]") {
return JSON.stringify(arg);
return JSON.stringify(arg, function(key, value) {
if (isTypedArray(value)) {
return "["+ value.constructor.name + " " + value.byteOffset + " " + value.byteLength + "]";
}
if (isArrayBuffer(arg)) {
return "[" + value.constructor.name + " " + value.byteLength + "]";
}
return value;
});
} else {
return argToString;
}
@ -387,11 +408,20 @@ Type.prototype = {
* Utility function used to determine whether an object is a typed array
*/
let isTypedArray = function isTypedArray(obj) {
return typeof obj == "object"
return obj != null && typeof obj == "object"
&& "byteOffset" in obj;
};
exports.isTypedArray = isTypedArray;
/**
* Utility function used to determine whether an object is an ArrayBuffer.
*/
let isArrayBuffer = function(obj) {
return obj != null && typeof obj == "object" &&
obj.constructor.name == "ArrayBuffer";
};
exports.isArrayBuffer = isArrayBuffer;
/**
* A |Type| of pointers.
*
@ -430,13 +460,16 @@ PtrType.prototype.toMsg = function ptr_toMsg(value) {
if (typeof value == "string") {
return { string: value };
}
if (isTypedArray(value)) {
// Automatically transfer typed arrays
return new Meta({data: value}, {transfers: [value.buffer]});
}
if (isArrayBuffer(value)) {
// Automatically transfer array buffers
return new Meta({data: value}, {transfers: [value]});
}
let normalized;
if (isTypedArray(value)) { // Typed array
normalized = Type.uint8_t.in_ptr.implementation(value.buffer);
if (value.byteOffset != 0) {
normalized = offsetBy(normalized, value.byteOffset);
}
} else if ("addressOfElement" in value) { // C array
if ("addressOfElement" in value) { // C array
normalized = value.addressOfElement(0);
} else if ("isNull" in value) { // C pointer
normalized = value;
@ -458,6 +491,9 @@ PtrType.prototype.fromMsg = function ptr_fromMsg(msg) {
if ("string" in msg) {
return msg.string;
}
if ("data" in msg) {
return msg.data;
}
if ("ptr" in msg) {
let address = ctypes.uintptr_t(msg.ptr);
return this.cast(address);
@ -1144,96 +1180,34 @@ function declareLazy(object, field, lib, ...declareArgs) {
}
exports.declareLazy = declareLazy;
// A bogus array type used to perform pointer arithmetics
let gOffsetByType;
/**
* Advance a pointer by a number of items.
* Utility function used to sanity check buffer and length arguments. The
* buffer must be a Typed Array.
*
* This method implements adding an integer to a pointer in C.
*
* Example:
* // ptr is a uint16_t*,
* offsetBy(ptr, 3)
* // returns a uint16_t* with the address ptr + 3 * 2 bytes
*
* @param {C pointer} pointer The start pointer.
* @param {number} length The number of items to advance. Must not be
* negative.
*
* @return {C pointer} |pointer| advanced by |length| items
*/
let offsetBy =
function offsetBy(pointer, length) {
if (length === undefined || length < 0) {
throw new TypeError("offsetBy expects a positive number");
}
if (!("isNull" in pointer)) {
throw new TypeError("offsetBy expects a pointer");
}
if (length == 0) {
return pointer;
}
let type = pointer.constructor;
let size = type.targetType.size;
if (size == 0 || size == null) {
throw new TypeError("offsetBy cannot be applied to a pointer without size");
}
let bytes = length * size;
if (!gOffsetByType || gOffsetByType.size <= bytes) {
gOffsetByType = ctypes.uint8_t.array(bytes * 2);
}
let addr = ctypes.cast(pointer, gOffsetByType.ptr).
contents.addressOfElement(bytes);
return ctypes.cast(addr, type);
};
exports.offsetBy = offsetBy;
/**
* Utility function used to normalize a Typed Array or C
* pointer into a uint8_t C pointer.
*
* Future versions might extend this to other data structures.
*
* @param {Typed array | C pointer} candidate The buffer. If
* a C pointer, it must be non-null.
* @param {Typed array} candidate The buffer.
* @param {number} bytes The number of bytes that |candidate| should contain.
* Used for sanity checking if the size of |candidate| can be determined.
*
* @return {ptr:{C pointer}, bytes:number} A C pointer of type uint8_t,
* corresponding to the start of |candidate|.
* @return number The bytes argument clamped to the length of the buffer.
*/
function normalizeToPointer(candidate, bytes) {
function normalizeBufferArgs(candidate, bytes) {
if (!candidate) {
throw new TypeError("Expecting a Typed Array or a C pointer");
throw new TypeError("Expecting a Typed Array");
}
let ptr;
if ("isNull" in candidate) {
if (candidate.isNull()) {
throw new TypeError("Expecting a non-null pointer");
}
ptr = Type.uint8_t.out_ptr.cast(candidate);
if (bytes == null) {
throw new TypeError("C pointer missing bytes indication.");
}
} else if (isTypedArray(candidate)) {
// Typed Array
ptr = Type.uint8_t.out_ptr.implementation(candidate.buffer);
if (bytes == null) {
bytes = candidate.byteLength;
} else if (candidate.byteLength < bytes) {
throw new TypeError("Buffer is too short. I need at least " +
bytes +
" bytes but I have only " +
candidate.byteLength +
"bytes");
}
} else {
throw new TypeError("Expecting a Typed Array or a C pointer");
if (!isTypedArray(candidate)) {
throw new TypeError("Expecting a Typed Array");
}
return {ptr: ptr, bytes: bytes};
if (bytes == null) {
bytes = candidate.byteLength;
} else if (candidate.byteLength < bytes) {
throw new TypeError("Buffer is too short. I need at least " +
bytes +
" bytes but I have only " +
candidate.byteLength +
"bytes");
}
return bytes;
};
exports.normalizeToPointer = normalizeToPointer;
exports.normalizeBufferArgs = normalizeBufferArgs;
///////////////////// OS interactions
@ -1274,8 +1248,7 @@ exports.OS = {
declareFFI: declareFFI,
projectValue: projectValue,
isTypedArray: isTypedArray,
defineLazyGetter: defineLazyGetter,
offsetBy: offsetBy
defineLazyGetter: defineLazyGetter
}
};

36
toolkit/components/osfile/modules/osfile_shared_front.jsm
Просмотреть файл

@ -62,23 +62,24 @@ AbstractFile.prototype = {
options = clone(maybeBytes);
maybeBytes = null;
} else {
options = clone(options || {});
options = options || {};
}
if(!("bytes" in options)) {
options.bytes = maybeBytes == null ? this.stat().size : maybeBytes;
let bytes = options.bytes || undefined;
if (bytes === undefined) {
bytes = maybeBytes == null ? this.stat().size : maybeBytes;
}
let buffer = new Uint8Array(options.bytes);
let {ptr, bytes} = SharedAll.normalizeToPointer(buffer, options.bytes);
let buffer = new Uint8Array(bytes);
let pos = 0;
while (pos < bytes) {
let chunkSize = this._read(ptr, bytes - pos, options);
let length = bytes - pos;
let view = new DataView(buffer.buffer, pos, length);
let chunkSize = this._read(view, length, options);
if (chunkSize == 0) {
break;
}
pos += chunkSize;
ptr = SharedAll.offsetBy(ptr, chunkSize);
}
if (pos == options.bytes) {
if (pos == bytes) {
return buffer;
} else {
return buffer.subarray(0, pos);
@ -91,27 +92,24 @@ AbstractFile.prototype = {
* Note that, by default, this function may perform several I/O
* operations to ensure that the buffer is fully written.
*
* @param {Typed array | C pointer} buffer The buffer in which the
* the bytes are stored. The buffer must be large enough to
* accomodate |bytes| bytes.
* @param {Typed array} buffer The buffer in which the the bytes are
* stored. The buffer must be large enough to accomodate |bytes| bytes.
* @param {*=} options Optionally, an object that may contain the
* following fields:
* - {number} bytes The number of |bytes| to write from the buffer. If
* unspecified, this is |buffer.byteLength|. Note that |bytes| is required
* if |buffer| is a C pointer.
* unspecified, this is |buffer.byteLength|.
*
* @return {number} The number of bytes actually written.
*/
write: function write(buffer, options = {}) {
let {ptr, bytes} =
SharedAll.normalizeToPointer(buffer, ("bytes" in options) ? options.bytes : undefined);
let bytes =
SharedAll.normalizeBufferArgs(buffer, ("bytes" in options) ? options.bytes : undefined);
let pos = 0;
while (pos < bytes) {
let chunkSize = this._write(ptr, bytes - pos, options);
let length = bytes - pos;
let view = new DataView(buffer.buffer, buffer.byteOffset + pos, length);
let chunkSize = this._write(view, length, options);
pos += chunkSize;
ptr = SharedAll.offsetBy(ptr, chunkSize);
}
return pos;
}

2
toolkit/components/osfile/modules/osfile_unix_front.jsm
Просмотреть файл

@ -112,7 +112,7 @@
/**
* Write some bytes to a file.
*
* @param {C pointer} buffer A buffer holding the data that must be
* @param {Typed array} buffer A buffer holding the data that must be
* written.
* @param {number} nbytes The number of bytes to write. It must not
* exceed the size of |buffer| in bytes.

2
toolkit/components/osfile/modules/osfile_win_front.jsm
Просмотреть файл

@ -125,7 +125,7 @@
/**
* Write some bytes to a file.
*
* @param {C pointer} buffer A buffer holding the data that must be
* @param {Typed array} buffer A buffer holding the data that must be
* written.
* @param {number} nbytes The number of bytes to write. It must not
* exceed the size of |buffer| in bytes.

12
toolkit/components/osfile/tests/mochi/worker_test_osfile_comms.js
Просмотреть файл

@ -36,10 +36,8 @@ self.onmessage = function(msg) {
})(),
type: OS.Shared.Type.char.in_ptr,
check: function check_ArrayBuffer(candidate, prefix) {
let cast = ctypes.cast(candidate, ctypes.uint8_t.ptr);
for (let i = 0; i < 15; ++i) {
is(cast.contents, i % 256, prefix + "Checking that the contents of the ArrayBuffer were preserved");
cast = cast.increment();
is(candidate[i], i % 256, prefix + "Checking that the contents of the ArrayBuffer were preserved");
}
}},
{ typename: "OS.Shared.Type.char.in_ptr",
@ -106,6 +104,11 @@ self.onmessage = function(msg) {
return;
}
if ("data" in serialized) {
// Unwrap from `Meta`
serialized = serialized.data;
}
// 2. Test deserialization
let deserialized;
try {
@ -120,7 +123,8 @@ self.onmessage = function(msg) {
}
// 3. Local test deserialized value
info("Running test on deserialized value " + serialized);
info("Running test on deserialized value " + deserialized +
" aka " + JSON.stringify(deserialized));
check(deserialized, "Local test: ");
// 4. Test sending serialized

53
toolkit/components/osfile/tests/mochi/worker_test_osfile_front.js
Просмотреть файл

@ -22,7 +22,6 @@ self.onmessage = function onmessage_start(msg) {
};
try {
test_init();
test_offsetby();
test_open_existing_file();
test_open_non_existing_file();
test_flush_open_file();
@ -48,58 +47,6 @@ function test_init() {
importScripts("resource://gre/modules/osfile.jsm");
}
function test_offsetby() {
info("Starting test_offsetby");
// Initialize one array
let LENGTH = 1024;
let buf = new ArrayBuffer(LENGTH);
let view = new Uint8Array(buf);
let i;
for (i = 0; i < LENGTH; ++i) {
view[i] = i;
}
// Walk through the array with offsetBy by 8 bits
let uint8 = SharedAll.Type.uint8_t.in_ptr.implementation(buf);
for (i = 0; i < LENGTH; ++i) {
let value = SharedAll.offsetBy(uint8, i).contents;
if (value != i%256) {
is(value, i % 256, "test_offsetby: Walking through array with offsetBy (8 bits)");
break;
}
}
// Walk again by 16 bits
let uint16 = SharedAll.Type.uint16_t.in_ptr.implementation(buf);
let view2 = new Uint16Array(buf);
for (i = 0; i < LENGTH/2; ++i) {
let value = SharedAll.offsetBy(uint16, i).contents;
if (value != view2[i]) {
is(value, view2[i], "test_offsetby: Walking through array with offsetBy (16 bits)");
break;
}
}
// Ensure that offsetBy(..., 0) is idempotent
let startptr = SharedAll.offsetBy(uint8, 0);
let startptr2 = SharedAll.offsetBy(startptr, 0);
is(startptr.toString(), startptr2.toString(), "test_offsetby: offsetBy(..., 0) is idmpotent");
// Ensure that offsetBy(ptr, ...) does not work if ptr is a void*
let ptr = ctypes.voidptr_t(0);
let exn;
try {
SharedAll.offsetBy(ptr, 1);
} catch (x) {
exn = x;
}
ok(!!exn, "test_offsetby: rejected offsetBy with void*");
info("test_offsetby: complete");
}
/**
* Test that we can open an existing file.
*/

8
toolkit/components/osfile/tests/xpcshell/test_compression.js
Просмотреть файл

@ -12,19 +12,21 @@ function run_test() {
add_task(function test_compress_lz4() {
let path = OS.Path.join(OS.Constants.Path.tmpDir, "compression.lz");
let array = new Uint8Array(1024);
let length = 1024;
let array = new Uint8Array(length);
for (let i = 0; i < array.byteLength; ++i) {
array[i] = i;
}
let arrayAsString = Array.prototype.join.call(array);
do_print("Writing data with lz4 compression");
let bytes = yield OS.File.writeAtomic(path, array, { compression: "lz4" });
do_print("Compressed " + array.byteLength + " bytes into " + bytes);
do_print("Compressed " + length + " bytes into " + bytes);
do_print("Reading back with lz4 decompression");
let decompressed = yield OS.File.read(path, { compression: "lz4" });
do_print("Decompressed into " + decompressed.byteLength + " bytes");
do_check_eq(Array.prototype.join.call(array), Array.prototype.join.call(decompressed));
do_check_eq(arrayAsString, Array.prototype.join.call(decompressed));
});
add_task(function test_uncompressed() {

15
toolkit/components/osfile/tests/xpcshell/test_read_write.js
Просмотреть файл

@ -41,9 +41,10 @@ add_test_pair(function* read_write_all() {
let pathSource = OS.Path.join(currentDir, EXISTING_FILE);
let contents = yield OS.File.read(pathSource);
do_check_true(!!contents); // Content is not empty
let bytesRead = contents.byteLength;
let bytesWritten = yield OS.File.writeAtomic(DEST_PATH, contents, options);
do_check_eq(contents.byteLength, bytesWritten); // Correct number of bytes written
do_check_eq(bytesRead, bytesWritten); // Correct number of bytes written
// Check that options are not altered
do_check_eq(JSON.stringify(options), JSON.stringify(optionsBackup));
@ -54,6 +55,7 @@ add_test_pair(function* read_write_all() {
// Check that writeAtomic fails if noOverwrite is true and the destination
// file already exists!
contents = new Uint8Array(300);
let view = new Uint8Array(contents.buffer, 10, 200);
try {
let opt = JSON.parse(JSON.stringify(options));
@ -71,16 +73,17 @@ add_test_pair(function* read_write_all() {
// Now write a subset
let START = 10;
let LENGTH = 100;
contents = new Uint8Array(300);
for (var i = 0; i < contents.byteLength; i++)
contents[i] = i % 256;
view = new Uint8Array(contents.buffer, START, LENGTH);
bytesWritten = yield OS.File.writeAtomic(DEST_PATH, view, options);
do_check_eq(bytesWritten, LENGTH);
let array2 = yield OS.File.read(DEST_PATH);
let view1 = new Uint8Array(contents.buffer, START, LENGTH);
do_check_eq(view1.length, array2.length);
let decoder = new TextDecoder();
do_check_eq(decoder.decode(view1), decoder.decode(array2));
do_check_eq(LENGTH, array2.length);
for (var i = 0; i < LENGTH; i++)
do_check_eq(array2[i], (i + START) % 256);
// Cleanup.
yield OS.File.remove(DEST_PATH);

20
toolkit/components/workerlz4/lz4.js
Просмотреть файл

@ -93,39 +93,39 @@ function compressFileContent(array, options = {}) {
exports.compressFileContent = compressFileContent;
function decompressFileContent(array, options = {}) {
let {ptr, bytes} = SharedAll.normalizeToPointer(array, options.bytes || null);
let bytes = SharedAll.normalizeBufferArgs(array, options.bytes || null);
if (bytes < HEADER_SIZE) {
throw new LZError("decompress", "becauseLZNoHeader", "Buffer is too short (no header)");
}
// Read headers
let expectMagicNumber = ctypes.cast(ptr, EXPECTED_HEADER_TYPE.ptr).contents;
let expectMagicNumber = new DataView(array.buffer, 0, MAGIC_NUMBER.byteLength);
for (let i = 0; i < MAGIC_NUMBER.byteLength; ++i) {
if (expectMagicNumber[i] != MAGIC_NUMBER[i]) {
if (expectMagicNumber.getUint8(i) != MAGIC_NUMBER[i]) {
throw new LZError("decompress", "becauseLZWrongMagicNumber", "Invalid header (no magic number");
}
}
let sizeBuf =
ctypes.cast(
SharedAll.offsetBy(ptr, MAGIC_NUMBER.byteLength),
EXPECTED_SIZE_BUFFER_TYPE.ptr).contents;
let sizeBuf = new DataView(array.buffer, MAGIC_NUMBER.byteLength, BYTES_IN_SIZE_HEADER);
let expectDecompressedSize =
sizeBuf[0] + (sizeBuf[1] << 8) + (sizeBuf[2] << 16) + (sizeBuf[3] << 24);
sizeBuf.getUint8(0) +
(sizeBuf.getUint8(1) << 8) +
(sizeBuf.getUint8(2) << 16) +
(sizeBuf.getUint8(3) << 24);
if (expectDecompressedSize == 0) {
// The underlying algorithm cannot handle a size of 0
return new Uint8Array(0);
}
// Prepare the input buffer
let inputPtr = SharedAll.offsetBy(ptr, HEADER_SIZE);
let inputData = new DataView(array.buffer, HEADER_SIZE);
// Prepare the output buffer
let outputBuffer = new Uint8Array(expectDecompressedSize);
let decompressedBytes = (new SharedAll.Type.size_t.implementation(0));
// Decompress
let success = Internals.decompress(inputPtr, bytes - HEADER_SIZE,
let success = Internals.decompress(inputData, bytes - HEADER_SIZE,
outputBuffer, outputBuffer.byteLength,
decompressedBytes.address());
if (!success) {

4
toolkit/modules/ZipUtils.jsm
Просмотреть файл

@ -52,7 +52,6 @@ function saveStreamAsync(aPath, aStream, aFile) {
createInstance(Ci.nsIBinaryInputStream);
source.setInputStream(input);
let data = new Uint8Array(EXTRACTION_BUFFER);
function readFailed(error) {
try {
@ -72,7 +71,8 @@ function saveStreamAsync(aPath, aStream, aFile) {
function readData() {
try {
let count = Math.min(source.available(), data.byteLength);
let count = Math.min(source.available(), EXTRACTION_BUFFER);
let data = new Uint8Array(count);
source.readArrayBuffer(count, data.buffer);
aFile.write(data, { bytes: count }).then(function() {