Bug 854169 - Bindings to lz4 for chrome workers. r=froydnj

toolkit/components/workerlz4/lz4.cpp

@@ -0,0 +1,73 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/Compression.h"
+
+/**
+ * LZ4 is a very fast byte-wise compression algorithm.
+ *
+ * Compared to Google's Snappy it is faster to compress and decompress and
+ * generally produces output of about the same size.
+ *
+ * Compared to zlib it compresses at about 10x the speed, decompresses at about
+ * 4x the speed and produces output of about 1.5x the size.
+ *
+ */
+
+using namespace mozilla::Compression;
+
+/**
+ * Compresses 'inputSize' bytes from 'source' into 'dest'.
+ * Destination buffer must be already allocated,
+ * and must be sized to handle worst cases situations (input data not compressible)
+ * Worst case size evaluation is provided by function LZ4_compressBound()
+ *
+ * @param inputSize is the input size. Max supported value is ~1.9GB
+ * @param return the number of bytes written in buffer dest
+ */
+extern "C" MOZ_EXPORT size_t
+workerlz4_compress(const char* source, size_t inputSize, char* dest) {
+  return LZ4::compress(source, inputSize, dest);
+}
+
+/**
+ * If the source stream is malformed, the function will stop decoding
+ * and return a negative result, indicating the byte position of the
+ * faulty instruction
+ *
+ * This function never writes outside of provided buffers, and never
+ * modifies input buffer.
+ *
+ * note : destination buffer must be already allocated.
+ *        its size must be a minimum of 'outputSize' bytes.
+ * @param outputSize is the output size, therefore the original size
+ * @return true/false
+ */
+extern "C" MOZ_EXPORT int
+workerlz4_decompress(const char* source, size_t inputSize,
+                     char* dest, size_t maxOutputSize,
+                     size_t *bytesOutput) {
+  return LZ4::decompress(source, inputSize,
+                         dest, maxOutputSize,
+                         bytesOutput);
+}
+
+
+/*
+  Provides the maximum size that LZ4 may output in a "worst case"
+  scenario (input data not compressible) primarily useful for memory
+  allocation of output buffer.
+  note : this function is limited by "int" range (2^31-1)
+
+  @param inputSize is the input size. Max supported value is ~1.9GB
+  @return maximum output size in a "worst case" scenario
+*/
+extern "C" MOZ_EXPORT size_t
+workerlz4_maxCompressedSize(size_t inputSize)
+{
+  return LZ4::maxCompressedSize(inputSize);
+}
+
+
+

toolkit/components/workerlz4/lz4.js

@@ -0,0 +1,136 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+if (typeof Components != "undefined") {
+  throw new Error("This file is meant to be loaded in a worker");
+}
+if (!module || !exports) {
+  throw new Error("Please load this module with require()");
+}
+
+const SharedAll = require("resource://gre/modules/osfile/osfile_shared_allthreads.jsm");
+const Internals = require("resource://gre/modules/workers/lz4_internal.js");
+
+const MAGIC_NUMBER = new Uint8Array([109, 111, 122, 76, 122, 52, 48, 0]); // "mozLz4a\0"
+
+const BYTES_IN_SIZE_HEADER = ctypes.uint32_t.size;
+
+const HEADER_SIZE = MAGIC_NUMBER.byteLength + BYTES_IN_SIZE_HEADER;
+
+const EXPECTED_HEADER_TYPE = new ctypes.ArrayType(ctypes.uint8_t, HEADER_SIZE);
+const EXPECTED_SIZE_BUFFER_TYPE = new ctypes.ArrayType(ctypes.uint8_t, BYTES_IN_SIZE_HEADER);
+
+/**
+ * An error during (de)compression
+ *
+ * @param {string} operation The name of the operation ("compress", "decompress")
+ * @param {string} reason A reason to be used when matching errors. Must start
+ * with "because", e.g. "becauseInvalidContent".
+ * @param {string} message A human-readable message.
+ */
+function LZError(operation, reason, message) {
+  SharedAll.OSError.call(this);
+  this.operation = operation;
+  this[reason] = true;
+  this.message = message;
+}
+LZError.prototype = Object.create(SharedAll.OSError);
+LZError.prototype.toString = function toString() {
+  return this.message;
+};
+exports.Error = LZError;
+
+/**
+ * Compress a block to a form suitable for writing to disk.
+ *
+ * Compatibility note: For the moment, we are basing our code on lz4
+ * 1.3, which does not specify a *file* format. Therefore, we define
+ * our own format. Once lz4 defines a complete file format, we will
+ * migrate both |compressFileContent| and |decompressFileContent| to this file
+ * format. For backwards-compatibility, |decompressFileContent| will however
+ * keep the ability to decompress files provided with older versions of
+ * |compressFileContent|.
+ *
+ * Compressed files have the following layout:
+ *
+ * | MAGIC_NUMBER (8 bytes) | content size (uint32_t, little endian) | content, as obtained from lz4_compress |
+ *
+ * @param {TypedArray|void*} buffer The buffer to write to the disk.
+ * @param {object=} options An object that may contain the following fields:
+ *  - {number} bytes The number of bytes to read from |buffer|. If |buffer|
+ *    is an |ArrayBuffer|, |bytes| defaults to |buffer.byteLength|. If
+ *    |buffer| is a |void*|, |bytes| MUST be provided.
+ * @return {Uint8Array} An array of bytes suitable for being written to the
+ * disk.
+ */
+function compressFileContent(array, options = {}) {
+  // Prepare the output array
+  let inputBytes;
+  if (SharedAll.isTypedArray(array) && !(options && "bytes" in options)) {
+    inputBytes = array.byteLength;
+  } else if (options && options.bytes) {
+    inputBytes = options.bytes;
+  } else {
+    throw new TypeError("compressFileContent requires a size");
+  }
+  let maxCompressedSize = Internals.maxCompressedSize(inputBytes);
+  let outputArray = new Uint8Array(HEADER_SIZE + maxCompressedSize);
+
+  // Compress to output array
+  let payload = new Uint8Array(outputArray.buffer, outputArray.byteOffset + HEADER_SIZE);
+  let compressedSize = Internals.compress(array, inputBytes, payload);
+
+  // Add headers
+  outputArray.set(MAGIC_NUMBER);
+  let view = new DataView(outputArray.buffer);
+  view.setUint32(MAGIC_NUMBER.byteLength, inputBytes, true);
+
+  return new Uint8Array(outputArray.buffer, 0, HEADER_SIZE + compressedSize);
+}
+exports.compressFileContent = compressFileContent;
+
+function decompressFileContent(array, options = {}) {
+  let {ptr, bytes} = SharedAll.normalizeToPointer(array, options.bytes);
+  if (bytes < HEADER_SIZE) {
+    throw new LZError("decompress", "becauseLZNoHeader", "Buffer is too short (no header)");
+  }
+
+  // Read headers
+  let expectMagicNumber = ctypes.cast(ptr, EXPECTED_HEADER_TYPE.ptr).contents;
+  for (let i = 0; i < MAGIC_NUMBER.byteLength; ++i) {
+    if (expectMagicNumber[i] != MAGIC_NUMBER[i]) {
+      throw new LZError("decompress", "becauseLZWrongMagicNumber", "Invalid header (no magic number");
+    }
+  }
+
+  let sizeBuf =
+    ctypes.cast(
+      SharedAll.offsetBy(ptr, MAGIC_NUMBER.byteLength),
+      EXPECTED_SIZE_BUFFER_TYPE.ptr).contents;
+  let expectDecompressedSize =
+    sizeBuf[0] + (sizeBuf[1] << 8) + (sizeBuf[2] << 16) + (sizeBuf[3] << 24);
+  if (expectDecompressedSize == 0) {
+    // The underlying algorithm cannot handle a size of 0
+    return new Uint8Array(0);
+  }
+
+  // Prepare the input buffer
+  let inputPtr = SharedAll.offsetBy(ptr, HEADER_SIZE);
+
+  // Prepare the output buffer
+  let outputBuffer = new Uint8Array(expectDecompressedSize);
+  let decompressedBytes = (new SharedAll.Type.size_t.implementation(0));
+
+  // Decompress
+  let success = Internals.decompress(inputPtr, bytes - HEADER_SIZE,
+                                     outputBuffer, outputBuffer.byteLength,
+                                     decompressedBytes.address());
+  if (!success) {
+    throw new LZError("decompress", "becauseLZInvalidContent", "Invalid content:Decompression stopped at " + decompressedBytes.value);
+  }
+  return new Uint8Array(outputBuffer.buffer, outputBuffer.byteOffset, decompressedBytes.value);
+}
+exports.decompressFileContent = decompressFileContent;

toolkit/components/workerlz4/lz4_internal.js

@@ -0,0 +1,58 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+if (typeof Components != "undefined") {
+  throw new Error("This file is meant to be loaded in a worker");
+}
+if (!module || !exports) {
+  throw new Error("Please load this module with require()");
+}
+
+let SharedAll = require("resource://gre/modules/osfile/osfile_shared_allthreads.jsm");
+let libxul = ctypes.open(SharedAll.Constants.Path.libxul);
+let declareLazyFII = SharedAll.declareLazyFFI;
+let Type = SharedAll.Type;
+
+let Primitives = {};
+
+declareLazyFII(Primitives, "compress", libxul,
+  "workerlz4_compress",
+  null,
+  /*return*/ Type.size_t,
+  /*const source*/ Type.void_t.in_ptr,
+  /*inputSize*/ Type.size_t,
+  /*dest*/ Type.void_t.out_ptr
+);
+
+declareLazyFII(Primitives, "decompress", libxul,
+  "workerlz4_decompress",
+  null,
+  /*return*/ Type.int,
+  /*const source*/ Type.void_t.in_ptr,
+  /*inputSize*/ Type.size_t,
+  /*dest*/ Type.void_t.out_ptr,
+  /*maxOutputSize*/ Type.size_t,
+  /*actualOutputSize*/ Type.size_t.out_ptr
+);
+
+declareLazyFII(Primitives, "maxCompressedSize", libxul,
+  "workerlz4_maxCompressedSize",
+  null,
+  /*return*/ Type.size_t,
+  /*inputSize*/ Type.size_t
+);
+
+module.exports = {
+  get compress() {
+    return Primitives.compress;
+  },
+  get decompress() {
+    return Primitives.decompress;
+  },
+  get maxCompressedSize() {
+    return Primitives.maxCompressedSize;
+  }
+};