mozilla
/
gecko-dev
зеркало из https://github.com/mozilla/gecko-dev.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 1660835: Add LZ4 compression options to IOUtils read and write methods r=barret,Gijs 

NB: This change breaks the IOUtils.read API, requiring that an options
dictionary is passed as the optional second argument, rather than a number
indicating the max bytes to read. This option is not used out of tests however.

Differential Revision: https://phabricator.services.mozilla.com/D88177
This commit is contained in:
Keefer Rourke 2020-08-28 00:35:15 +00:00
Родитель f283542454
Коммит b09555373a
6 изменённых файлов: 412 добавлений и 37 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

40
dom/chrome-webidl/IOUtils.webidl
Просмотреть файл

@ -24,16 +24,14 @@
[ChromeOnly, Exposed=(Window, Worker)]
namespace IOUtils {
/**
* Reads up to |maxBytes| of the file at |path|. If |maxBytes| is unspecified,
* the entire file is read.
* Reads up to |maxBytes| of the file at |path| according to |opts|.
*
* @param path An absolute file path.
* @param maxBytes The max bytes to read from the file at path.
* @param path An absolute file path.
*
* @return Resolves with an array of unsigned byte values read from disk,
* otherwise rejects with a DOMException.
*/
Promise<Uint8Array> read(DOMString path, optional unsigned long maxBytes);
Promise<Uint8Array> read(DOMString path, optional ReadOptions opts = {});
/**
* Reads the UTF-8 text file located at |path| and returns the decoded
* contents as a |DOMString|.
@ -43,7 +41,7 @@ namespace IOUtils {
* @return Resolves with the file contents encoded as a string, otherwise
* rejects with a DOMException.
*/
Promise<DOMString> readUTF8(DOMString path);
Promise<DOMString> readUTF8(DOMString path, optional ReadUTF8Options opts = {});
/**
* Attempts to safely write |data| to a file at |path|.
*
@ -160,7 +158,31 @@ namespace IOUtils {
};
/**
* Options to be passed to the |IOUtils.writeAtomic| method.
* Options to be passed to the |IOUtils.readUTF8| method.
*/
dictionary ReadUTF8Options {
/**
* If true, this option indicates that the file to be read is compressed with
* LZ4-encoding, and should be decompressed before the data is returned to
* the caller.
*/
boolean decompress = false;
};
/**
* Options to be passed to the |IOUtils.read| method.
*/
dictionary ReadOptions : ReadUTF8Options {
/**
* The max bytes to read from the file at path. If unspecified, the entire
* file will be read. This option is incompatible with |decompress|.
*/
unsigned long? maxBytes = null;
};
/**
* Options to be passed to the |IOUtils.writeAtomic| and |writeAtomicUTF8|
* methods.
*/
dictionary WriteAtomicOptions {
/**
@ -185,6 +207,10 @@ dictionary WriteAtomicOptions {
* disconnection before the buffers are flushed.
*/
boolean flush = false;
/**
* If true, compress the data with LZ4-encoding before writing to the file.
*/
boolean compress = false;
};
/**

154
dom/system/IOUtils.cpp
Просмотреть файл

@ -4,10 +4,14 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include <cstdint>
#include "mozilla/dom/IOUtils.h"
#include "ErrorList.h"
#include "mozilla/dom/IOUtilsBinding.h"
#include "mozilla/dom/Promise.h"
#include "mozilla/Compression.h"
#include "mozilla/EndianUtils.h"
#include "mozilla/ErrorNames.h"
#include "mozilla/ResultExtensions.h"
#include "mozilla/Span.h"
@ -15,6 +19,7 @@
#include "nsError.h"
#include "nsIDirectoryEnumerator.h"
#include "nsPrintfCString.h"
#include "nsTArray.h"
#include "nspr/prerror.h"
#include "nspr/prio.h"
#include "nspr/private/pprio.h"
@ -240,7 +245,7 @@ already_AddRefed<Promise> IOUtils::RunOnBackgroundThread(
/* static */
already_AddRefed<Promise> IOUtils::Read(GlobalObject& aGlobal,
const nsAString& aPath,
const Optional<uint32_t>& aMaxBytes) {
const ReadOptions& aOptions) {
MOZ_DIAGNOSTIC_ASSERT(XRE_IsParentProcess());
RefPtr<Promise> promise = CreateJSPromise(aGlobal);
NS_ENSURE_TRUE(!!promise, nullptr);
@ -250,23 +255,24 @@ already_AddRefed<Promise> IOUtils::Read(GlobalObject& aGlobal,
REJECT_IF_RELATIVE_PATH(aPath, promise);
nsAutoString path(aPath);
Maybe<uint32_t> toRead = Nothing();
if (aMaxBytes.WasPassed()) {
if (aMaxBytes.Value() == 0) {
if (!aOptions.mMaxBytes.IsNull()) {
if (aOptions.mMaxBytes.Value() == 0) {
// Resolve with an empty buffer.
nsTArray<uint8_t> arr(0);
promise->MaybeResolve(TypedArrayCreator<Uint8Array>(arr));
return promise.forget();
}
toRead.emplace(aMaxBytes.Value());
toRead.emplace(aOptions.mMaxBytes.Value());
}
return RunOnBackgroundThread<nsTArray<uint8_t>>(promise, &ReadSync, path,
toRead);
toRead, aOptions.mDecompress);
}
/* static */
already_AddRefed<Promise> IOUtils::ReadUTF8(GlobalObject& aGlobal,
const nsAString& aPath) {
const nsAString& aPath,
const ReadUTF8Options& aOptions) {
MOZ_DIAGNOSTIC_ASSERT(XRE_IsParentProcess());
RefPtr<Promise> promise = CreateJSPromise(aGlobal);
NS_ENSURE_TRUE(!!promise, nullptr);
@ -275,7 +281,8 @@ already_AddRefed<Promise> IOUtils::ReadUTF8(GlobalObject& aGlobal,
REJECT_IF_RELATIVE_PATH(aPath, promise);
nsAutoString path(aPath);
return RunOnBackgroundThread<nsString>(promise, &ReadUTF8Sync, path);
return RunOnBackgroundThread<nsString>(promise, &ReadUTF8Sync, path,
aOptions.mDecompress);
}
/* static */
@ -592,8 +599,9 @@ UniquePtr<PRFileDesc, PR_CloseDelete> IOUtils::OpenExistingSync(
PRFileDesc* fd;
rv = file->OpenNSPRFileDesc(aFlags, /* mode */ 0, &fd);
NS_ENSURE_SUCCESS(rv, nullptr);
if (NS_FAILED(rv)) {
return nullptr;
}
return UniquePtr<PRFileDesc, PR_CloseDelete>(fd);
}
@ -617,9 +625,17 @@ UniquePtr<PRFileDesc, PR_CloseDelete> IOUtils::CreateFileSync(
/* static */
Result<nsTArray<uint8_t>, IOUtils::IOError> IOUtils::ReadSync(
const nsAString& aPath, const Maybe<uint32_t>& aMaxBytes) {
const nsAString& aPath, const Maybe<uint32_t>& aMaxBytes,
const bool aDecompress) {
MOZ_ASSERT(!NS_IsMainThread());
if (aMaxBytes.isSome() && aDecompress) {
return Err(
IOError(NS_ERROR_ILLEGAL_INPUT)
.WithMessage(
"The `maxBytes` and `decompress` options are not compatible"));
}
UniquePtr<PRFileDesc, PR_CloseDelete> fd = OpenExistingSync(aPath, PR_RDONLY);
if (!fd) {
return Err(IOError(NS_ERROR_FILE_NOT_FOUND)
@ -652,7 +668,9 @@ Result<nsTArray<uint8_t>, IOUtils::IOError> IOUtils::ReadSync(
nsTArray<uint8_t> buffer;
if (!buffer.SetCapacity(bufSize, fallible)) {
return Err(IOError(NS_ERROR_OUT_OF_MEMORY));
return Err(IOError(NS_ERROR_OUT_OF_MEMORY)
.WithMessage("Could not allocate buffer to read file(%s)",
NS_ConvertUTF16toUTF8(aPath).get()));
}
// If possible, advise the operating system that we will be reading the file
@ -663,6 +681,7 @@ Result<nsTArray<uint8_t>, IOUtils::IOError> IOUtils::ReadSync(
POSIX_FADV_SEQUENTIAL);
#endif
// Read the file from disk.
uint32_t totalRead = 0;
while (totalRead != bufSize) {
int32_t nRead =
@ -684,15 +703,20 @@ Result<nsTArray<uint8_t>, IOUtils::IOError> IOUtils::ReadSync(
DebugOnly<bool> success = buffer.SetLength(totalRead, fallible);
MOZ_ASSERT(success);
}
// Decompress the file contents, if required.
if (aDecompress) {
return MozLZ4::Decompress(Span(buffer));
}
return std::move(buffer);
}
/* static */
Result<nsString, IOUtils::IOError> IOUtils::ReadUTF8Sync(
const nsAString& aPath) {
const nsAString& aPath, const bool aDecompress) {
MOZ_ASSERT(!NS_IsMainThread());
return ReadSync(aPath, Nothing())
return ReadSync(aPath, Nothing(), aDecompress)
.andThen([&aPath](const nsTArray<uint8_t>& bytes)
-> Result<nsString, IOError> {
auto utf8Span = Span(reinterpret_cast<const char*>(bytes.Elements()),
@ -773,6 +797,21 @@ Result<uint32_t, IOUtils::IOError> IOUtils::WriteAtomicSync(
// continuing.
uint32_t result = 0;
{
// Compress the byte array if required.
nsTArray<uint8_t> compressed;
Span<const uint8_t> bytes;
if (aOptions.mCompress) {
auto rv = MozLZ4::Compress(aByteArray);
if (rv.isErr()) {
return rv.propagateErr();
}
compressed = rv.unwrap();
bytes = Span(compressed);
} else {
bytes = aByteArray;
}
// Then open the file and perform the write.
UniquePtr<PRFileDesc, PR_CloseDelete> fd = OpenExistingSync(tmpPath, flags);
if (!fd) {
fd = CreateFileSync(tmpPath, flags);
@ -782,8 +821,7 @@ Result<uint32_t, IOUtils::IOError> IOUtils::WriteAtomicSync(
.WithMessage("Could not open the file at %s for writing",
NS_ConvertUTF16toUTF8(tmpPath).get()));
}
auto rv = WriteSync(fd.get(), NS_ConvertUTF16toUTF8(tmpPath), aByteArray);
auto rv = WriteSync(fd.get(), NS_ConvertUTF16toUTF8(tmpPath), bytes);
if (rv.isErr()) {
return rv.propagateErr();
}
@ -1253,6 +1291,92 @@ Result<nsTArray<nsString>, IOUtils::IOError> IOUtils::GetChildrenSync(
return children;
}
/* static */
Result<nsTArray<uint8_t>, IOUtils::IOError> IOUtils::MozLZ4::Compress(
Span<const uint8_t> aUncompressed) {
nsTArray<uint8_t> result;
size_t worstCaseSize =
Compression::LZ4::maxCompressedSize(aUncompressed.Length()) + HEADER_SIZE;
if (!result.SetCapacity(worstCaseSize, fallible)) {
return Err(IOError(NS_ERROR_OUT_OF_MEMORY)
.WithMessage("Could not allocate buffer to compress data"));
}
result.AppendElements(Span(MAGIC_NUMBER.data(), MAGIC_NUMBER.size()));
std::array<uint8_t, sizeof(uint32_t)> contentSizeBytes{};
LittleEndian::writeUint32(contentSizeBytes.data(), aUncompressed.Length());
result.AppendElements(Span(contentSizeBytes.data(), contentSizeBytes.size()));
if (aUncompressed.Length() == 0) {
// Don't try to compress an empty buffer.
// Just return the correctly formed header.
result.SetLength(HEADER_SIZE);
return result;
}
size_t compressed = Compression::LZ4::compress(
reinterpret_cast<const char*>(aUncompressed.Elements()),
aUncompressed.Length(),
reinterpret_cast<char*>(result.Elements()) + HEADER_SIZE);
if (!compressed) {
return Err(
IOError(NS_ERROR_UNEXPECTED).WithMessage("Could not compress data"));
}
result.SetLength(HEADER_SIZE + compressed);
return result;
}
/* static */
Result<nsTArray<uint8_t>, IOUtils::IOError> IOUtils::MozLZ4::Decompress(
Span<const uint8_t> aFileContents) {
if (aFileContents.LengthBytes() < HEADER_SIZE) {
return Err(
IOError(NS_ERROR_FILE_CORRUPTED)
.WithMessage(
"Could not decompress file because the buffer is too short"));
}
auto header = aFileContents.To(HEADER_SIZE);
if (!std::equal(std::begin(MAGIC_NUMBER), std::end(MAGIC_NUMBER),
std::begin(header))) {
nsCString magicStr;
uint32_t i = 0;
for (; i < header.Length() - 1; ++i) {
magicStr.AppendPrintf("%02X ", header.at(i));
}
magicStr.AppendPrintf("%02X", header.at(i));
return Err(IOError(NS_ERROR_FILE_CORRUPTED)
.WithMessage("Could not decompress file because it has an "
"invalid LZ4 header (wrong magic number: '%s')",
magicStr.get()));
}
size_t numBytes = sizeof(uint32_t);
Span<const uint8_t> sizeBytes = header.Last(numBytes);
uint32_t expectedDecompressedSize =
LittleEndian::readUint32(sizeBytes.data());
if (expectedDecompressedSize == 0) {
return nsTArray<uint8_t>(0);
}
auto contents = aFileContents.From(HEADER_SIZE);
nsTArray<uint8_t> decompressed;
if (!decompressed.SetCapacity(expectedDecompressedSize, fallible)) {
return Err(
IOError(NS_ERROR_OUT_OF_MEMORY)
.WithMessage("Could not allocate buffer to decompress data"));
}
size_t actualSize = 0;
if (!Compression::LZ4::decompress(
reinterpret_cast<const char*>(contents.Elements()), contents.Length(),
reinterpret_cast<char*>(decompressed.Elements()),
expectedDecompressedSize, &actualSize)) {
return Err(
IOError(NS_ERROR_FILE_CORRUPTED)
.WithMessage(
"Could not decompress file contents, the file may be corrupt"));
}
decompressed.SetLength(actualSize);
return decompressed;
}
NS_IMPL_ISUPPORTS(IOUtilsShutdownBlocker, nsIAsyncShutdownBlocker);
NS_IMETHODIMP IOUtilsShutdownBlocker::GetName(nsAString& aName) {

67
dom/system/IOUtils.h
Просмотреть файл

@ -8,6 +8,7 @@
#define mozilla_dom_IOUtils__
#include "mozilla/AlreadyAddRefed.h"
#include "mozilla/Attributes.h"
#include "mozilla/Buffer.h"
#include "mozilla/DataMutex.h"
#include "mozilla/dom/BindingDeclarations.h"
@ -16,6 +17,7 @@
#include "mozilla/MozPromise.h"
#include "mozilla/Result.h"
#include "nsStringFwd.h"
#include "nsTArray.h"
#include "nspr/prio.h"
#include "nsIAsyncShutdown.h"
#include "nsISerialEventTarget.h"
@ -56,10 +58,11 @@ class IOUtils final {
static already_AddRefed<Promise> Read(GlobalObject& aGlobal,
const nsAString& aPath,
const Optional<uint32_t>& aMaxBytes);
const ReadOptions& aOptions);
static already_AddRefed<Promise> ReadUTF8(GlobalObject& aGlobal,
const nsAString& aPath);
const nsAString& aPath,
const ReadUTF8Options& aOptions);
static already_AddRefed<Promise> WriteAtomic(
GlobalObject& aGlobal, const nsAString& aPath, const Uint8Array& aData,
@ -105,6 +108,7 @@ class IOUtils final {
friend class IOUtilsShutdownBlocker;
struct InternalFileInfo;
struct InternalWriteAtomicOpts;
class MozLZ4;
static StaticDataMutex<StaticRefPtr<nsISerialEventTarget>>
sBackgroundEventTarget;
@ -167,23 +171,31 @@ class IOUtils final {
/**
* Attempts to read the entire file at |aPath| into a buffer.
*
* @param aPath The location of the file as an absolute path string.
* @param aMaxBytes If |Some|, then only read up this this number of bytes,
* otherwise attempt to read the whole file.
* @param aPath The location of the file as an absolute path string.
* @param aMaxBytes If |Some|, then only read up this this number of bytes,
* otherwise attempt to read the whole file.
* @param aDecompress If true, decompress the bytes read from disk before
* returning the result to the caller.
*
* @return A byte array of the entire file contents, or an error.
* @return A byte array of the entire (decompressed) file contents, or an
* error.
*/
static Result<nsTArray<uint8_t>, IOError> ReadSync(
const nsAString& aPath, const Maybe<uint32_t>& aMaxBytes);
const nsAString& aPath, const Maybe<uint32_t>& aMaxBytes,
const bool aDecompress);
/**
* Attempts to read the entire file at |aPath| as a UTF-8 string.
*
* @param aPath The location of the file as an absolute path string.
* @param aPath The location of the file as an absolute path string.
* @param aDecompress If true, decompress the bytes read from disk before
* returning the result to the caller.
*
* @return The contents of the file re-encoded as a UTF-16 string.
* @return The (decompressed) contents of the file re-encoded as a UTF-16
* string.
*/
static Result<nsString, IOError> ReadUTF8Sync(const nsAString& aPath);
static Result<nsString, IOError> ReadUTF8Sync(const nsAString& aPath,
const bool aDecompress);
/**
* Attempts to write the entirety of |aByteArray| to the file at |aPath|.
@ -409,6 +421,7 @@ struct IOUtils::InternalWriteAtomicOpts {
bool mFlush;
bool mNoOverwrite;
Maybe<nsString> mTmpPath;
bool mCompress;
static inline InternalWriteAtomicOpts FromBinding(
const WriteAtomicOptions& aOptions) {
@ -421,10 +434,44 @@ struct IOUtils::InternalWriteAtomicOpts {
if (aOptions.mTmpPath.WasPassed()) {
opts.mTmpPath.emplace(aOptions.mTmpPath.Value());
}
opts.mCompress = aOptions.mCompress;
return opts;
}
};
/**
* Re-implements the file compression and decompression utilities found
* in toolkit/components/lz4/lz4.js
*
* This implementation uses the non-standard data layout:
*
* - MAGIC_NUMBER (8 bytes)
* - content size (uint32_t, little endian)
* - content, as obtained from mozilla::Compression::LZ4::compress
*
* See bug 1209390 for more info.
*/
class IOUtils::MozLZ4 {
public:
static constexpr std::array<uint8_t, 8> MAGIC_NUMBER{'m', 'o', 'z', 'L',
'z', '4', '0', '\0'};
static const uint32_t HEADER_SIZE = 8 + sizeof(uint32_t);
/**
* Compresses |aUncompressed| byte array, and returns a byte array with the
* correct format whose contents may be written to disk.
*/
static Result<nsTArray<uint8_t>, IOError> Compress(
Span<const uint8_t> aUncompressed);
/**
* Checks |aFileContents| for the correct file header, and returns the
* decompressed content.
*/
static Result<nsTArray<uint8_t>, IOError> Decompress(
Span<const uint8_t> aFileContents);
};
class IOUtilsShutdownBlocker : public nsIAsyncShutdownBlocker {
public:
NS_DECL_THREADSAFE_ISUPPORTS

2
dom/system/tests/ioutils/file_ioutils_worker.js
Просмотреть файл

@ -53,7 +53,7 @@ self.onmessage = async function(msg) {
);
const tooManyBytes = bytes.length + 1;
fileContents = await IOUtils.read(tmpFileName, tooManyBytes);
fileContents = await IOUtils.read(tmpFileName, { maxBytes: tooManyBytes });
ok(
ObjectUtils.deepEqual(bytes, fileContents) &&
fileContents.length == bytes.length,

97
dom/system/tests/ioutils/test_ioutils_read_write.html
Просмотреть файл

@ -175,7 +175,7 @@
// Read just the first 10 bytes.
const first10 = bytes.slice(0, 10);
const bytes10 = await IOUtils.read(tmpFileName, 10);
const bytes10 = await IOUtils.read(tmpFileName, { maxBytes: 10 });
ok(
ObjectUtils.deepEqual(bytes10, first10),
"IOUtils::read can read part of a file, up to specified max bytes"
@ -183,7 +183,7 @@
// Trying to explicitly read nothing isn't useful, but it should still
// succeed.
const bytes0 = await IOUtils.read(tmpFileName, 0);
const bytes0 = await IOUtils.read(tmpFileName, { maxBytes: 0 });
is(bytes0.length, 0, "IOUtils::read can read 0 bytes");
await cleanup(tmpFileName);
@ -202,7 +202,7 @@
// Trying to explicitly read nothing isn't useful, but it should still
// succeed.
const bytes0 = await IOUtils.read(tmpFileName, 0);
const bytes0 = await IOUtils.read(tmpFileName, { maxBytes: 0 });
is(bytes0.length, 0, "IOUtils::read can read 0 bytes");
// Implicitly try to read nothing.
@ -234,7 +234,7 @@
);
const tooManyBytes = bytes.length + 1;
fileContents = await IOUtils.read(tmpFileName, tooManyBytes);
fileContents = await IOUtils.read(tmpFileName, { maxBytes: tooManyBytes });
ok(
ObjectUtils.deepEqual(bytes, fileContents) &&
fileContents.length == bytes.length,
@ -267,6 +267,95 @@
"IOUtils::writeAtomic only works with absolute paths"
);
});
add_task(async function test_lz4() {
const tmpFileName = OS.Path.join(tmpDir, "test_ioutils_lz4.tmp");
info("Test writing lz4 encoded data");
const varyingBytes = Uint8Array.of(...new Array(50).keys());
let bytesWritten = await IOUtils.writeAtomic(tmpFileName, varyingBytes, { compress: true });
is(bytesWritten, 64, "Expected to write 64 bytes");
info("Test reading lz4 encoded data");
let readData = await IOUtils.read(tmpFileName, { decompress: true });
ok(readData.equals(varyingBytes), "IOUtils can write and read back LZ4 encoded data");
info("Test writing lz4 compressed data");
const repeatedBytes = Uint8Array.of(...new Array(50).fill(1));
bytesWritten = await IOUtils.writeAtomic(tmpFileName, repeatedBytes, { compress: true });
is(bytesWritten, 23, "Expected 50 bytes to compress to 23 bytes");
info("Test reading lz4 encoded data");
readData = await IOUtils.read(tmpFileName, { decompress: true });
ok(readData.equals(repeatedBytes), "IOUtils can write and read back LZ4 compressed data");
info("Test writing empty lz4 compressed data")
const empty = new Uint8Array();
bytesWritten = await IOUtils.writeAtomic(tmpFileName, empty, { compress: true });
is(bytesWritten, 12, "Expected to write just the LZ4 header");
info("Test reading empty lz4 compressed data")
const readEmpty = await IOUtils.read(tmpFileName, { decompress: true });
ok(readEmpty.equals(empty), "IOUtils can write and read back empty buffers with LZ4");
const readEmptyRaw = await IOUtils.read(tmpFileName, { decompress: false });
is(readEmptyRaw.length, 12, "Expected to read back just the LZ4 header");
await cleanup(tmpFileName);
});
add_task(async function test_lz4_bad_call() {
const tmpFileName = OS.Path.join(tmpDir, "test_ioutils_lz4_bad_call.tmp");
info("Test decompression with invalid options");
const varyingBytes = Uint8Array.of(...new Array(50).keys());
let bytesWritten = await IOUtils.writeAtomic(tmpFileName, varyingBytes, { compress: true });
is(bytesWritten, 64, "Expected to write 64 bytes");
await Assert.rejects(
IOUtils.read(tmpFileName, { maxBytes: 4, decompress: true}),
/The `maxBytes` and `decompress` options are not compatible/,
"IOUtils::read rejects when maxBytes and decompress options are both used"
);
await cleanup(tmpFileName)
});
add_task(async function test_lz4_failure() {
const tmpFileName = OS.Path.join(tmpDir, "test_ioutils_lz4_fail.tmp");
info("Test decompression of non-lz4 data");
const repeatedBytes = Uint8Array.of(...new Array(50).fill(1));
await IOUtils.writeAtomic(tmpFileName, repeatedBytes, { compress: false });
await Assert.rejects(
IOUtils.read(tmpFileName, { decompress: true }),
/Could not decompress file because it has an invalid LZ4 header \(wrong magic number: .*\)/,
"IOUtils::read fails to decompress LZ4 data with a bad header"
);
info("Test decompression of short byte buffer");
const elevenBytes = Uint8Array.of(...new Array(11).fill(1));
await IOUtils.writeAtomic(tmpFileName, elevenBytes, { compress: false });
await Assert.rejects(
IOUtils.read(tmpFileName, { decompress: true }),
/Could not decompress file because the buffer is too short/,
"IOUtils::read fails to decompress LZ4 data with missing header"
);
info("Test decompression of valid header, but corrupt contents");
const headerFor10bytes = [109, 111, 122, 76, 122, 52, 48, 0, 10, 0, 0, 0] // 'mozlz40\0' + 4 byte length
const badContents = new Array(11).fill(255); // Bad leading byte, followed by uncompressed stream.
const goodHeaderBadContents = Uint8Array.of(...headerFor10bytes, ...badContents);
await IOUtils.writeAtomic(tmpFileName, goodHeaderBadContents, { compress: false });
await Assert.rejects(
IOUtils.read(tmpFileName, { decompress: true }),
/Could not decompress file contents, the file may be corrupt/,
"IOUtils::read fails to read corrupt LZ4 contents with a correct header"
);
await cleanup(tmpFileName);
});
</script>
</head>

89
dom/system/tests/ioutils/test_ioutils_read_write_utf8.html
Просмотреть файл

@ -254,6 +254,95 @@
"IOUtils::readUTF8 only works with absolute paths"
);
});
add_task(async function test_utf8_lz4() {
const tmpFileName = OS.Path.join(tmpDir, "test_ioutils_lz4.tmp");
info("Test writing lz4 encoded UTF-8 string");
const emoji = "☕️ ⚧️ 😀 🖖🏿 🤠 🏳️‍🌈 🥠 🏴‍☠️ 🪐";
let bytesWritten = await IOUtils.writeAtomicUTF8(tmpFileName, emoji, { compress: true });
is(bytesWritten, 83, "Expected to write 64 bytes");
info("Test reading lz4 encoded UTF-8 string");
let readData = await IOUtils.readUTF8(tmpFileName, { decompress: true });
is(readData, emoji, "IOUtils can write and read back UTF-8 LZ4 encoded data");
info("Test writing lz4 compressed UTF-8 string");
const lotsOfCoffee = new Array(24).fill('☕️').join(''); // ☕️ is 3 bytes in UTF-8: \0xe2 \0x98 \0x95
bytesWritten = await IOUtils.writeAtomicUTF8(tmpFileName, lotsOfCoffee, { compress: true });
console.log(bytesWritten);
is(bytesWritten, 28, "Expected 72 bytes to compress to 28 bytes");
info("Test reading lz4 encoded UTF-8 string");
readData = await IOUtils.readUTF8(tmpFileName, { decompress: true });
is(readData, lotsOfCoffee, "IOUtils can write and read back UTF-8 LZ4 compressed data");
info("Test writing empty lz4 compressed UTF-8 string")
const empty = "";
bytesWritten = await IOUtils.writeAtomicUTF8(tmpFileName, empty, { compress: true });
is(bytesWritten, 12, "Expected to write just the LZ4 header");
info("Test reading empty lz4 compressed UTF-8 string")
const readEmpty = await IOUtils.readUTF8(tmpFileName, { decompress: true });
is(readEmpty, empty, "IOUtils can write and read back empty buffers with LZ4");
const readEmptyRaw = await IOUtils.readUTF8(tmpFileName, { decompress: false });
is(readEmptyRaw.length, 12, "Expected to read back just the LZ4 header");
await cleanup(tmpFileName);
});
add_task(async function test_lz4_bad_call() {
const tmpFileName = OS.Path.join(tmpDir, "test_ioutils_lz4_bad_call.tmp");
info("readUTF8 ignores the maxBytes option if provided");
const emoji = "☕️ ⚧️ 😀 🖖🏿 🤠 🏳️‍🌈 🥠 🏴‍☠️ 🪐";
let bytesWritten = await IOUtils.writeAtomicUTF8(tmpFileName, emoji, { compress: true });
is(bytesWritten, 83, "Expected to write 83 bytes");
let readData = await IOUtils.readUTF8(tmpFileName, { maxBytes: 4, decompress: true });
is(readData, emoji, "IOUtils can write and read back UTF-8 LZ4 encoded data");
await cleanup(tmpFileName)
});
add_task(async function test_utf8_lz4_failure() {
const tmpFileName = OS.Path.join(tmpDir, "test_ioutils_lz4_fail.tmp");
info("Test decompression of non-lz4 UTF-8 string");
const repeatedBytes = Uint8Array.of(...new Array(50).fill(1));
await IOUtils.writeAtomic(tmpFileName, repeatedBytes, { compress: false });
await Assert.rejects(
IOUtils.readUTF8(tmpFileName, { decompress: true }),
/Could not decompress file because it has an invalid LZ4 header \(wrong magic number: .*\)/,
"IOUtils::readUTF8 fails to decompress LZ4 data with a bad header"
);
info("Test UTF-8 decompression of short byte buffer");
const elevenBytes = Uint8Array.of(...new Array(11).fill(1));
await IOUtils.writeAtomic(tmpFileName, elevenBytes, { compress: false });
await Assert.rejects(
IOUtils.readUTF8(tmpFileName, { decompress: true }),
/Could not decompress file because the buffer is too short/,
"IOUtils::readUTF8 fails to decompress LZ4 data with missing header"
);
info("Test UTF-8 decompression of valid header, but corrupt contents");
const headerFor10bytes = [109, 111, 122, 76, 122, 52, 48, 0, 10, 0, 0, 0] // 'mozlz40\0' + 4 byte length
const badContents = new Array(11).fill(255); // Bad leading byte, followed by uncompressed stream.
const goodHeaderBadContents = Uint8Array.of(...headerFor10bytes, ...badContents);
await IOUtils.writeAtomic(tmpFileName, goodHeaderBadContents, { compress: false });
await Assert.rejects(
IOUtils.readUTF8(tmpFileName, { decompress: true }),
/Could not decompress file contents, the file may be corrupt/,
"IOUtils::readUTF8 fails to read corrupt LZ4 contents with a correct header"
);
await cleanup(tmpFileName);
});
</script>
</head>