зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1660835: Add LZ4 compression options to IOUtils read and write methods r=barret,Gijs
NB: This change breaks the IOUtils.read API, requiring that an options dictionary is passed as the optional second argument, rather than a number indicating the max bytes to read. This option is not used out of tests however. Differential Revision: https://phabricator.services.mozilla.com/D88177
This commit is contained in:
Keefer Rourke
Родитель
f283542454
Коммит
b09555373a
|
|
@ -24,16 +24,14 @@
|
|||
[ChromeOnly, Exposed=(Window, Worker)]
|
||||
namespace IOUtils {
|
||||
/**
|
||||
* Reads up to |maxBytes| of the file at |path|. If |maxBytes| is unspecified,
|
||||
* the entire file is read.
|
||||
* Reads up to |maxBytes| of the file at |path| according to |opts|.
|
||||
*
|
||||
* @param path An absolute file path.
|
||||
* @param maxBytes The max bytes to read from the file at path.
|
||||
* @param path An absolute file path.
|
||||
*
|
||||
* @return Resolves with an array of unsigned byte values read from disk,
|
||||
* otherwise rejects with a DOMException.
|
||||
*/
|
||||
Promise<Uint8Array> read(DOMString path, optional unsigned long maxBytes);
|
||||
Promise<Uint8Array> read(DOMString path, optional ReadOptions opts = {});
|
||||
/**
|
||||
* Reads the UTF-8 text file located at |path| and returns the decoded
|
||||
* contents as a |DOMString|.
|
||||
|
|
@ -43,7 +41,7 @@ namespace IOUtils {
|
|||
* @return Resolves with the file contents encoded as a string, otherwise
|
||||
* rejects with a DOMException.
|
||||
*/
|
||||
Promise<DOMString> readUTF8(DOMString path);
|
||||
Promise<DOMString> readUTF8(DOMString path, optional ReadUTF8Options opts = {});
|
||||
/**
|
||||
* Attempts to safely write |data| to a file at |path|.
|
||||
*
|
||||
|
|
@ -160,7 +158,31 @@ namespace IOUtils {
|
|||
};
|
||||
|
||||
/**
|
||||
* Options to be passed to the |IOUtils.writeAtomic| method.
|
||||
* Options to be passed to the |IOUtils.readUTF8| method.
|
||||
*/
|
||||
dictionary ReadUTF8Options {
|
||||
/**
|
||||
* If true, this option indicates that the file to be read is compressed with
|
||||
* LZ4-encoding, and should be decompressed before the data is returned to
|
||||
* the caller.
|
||||
*/
|
||||
boolean decompress = false;
|
||||
};
|
||||
|
||||
/**
|
||||
* Options to be passed to the |IOUtils.read| method.
|
||||
*/
|
||||
dictionary ReadOptions : ReadUTF8Options {
|
||||
/**
|
||||
* The max bytes to read from the file at path. If unspecified, the entire
|
||||
* file will be read. This option is incompatible with |decompress|.
|
||||
*/
|
||||
unsigned long? maxBytes = null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Options to be passed to the |IOUtils.writeAtomic| and |writeAtomicUTF8|
|
||||
* methods.
|
||||
*/
|
||||
dictionary WriteAtomicOptions {
|
||||
/**
|
||||
|
|
@ -185,6 +207,10 @@ dictionary WriteAtomicOptions {
|
|||
* disconnection before the buffers are flushed.
|
||||
*/
|
||||
boolean flush = false;
|
||||
/**
|
||||
* If true, compress the data with LZ4-encoding before writing to the file.
|
||||
*/
|
||||
boolean compress = false;
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -4,10 +4,14 @@
|
|||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
#include "mozilla/dom/IOUtils.h"
|
||||
#include "ErrorList.h"
|
||||
#include "mozilla/dom/IOUtilsBinding.h"
|
||||
#include "mozilla/dom/Promise.h"
|
||||
#include "mozilla/Compression.h"
|
||||
#include "mozilla/EndianUtils.h"
|
||||
#include "mozilla/ErrorNames.h"
|
||||
#include "mozilla/ResultExtensions.h"
|
||||
#include "mozilla/Span.h"
|
||||
|
|
@ -15,6 +19,7 @@
|
|||
#include "nsError.h"
|
||||
#include "nsIDirectoryEnumerator.h"
|
||||
#include "nsPrintfCString.h"
|
||||
#include "nsTArray.h"
|
||||
#include "nspr/prerror.h"
|
||||
#include "nspr/prio.h"
|
||||
#include "nspr/private/pprio.h"
|
||||
|
|
@ -240,7 +245,7 @@ already_AddRefed<Promise> IOUtils::RunOnBackgroundThread(
|
|||
/* static */
|
||||
already_AddRefed<Promise> IOUtils::Read(GlobalObject& aGlobal,
|
||||
const nsAString& aPath,
|
||||
const Optional<uint32_t>& aMaxBytes) {
|
||||
const ReadOptions& aOptions) {
|
||||
MOZ_DIAGNOSTIC_ASSERT(XRE_IsParentProcess());
|
||||
RefPtr<Promise> promise = CreateJSPromise(aGlobal);
|
||||
NS_ENSURE_TRUE(!!promise, nullptr);
|
||||
|
|
@ -250,23 +255,24 @@ already_AddRefed<Promise> IOUtils::Read(GlobalObject& aGlobal,
|
|||
REJECT_IF_RELATIVE_PATH(aPath, promise);
|
||||
nsAutoString path(aPath);
|
||||
Maybe<uint32_t> toRead = Nothing();
|
||||
if (aMaxBytes.WasPassed()) {
|
||||
if (aMaxBytes.Value() == 0) {
|
||||
if (!aOptions.mMaxBytes.IsNull()) {
|
||||
if (aOptions.mMaxBytes.Value() == 0) {
|
||||
// Resolve with an empty buffer.
|
||||
nsTArray<uint8_t> arr(0);
|
||||
promise->MaybeResolve(TypedArrayCreator<Uint8Array>(arr));
|
||||
return promise.forget();
|
||||
}
|
||||
toRead.emplace(aMaxBytes.Value());
|
||||
toRead.emplace(aOptions.mMaxBytes.Value());
|
||||
}
|
||||
|
||||
return RunOnBackgroundThread<nsTArray<uint8_t>>(promise, &ReadSync, path,
|
||||
toRead);
|
||||
toRead, aOptions.mDecompress);
|
||||
}
|
||||
|
||||
/* static */
|
||||
already_AddRefed<Promise> IOUtils::ReadUTF8(GlobalObject& aGlobal,
|
||||
const nsAString& aPath) {
|
||||
const nsAString& aPath,
|
||||
const ReadUTF8Options& aOptions) {
|
||||
MOZ_DIAGNOSTIC_ASSERT(XRE_IsParentProcess());
|
||||
RefPtr<Promise> promise = CreateJSPromise(aGlobal);
|
||||
NS_ENSURE_TRUE(!!promise, nullptr);
|
||||
|
|
@ -275,7 +281,8 @@ already_AddRefed<Promise> IOUtils::ReadUTF8(GlobalObject& aGlobal,
|
|||
REJECT_IF_RELATIVE_PATH(aPath, promise);
|
||||
nsAutoString path(aPath);
|
||||
|
||||
return RunOnBackgroundThread<nsString>(promise, &ReadUTF8Sync, path);
|
||||
return RunOnBackgroundThread<nsString>(promise, &ReadUTF8Sync, path,
|
||||
aOptions.mDecompress);
|
||||
}
|
||||
|
||||
/* static */
|
||||
|
|
@ -592,8 +599,9 @@ UniquePtr<PRFileDesc, PR_CloseDelete> IOUtils::OpenExistingSync(
|
|||
|
||||
PRFileDesc* fd;
|
||||
rv = file->OpenNSPRFileDesc(aFlags, /* mode */ 0, &fd);
|
||||
NS_ENSURE_SUCCESS(rv, nullptr);
|
||||
|
||||
if (NS_FAILED(rv)) {
|
||||
return nullptr;
|
||||
}
|
||||
return UniquePtr<PRFileDesc, PR_CloseDelete>(fd);
|
||||
}
|
||||
|
||||
|
|
@ -617,9 +625,17 @@ UniquePtr<PRFileDesc, PR_CloseDelete> IOUtils::CreateFileSync(
|
|||
|
||||
/* static */
|
||||
Result<nsTArray<uint8_t>, IOUtils::IOError> IOUtils::ReadSync(
|
||||
const nsAString& aPath, const Maybe<uint32_t>& aMaxBytes) {
|
||||
const nsAString& aPath, const Maybe<uint32_t>& aMaxBytes,
|
||||
const bool aDecompress) {
|
||||
MOZ_ASSERT(!NS_IsMainThread());
|
||||
|
||||
if (aMaxBytes.isSome() && aDecompress) {
|
||||
return Err(
|
||||
IOError(NS_ERROR_ILLEGAL_INPUT)
|
||||
.WithMessage(
|
||||
"The `maxBytes` and `decompress` options are not compatible"));
|
||||
}
|
||||
|
||||
UniquePtr<PRFileDesc, PR_CloseDelete> fd = OpenExistingSync(aPath, PR_RDONLY);
|
||||
if (!fd) {
|
||||
return Err(IOError(NS_ERROR_FILE_NOT_FOUND)
|
||||
|
|
@ -652,7 +668,9 @@ Result<nsTArray<uint8_t>, IOUtils::IOError> IOUtils::ReadSync(
|
|||
|
||||
nsTArray<uint8_t> buffer;
|
||||
if (!buffer.SetCapacity(bufSize, fallible)) {
|
||||
return Err(IOError(NS_ERROR_OUT_OF_MEMORY));
|
||||
return Err(IOError(NS_ERROR_OUT_OF_MEMORY)
|
||||
.WithMessage("Could not allocate buffer to read file(%s)",
|
||||
NS_ConvertUTF16toUTF8(aPath).get()));
|
||||
}
|
||||
|
||||
// If possible, advise the operating system that we will be reading the file
|
||||
|
|
@ -663,6 +681,7 @@ Result<nsTArray<uint8_t>, IOUtils::IOError> IOUtils::ReadSync(
|
|||
POSIX_FADV_SEQUENTIAL);
|
||||
#endif
|
||||
|
||||
// Read the file from disk.
|
||||
uint32_t totalRead = 0;
|
||||
while (totalRead != bufSize) {
|
||||
int32_t nRead =
|
||||
|
|
@ -684,15 +703,20 @@ Result<nsTArray<uint8_t>, IOUtils::IOError> IOUtils::ReadSync(
|
|||
DebugOnly<bool> success = buffer.SetLength(totalRead, fallible);
|
||||
MOZ_ASSERT(success);
|
||||
}
|
||||
|
||||
// Decompress the file contents, if required.
|
||||
if (aDecompress) {
|
||||
return MozLZ4::Decompress(Span(buffer));
|
||||
}
|
||||
return std::move(buffer);
|
||||
}
|
||||
|
||||
/* static */
|
||||
Result<nsString, IOUtils::IOError> IOUtils::ReadUTF8Sync(
|
||||
const nsAString& aPath) {
|
||||
const nsAString& aPath, const bool aDecompress) {
|
||||
MOZ_ASSERT(!NS_IsMainThread());
|
||||
|
||||
return ReadSync(aPath, Nothing())
|
||||
return ReadSync(aPath, Nothing(), aDecompress)
|
||||
.andThen([&aPath](const nsTArray<uint8_t>& bytes)
|
||||
-> Result<nsString, IOError> {
|
||||
auto utf8Span = Span(reinterpret_cast<const char*>(bytes.Elements()),
|
||||
|
|
@ -773,6 +797,21 @@ Result<uint32_t, IOUtils::IOError> IOUtils::WriteAtomicSync(
|
|||
// continuing.
|
||||
uint32_t result = 0;
|
||||
{
|
||||
// Compress the byte array if required.
|
||||
nsTArray<uint8_t> compressed;
|
||||
Span<const uint8_t> bytes;
|
||||
if (aOptions.mCompress) {
|
||||
auto rv = MozLZ4::Compress(aByteArray);
|
||||
if (rv.isErr()) {
|
||||
return rv.propagateErr();
|
||||
}
|
||||
compressed = rv.unwrap();
|
||||
bytes = Span(compressed);
|
||||
} else {
|
||||
bytes = aByteArray;
|
||||
}
|
||||
|
||||
// Then open the file and perform the write.
|
||||
UniquePtr<PRFileDesc, PR_CloseDelete> fd = OpenExistingSync(tmpPath, flags);
|
||||
if (!fd) {
|
||||
fd = CreateFileSync(tmpPath, flags);
|
||||
|
|
@ -782,8 +821,7 @@ Result<uint32_t, IOUtils::IOError> IOUtils::WriteAtomicSync(
|
|||
.WithMessage("Could not open the file at %s for writing",
|
||||
NS_ConvertUTF16toUTF8(tmpPath).get()));
|
||||
}
|
||||
|
||||
auto rv = WriteSync(fd.get(), NS_ConvertUTF16toUTF8(tmpPath), aByteArray);
|
||||
auto rv = WriteSync(fd.get(), NS_ConvertUTF16toUTF8(tmpPath), bytes);
|
||||
if (rv.isErr()) {
|
||||
return rv.propagateErr();
|
||||
}
|
||||
|
|
@ -1253,6 +1291,92 @@ Result<nsTArray<nsString>, IOUtils::IOError> IOUtils::GetChildrenSync(
|
|||
return children;
|
||||
}
|
||||
|
||||
/* static */
|
||||
Result<nsTArray<uint8_t>, IOUtils::IOError> IOUtils::MozLZ4::Compress(
|
||||
Span<const uint8_t> aUncompressed) {
|
||||
nsTArray<uint8_t> result;
|
||||
size_t worstCaseSize =
|
||||
Compression::LZ4::maxCompressedSize(aUncompressed.Length()) + HEADER_SIZE;
|
||||
if (!result.SetCapacity(worstCaseSize, fallible)) {
|
||||
return Err(IOError(NS_ERROR_OUT_OF_MEMORY)
|
||||
.WithMessage("Could not allocate buffer to compress data"));
|
||||
}
|
||||
result.AppendElements(Span(MAGIC_NUMBER.data(), MAGIC_NUMBER.size()));
|
||||
std::array<uint8_t, sizeof(uint32_t)> contentSizeBytes{};
|
||||
LittleEndian::writeUint32(contentSizeBytes.data(), aUncompressed.Length());
|
||||
result.AppendElements(Span(contentSizeBytes.data(), contentSizeBytes.size()));
|
||||
|
||||
if (aUncompressed.Length() == 0) {
|
||||
// Don't try to compress an empty buffer.
|
||||
// Just return the correctly formed header.
|
||||
result.SetLength(HEADER_SIZE);
|
||||
return result;
|
||||
}
|
||||
|
||||
size_t compressed = Compression::LZ4::compress(
|
||||
reinterpret_cast<const char*>(aUncompressed.Elements()),
|
||||
aUncompressed.Length(),
|
||||
reinterpret_cast<char*>(result.Elements()) + HEADER_SIZE);
|
||||
if (!compressed) {
|
||||
return Err(
|
||||
IOError(NS_ERROR_UNEXPECTED).WithMessage("Could not compress data"));
|
||||
}
|
||||
result.SetLength(HEADER_SIZE + compressed);
|
||||
return result;
|
||||
}
|
||||
|
||||
/* static */
|
||||
Result<nsTArray<uint8_t>, IOUtils::IOError> IOUtils::MozLZ4::Decompress(
|
||||
Span<const uint8_t> aFileContents) {
|
||||
if (aFileContents.LengthBytes() < HEADER_SIZE) {
|
||||
return Err(
|
||||
IOError(NS_ERROR_FILE_CORRUPTED)
|
||||
.WithMessage(
|
||||
"Could not decompress file because the buffer is too short"));
|
||||
}
|
||||
auto header = aFileContents.To(HEADER_SIZE);
|
||||
if (!std::equal(std::begin(MAGIC_NUMBER), std::end(MAGIC_NUMBER),
|
||||
std::begin(header))) {
|
||||
nsCString magicStr;
|
||||
uint32_t i = 0;
|
||||
for (; i < header.Length() - 1; ++i) {
|
||||
magicStr.AppendPrintf("%02X ", header.at(i));
|
||||
}
|
||||
magicStr.AppendPrintf("%02X", header.at(i));
|
||||
|
||||
return Err(IOError(NS_ERROR_FILE_CORRUPTED)
|
||||
.WithMessage("Could not decompress file because it has an "
|
||||
"invalid LZ4 header (wrong magic number: '%s')",
|
||||
magicStr.get()));
|
||||
}
|
||||
size_t numBytes = sizeof(uint32_t);
|
||||
Span<const uint8_t> sizeBytes = header.Last(numBytes);
|
||||
uint32_t expectedDecompressedSize =
|
||||
LittleEndian::readUint32(sizeBytes.data());
|
||||
if (expectedDecompressedSize == 0) {
|
||||
return nsTArray<uint8_t>(0);
|
||||
}
|
||||
auto contents = aFileContents.From(HEADER_SIZE);
|
||||
nsTArray<uint8_t> decompressed;
|
||||
if (!decompressed.SetCapacity(expectedDecompressedSize, fallible)) {
|
||||
return Err(
|
||||
IOError(NS_ERROR_OUT_OF_MEMORY)
|
||||
.WithMessage("Could not allocate buffer to decompress data"));
|
||||
}
|
||||
size_t actualSize = 0;
|
||||
if (!Compression::LZ4::decompress(
|
||||
reinterpret_cast<const char*>(contents.Elements()), contents.Length(),
|
||||
reinterpret_cast<char*>(decompressed.Elements()),
|
||||
expectedDecompressedSize, &actualSize)) {
|
||||
return Err(
|
||||
IOError(NS_ERROR_FILE_CORRUPTED)
|
||||
.WithMessage(
|
||||
"Could not decompress file contents, the file may be corrupt"));
|
||||
}
|
||||
decompressed.SetLength(actualSize);
|
||||
return decompressed;
|
||||
}
|
||||
|
||||
NS_IMPL_ISUPPORTS(IOUtilsShutdownBlocker, nsIAsyncShutdownBlocker);
|
||||
|
||||
NS_IMETHODIMP IOUtilsShutdownBlocker::GetName(nsAString& aName) {
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
#define mozilla_dom_IOUtils__
|
||||
|
||||
#include "mozilla/AlreadyAddRefed.h"
|
||||
#include "mozilla/Attributes.h"
|
||||
#include "mozilla/Buffer.h"
|
||||
#include "mozilla/DataMutex.h"
|
||||
#include "mozilla/dom/BindingDeclarations.h"
|
||||
|
|
@ -16,6 +17,7 @@
|
|||
#include "mozilla/MozPromise.h"
|
||||
#include "mozilla/Result.h"
|
||||
#include "nsStringFwd.h"
|
||||
#include "nsTArray.h"
|
||||
#include "nspr/prio.h"
|
||||
#include "nsIAsyncShutdown.h"
|
||||
#include "nsISerialEventTarget.h"
|
||||
|
|
@ -56,10 +58,11 @@ class IOUtils final {
|
|||
|
||||
static already_AddRefed<Promise> Read(GlobalObject& aGlobal,
|
||||
const nsAString& aPath,
|
||||
const Optional<uint32_t>& aMaxBytes);
|
||||
const ReadOptions& aOptions);
|
||||
|
||||
static already_AddRefed<Promise> ReadUTF8(GlobalObject& aGlobal,
|
||||
const nsAString& aPath);
|
||||
const nsAString& aPath,
|
||||
const ReadUTF8Options& aOptions);
|
||||
|
||||
static already_AddRefed<Promise> WriteAtomic(
|
||||
GlobalObject& aGlobal, const nsAString& aPath, const Uint8Array& aData,
|
||||
|
|
@ -105,6 +108,7 @@ class IOUtils final {
|
|||
friend class IOUtilsShutdownBlocker;
|
||||
struct InternalFileInfo;
|
||||
struct InternalWriteAtomicOpts;
|
||||
class MozLZ4;
|
||||
|
||||
static StaticDataMutex<StaticRefPtr<nsISerialEventTarget>>
|
||||
sBackgroundEventTarget;
|
||||
|
|
@ -167,23 +171,31 @@ class IOUtils final {
|
|||
/**
|
||||
* Attempts to read the entire file at |aPath| into a buffer.
|
||||
*
|
||||
* @param aPath The location of the file as an absolute path string.
|
||||
* @param aMaxBytes If |Some|, then only read up this this number of bytes,
|
||||
* otherwise attempt to read the whole file.
|
||||
* @param aPath The location of the file as an absolute path string.
|
||||
* @param aMaxBytes If |Some|, then only read up this this number of bytes,
|
||||
* otherwise attempt to read the whole file.
|
||||
* @param aDecompress If true, decompress the bytes read from disk before
|
||||
* returning the result to the caller.
|
||||
*
|
||||
* @return A byte array of the entire file contents, or an error.
|
||||
* @return A byte array of the entire (decompressed) file contents, or an
|
||||
* error.
|
||||
*/
|
||||
static Result<nsTArray<uint8_t>, IOError> ReadSync(
|
||||
const nsAString& aPath, const Maybe<uint32_t>& aMaxBytes);
|
||||
const nsAString& aPath, const Maybe<uint32_t>& aMaxBytes,
|
||||
const bool aDecompress);
|
||||
|
||||
/**
|
||||
* Attempts to read the entire file at |aPath| as a UTF-8 string.
|
||||
*
|
||||
* @param aPath The location of the file as an absolute path string.
|
||||
* @param aPath The location of the file as an absolute path string.
|
||||
* @param aDecompress If true, decompress the bytes read from disk before
|
||||
* returning the result to the caller.
|
||||
*
|
||||
* @return The contents of the file re-encoded as a UTF-16 string.
|
||||
* @return The (decompressed) contents of the file re-encoded as a UTF-16
|
||||
* string.
|
||||
*/
|
||||
static Result<nsString, IOError> ReadUTF8Sync(const nsAString& aPath);
|
||||
static Result<nsString, IOError> ReadUTF8Sync(const nsAString& aPath,
|
||||
const bool aDecompress);
|
||||
|
||||
/**
|
||||
* Attempts to write the entirety of |aByteArray| to the file at |aPath|.
|
||||
|
|
@ -409,6 +421,7 @@ struct IOUtils::InternalWriteAtomicOpts {
|
|||
bool mFlush;
|
||||
bool mNoOverwrite;
|
||||
Maybe<nsString> mTmpPath;
|
||||
bool mCompress;
|
||||
|
||||
static inline InternalWriteAtomicOpts FromBinding(
|
||||
const WriteAtomicOptions& aOptions) {
|
||||
|
|
@ -421,10 +434,44 @@ struct IOUtils::InternalWriteAtomicOpts {
|
|||
if (aOptions.mTmpPath.WasPassed()) {
|
||||
opts.mTmpPath.emplace(aOptions.mTmpPath.Value());
|
||||
}
|
||||
opts.mCompress = aOptions.mCompress;
|
||||
return opts;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Re-implements the file compression and decompression utilities found
|
||||
* in toolkit/components/lz4/lz4.js
|
||||
*
|
||||
* This implementation uses the non-standard data layout:
|
||||
*
|
||||
* - MAGIC_NUMBER (8 bytes)
|
||||
* - content size (uint32_t, little endian)
|
||||
* - content, as obtained from mozilla::Compression::LZ4::compress
|
||||
*
|
||||
* See bug 1209390 for more info.
|
||||
*/
|
||||
class IOUtils::MozLZ4 {
|
||||
public:
|
||||
static constexpr std::array<uint8_t, 8> MAGIC_NUMBER{'m', 'o', 'z', 'L',
|
||||
'z', '4', '0', '\0'};
|
||||
static const uint32_t HEADER_SIZE = 8 + sizeof(uint32_t);
|
||||
|
||||
/**
|
||||
* Compresses |aUncompressed| byte array, and returns a byte array with the
|
||||
* correct format whose contents may be written to disk.
|
||||
*/
|
||||
static Result<nsTArray<uint8_t>, IOError> Compress(
|
||||
Span<const uint8_t> aUncompressed);
|
||||
|
||||
/**
|
||||
* Checks |aFileContents| for the correct file header, and returns the
|
||||
* decompressed content.
|
||||
*/
|
||||
static Result<nsTArray<uint8_t>, IOError> Decompress(
|
||||
Span<const uint8_t> aFileContents);
|
||||
};
|
||||
|
||||
class IOUtilsShutdownBlocker : public nsIAsyncShutdownBlocker {
|
||||
public:
|
||||
NS_DECL_THREADSAFE_ISUPPORTS
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ self.onmessage = async function(msg) {
|
|||
);
|
||||
|
||||
const tooManyBytes = bytes.length + 1;
|
||||
fileContents = await IOUtils.read(tmpFileName, tooManyBytes);
|
||||
fileContents = await IOUtils.read(tmpFileName, { maxBytes: tooManyBytes });
|
||||
ok(
|
||||
ObjectUtils.deepEqual(bytes, fileContents) &&
|
||||
fileContents.length == bytes.length,
|
||||
|
|
|
|||
|
|
@ -175,7 +175,7 @@
|
|||
|
||||
// Read just the first 10 bytes.
|
||||
const first10 = bytes.slice(0, 10);
|
||||
const bytes10 = await IOUtils.read(tmpFileName, 10);
|
||||
const bytes10 = await IOUtils.read(tmpFileName, { maxBytes: 10 });
|
||||
ok(
|
||||
ObjectUtils.deepEqual(bytes10, first10),
|
||||
"IOUtils::read can read part of a file, up to specified max bytes"
|
||||
|
|
@ -183,7 +183,7 @@
|
|||
|
||||
// Trying to explicitly read nothing isn't useful, but it should still
|
||||
// succeed.
|
||||
const bytes0 = await IOUtils.read(tmpFileName, 0);
|
||||
const bytes0 = await IOUtils.read(tmpFileName, { maxBytes: 0 });
|
||||
is(bytes0.length, 0, "IOUtils::read can read 0 bytes");
|
||||
|
||||
await cleanup(tmpFileName);
|
||||
|
|
@ -202,7 +202,7 @@
|
|||
|
||||
// Trying to explicitly read nothing isn't useful, but it should still
|
||||
// succeed.
|
||||
const bytes0 = await IOUtils.read(tmpFileName, 0);
|
||||
const bytes0 = await IOUtils.read(tmpFileName, { maxBytes: 0 });
|
||||
is(bytes0.length, 0, "IOUtils::read can read 0 bytes");
|
||||
|
||||
// Implicitly try to read nothing.
|
||||
|
|
@ -234,7 +234,7 @@
|
|||
);
|
||||
|
||||
const tooManyBytes = bytes.length + 1;
|
||||
fileContents = await IOUtils.read(tmpFileName, tooManyBytes);
|
||||
fileContents = await IOUtils.read(tmpFileName, { maxBytes: tooManyBytes });
|
||||
ok(
|
||||
ObjectUtils.deepEqual(bytes, fileContents) &&
|
||||
fileContents.length == bytes.length,
|
||||
|
|
@ -267,6 +267,95 @@
|
|||
"IOUtils::writeAtomic only works with absolute paths"
|
||||
);
|
||||
});
|
||||
|
||||
add_task(async function test_lz4() {
|
||||
const tmpFileName = OS.Path.join(tmpDir, "test_ioutils_lz4.tmp");
|
||||
|
||||
info("Test writing lz4 encoded data");
|
||||
const varyingBytes = Uint8Array.of(...new Array(50).keys());
|
||||
let bytesWritten = await IOUtils.writeAtomic(tmpFileName, varyingBytes, { compress: true });
|
||||
is(bytesWritten, 64, "Expected to write 64 bytes");
|
||||
|
||||
info("Test reading lz4 encoded data");
|
||||
let readData = await IOUtils.read(tmpFileName, { decompress: true });
|
||||
ok(readData.equals(varyingBytes), "IOUtils can write and read back LZ4 encoded data");
|
||||
|
||||
info("Test writing lz4 compressed data");
|
||||
const repeatedBytes = Uint8Array.of(...new Array(50).fill(1));
|
||||
bytesWritten = await IOUtils.writeAtomic(tmpFileName, repeatedBytes, { compress: true });
|
||||
is(bytesWritten, 23, "Expected 50 bytes to compress to 23 bytes");
|
||||
|
||||
info("Test reading lz4 encoded data");
|
||||
readData = await IOUtils.read(tmpFileName, { decompress: true });
|
||||
ok(readData.equals(repeatedBytes), "IOUtils can write and read back LZ4 compressed data");
|
||||
|
||||
info("Test writing empty lz4 compressed data")
|
||||
const empty = new Uint8Array();
|
||||
bytesWritten = await IOUtils.writeAtomic(tmpFileName, empty, { compress: true });
|
||||
is(bytesWritten, 12, "Expected to write just the LZ4 header");
|
||||
|
||||
info("Test reading empty lz4 compressed data")
|
||||
const readEmpty = await IOUtils.read(tmpFileName, { decompress: true });
|
||||
ok(readEmpty.equals(empty), "IOUtils can write and read back empty buffers with LZ4");
|
||||
const readEmptyRaw = await IOUtils.read(tmpFileName, { decompress: false });
|
||||
is(readEmptyRaw.length, 12, "Expected to read back just the LZ4 header");
|
||||
|
||||
await cleanup(tmpFileName);
|
||||
});
|
||||
|
||||
add_task(async function test_lz4_bad_call() {
|
||||
const tmpFileName = OS.Path.join(tmpDir, "test_ioutils_lz4_bad_call.tmp");
|
||||
|
||||
info("Test decompression with invalid options");
|
||||
const varyingBytes = Uint8Array.of(...new Array(50).keys());
|
||||
let bytesWritten = await IOUtils.writeAtomic(tmpFileName, varyingBytes, { compress: true });
|
||||
is(bytesWritten, 64, "Expected to write 64 bytes");
|
||||
await Assert.rejects(
|
||||
IOUtils.read(tmpFileName, { maxBytes: 4, decompress: true}),
|
||||
/The `maxBytes` and `decompress` options are not compatible/,
|
||||
"IOUtils::read rejects when maxBytes and decompress options are both used"
|
||||
);
|
||||
|
||||
await cleanup(tmpFileName)
|
||||
});
|
||||
|
||||
add_task(async function test_lz4_failure() {
|
||||
const tmpFileName = OS.Path.join(tmpDir, "test_ioutils_lz4_fail.tmp");
|
||||
|
||||
info("Test decompression of non-lz4 data");
|
||||
const repeatedBytes = Uint8Array.of(...new Array(50).fill(1));
|
||||
await IOUtils.writeAtomic(tmpFileName, repeatedBytes, { compress: false });
|
||||
|
||||
await Assert.rejects(
|
||||
IOUtils.read(tmpFileName, { decompress: true }),
|
||||
/Could not decompress file because it has an invalid LZ4 header \(wrong magic number: .*\)/,
|
||||
"IOUtils::read fails to decompress LZ4 data with a bad header"
|
||||
);
|
||||
|
||||
info("Test decompression of short byte buffer");
|
||||
const elevenBytes = Uint8Array.of(...new Array(11).fill(1));
|
||||
await IOUtils.writeAtomic(tmpFileName, elevenBytes, { compress: false });
|
||||
|
||||
await Assert.rejects(
|
||||
IOUtils.read(tmpFileName, { decompress: true }),
|
||||
/Could not decompress file because the buffer is too short/,
|
||||
"IOUtils::read fails to decompress LZ4 data with missing header"
|
||||
);
|
||||
|
||||
info("Test decompression of valid header, but corrupt contents");
|
||||
const headerFor10bytes = [109, 111, 122, 76, 122, 52, 48, 0, 10, 0, 0, 0] // 'mozlz40\0' + 4 byte length
|
||||
const badContents = new Array(11).fill(255); // Bad leading byte, followed by uncompressed stream.
|
||||
const goodHeaderBadContents = Uint8Array.of(...headerFor10bytes, ...badContents);
|
||||
await IOUtils.writeAtomic(tmpFileName, goodHeaderBadContents, { compress: false });
|
||||
|
||||
await Assert.rejects(
|
||||
IOUtils.read(tmpFileName, { decompress: true }),
|
||||
/Could not decompress file contents, the file may be corrupt/,
|
||||
"IOUtils::read fails to read corrupt LZ4 contents with a correct header"
|
||||
);
|
||||
|
||||
await cleanup(tmpFileName);
|
||||
});
|
||||
</script>
|
||||
</head>
|
||||
|
||||
|
|
|
|||
|
|
@ -254,6 +254,95 @@
|
|||
"IOUtils::readUTF8 only works with absolute paths"
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
add_task(async function test_utf8_lz4() {
|
||||
const tmpFileName = OS.Path.join(tmpDir, "test_ioutils_lz4.tmp");
|
||||
|
||||
info("Test writing lz4 encoded UTF-8 string");
|
||||
const emoji = "☕️ ⚧️ 😀 🖖🏿 🤠 🏳️🌈 🥠 🏴☠️ 🪐";
|
||||
let bytesWritten = await IOUtils.writeAtomicUTF8(tmpFileName, emoji, { compress: true });
|
||||
is(bytesWritten, 83, "Expected to write 64 bytes");
|
||||
|
||||
info("Test reading lz4 encoded UTF-8 string");
|
||||
let readData = await IOUtils.readUTF8(tmpFileName, { decompress: true });
|
||||
is(readData, emoji, "IOUtils can write and read back UTF-8 LZ4 encoded data");
|
||||
|
||||
info("Test writing lz4 compressed UTF-8 string");
|
||||
const lotsOfCoffee = new Array(24).fill('☕️').join(''); // ☕️ is 3 bytes in UTF-8: \0xe2 \0x98 \0x95
|
||||
bytesWritten = await IOUtils.writeAtomicUTF8(tmpFileName, lotsOfCoffee, { compress: true });
|
||||
console.log(bytesWritten);
|
||||
is(bytesWritten, 28, "Expected 72 bytes to compress to 28 bytes");
|
||||
|
||||
info("Test reading lz4 encoded UTF-8 string");
|
||||
readData = await IOUtils.readUTF8(tmpFileName, { decompress: true });
|
||||
is(readData, lotsOfCoffee, "IOUtils can write and read back UTF-8 LZ4 compressed data");
|
||||
|
||||
info("Test writing empty lz4 compressed UTF-8 string")
|
||||
const empty = "";
|
||||
bytesWritten = await IOUtils.writeAtomicUTF8(tmpFileName, empty, { compress: true });
|
||||
is(bytesWritten, 12, "Expected to write just the LZ4 header");
|
||||
|
||||
info("Test reading empty lz4 compressed UTF-8 string")
|
||||
const readEmpty = await IOUtils.readUTF8(tmpFileName, { decompress: true });
|
||||
is(readEmpty, empty, "IOUtils can write and read back empty buffers with LZ4");
|
||||
const readEmptyRaw = await IOUtils.readUTF8(tmpFileName, { decompress: false });
|
||||
is(readEmptyRaw.length, 12, "Expected to read back just the LZ4 header");
|
||||
|
||||
await cleanup(tmpFileName);
|
||||
});
|
||||
|
||||
add_task(async function test_lz4_bad_call() {
|
||||
const tmpFileName = OS.Path.join(tmpDir, "test_ioutils_lz4_bad_call.tmp");
|
||||
|
||||
info("readUTF8 ignores the maxBytes option if provided");
|
||||
const emoji = "☕️ ⚧️ 😀 🖖🏿 🤠 🏳️🌈 🥠 🏴☠️ 🪐";
|
||||
let bytesWritten = await IOUtils.writeAtomicUTF8(tmpFileName, emoji, { compress: true });
|
||||
is(bytesWritten, 83, "Expected to write 83 bytes");
|
||||
|
||||
let readData = await IOUtils.readUTF8(tmpFileName, { maxBytes: 4, decompress: true });
|
||||
is(readData, emoji, "IOUtils can write and read back UTF-8 LZ4 encoded data");
|
||||
|
||||
await cleanup(tmpFileName)
|
||||
});
|
||||
|
||||
add_task(async function test_utf8_lz4_failure() {
|
||||
const tmpFileName = OS.Path.join(tmpDir, "test_ioutils_lz4_fail.tmp");
|
||||
|
||||
info("Test decompression of non-lz4 UTF-8 string");
|
||||
const repeatedBytes = Uint8Array.of(...new Array(50).fill(1));
|
||||
await IOUtils.writeAtomic(tmpFileName, repeatedBytes, { compress: false });
|
||||
|
||||
await Assert.rejects(
|
||||
IOUtils.readUTF8(tmpFileName, { decompress: true }),
|
||||
/Could not decompress file because it has an invalid LZ4 header \(wrong magic number: .*\)/,
|
||||
"IOUtils::readUTF8 fails to decompress LZ4 data with a bad header"
|
||||
);
|
||||
|
||||
info("Test UTF-8 decompression of short byte buffer");
|
||||
const elevenBytes = Uint8Array.of(...new Array(11).fill(1));
|
||||
await IOUtils.writeAtomic(tmpFileName, elevenBytes, { compress: false });
|
||||
|
||||
await Assert.rejects(
|
||||
IOUtils.readUTF8(tmpFileName, { decompress: true }),
|
||||
/Could not decompress file because the buffer is too short/,
|
||||
"IOUtils::readUTF8 fails to decompress LZ4 data with missing header"
|
||||
);
|
||||
|
||||
info("Test UTF-8 decompression of valid header, but corrupt contents");
|
||||
const headerFor10bytes = [109, 111, 122, 76, 122, 52, 48, 0, 10, 0, 0, 0] // 'mozlz40\0' + 4 byte length
|
||||
const badContents = new Array(11).fill(255); // Bad leading byte, followed by uncompressed stream.
|
||||
const goodHeaderBadContents = Uint8Array.of(...headerFor10bytes, ...badContents);
|
||||
await IOUtils.writeAtomic(tmpFileName, goodHeaderBadContents, { compress: false });
|
||||
|
||||
await Assert.rejects(
|
||||
IOUtils.readUTF8(tmpFileName, { decompress: true }),
|
||||
/Could not decompress file contents, the file may be corrupt/,
|
||||
"IOUtils::readUTF8 fails to read corrupt LZ4 contents with a correct header"
|
||||
);
|
||||
|
||||
await cleanup(tmpFileName);
|
||||
});
|
||||
</script>
|
||||
</head>
|
||||
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче