From b41725bb4faea918d25aadf730f9209792b6e1a1 Mon Sep 17 00:00:00 2001 From: Tooru Fujisawa Date: Tue, 21 May 2019 15:34:30 +0000 Subject: [PATCH] Bug 1551473 - Provide utility methods to read brotli stream in BinASTTokenReaderContext. r=Yoric Differential Revision: https://phabricator.services.mozilla.com/D31212 --HG-- extra : moz-landing-system : lando --- js/src/frontend/BinASTTokenReaderContext.cpp | 88 ++++++++++++++++++-- js/src/frontend/BinASTTokenReaderContext.h | 56 ++++++++++++- 2 files changed, 136 insertions(+), 8 deletions(-) diff --git a/js/src/frontend/BinASTTokenReaderContext.cpp b/js/src/frontend/BinASTTokenReaderContext.cpp index 5daec122cab5..91c3737ea1c1 100644 --- a/js/src/frontend/BinASTTokenReaderContext.cpp +++ b/js/src/frontend/BinASTTokenReaderContext.cpp @@ -8,7 +8,7 @@ #include "mozilla/Result.h" // MOZ_TRY* -#include // memcmp +#include // memcmp, memmove #include "frontend/BinAST-macros.h" // BINJS_TRY*, BINJS_MOZ_TRY* #include "vm/JSScript.h" // ScriptSource @@ -43,6 +43,72 @@ BinASTTokenReaderContext::~BinASTTokenReaderContext() { if (metadata_ && metadataOwned_ == MetadataOwnership::Owned) { UniqueBinASTSourceMetadataPtr ptr(metadata_); } + if (decoder_) { + BrotliDecoderDestroyInstance(decoder_); + } +} + +template <> +JS::Result +BinASTTokenReaderContext::readBuf( + uint8_t* bytes, uint32_t len) { + return Base::readBuf(bytes, len); +} + +template <> +JS::Result +BinASTTokenReaderContext::readBuf( + uint8_t* bytes, uint32_t len) { + while (availableDecodedLength() < len) { + if (availableDecodedLength()) { + memmove(bytes, decodedBufferBegin(), availableDecodedLength()); + bytes += availableDecodedLength(); + len -= availableDecodedLength(); + } + + if (isEOF()) { + return raiseError("Unexpected end of file"); + } + + // We have exhausted the in-memory buffer. Start from the beginning. + decodedBegin_ = 0; + + size_t inSize = stop_ - current_; + size_t outSize = DECODED_BUFFER_SIZE; + uint8_t* out = decodedBuffer_; + + BrotliDecoderResult result; + result = BrotliDecoderDecompressStream(decoder_, &inSize, ¤t_, + &outSize, &out, + /* total_out = */ nullptr); + if (result == BROTLI_DECODER_RESULT_ERROR) { + return raiseError("Failed to decompress brotli stream"); + } + + decodedEnd_ = out - decodedBuffer_; + } + + memmove(bytes, decodedBufferBegin(), len); + decodedBegin_ += len; + return Ok(); +} + +bool BinASTTokenReaderContext::isEOF() const { + return BrotliDecoderIsFinished(decoder_); +} + +template <> +JS::Result BinASTTokenReaderContext::readByte< + BinASTTokenReaderContext::Compression::No>() { + return Base::readByte(); +} + +template <> +JS::Result BinASTTokenReaderContext::readByte< + BinASTTokenReaderContext::Compression::Yes>() { + uint8_t buf; + MOZ_TRY(readBuf(&buf, 1)); + return buf; } BinASTSourceMetadata* BinASTTokenReaderContext::takeMetadata() { @@ -65,13 +131,20 @@ JS::Result BinASTTokenReaderContext::readHeader() { // Read global headers. MOZ_TRY(readConst(CX_MAGIC_HEADER)); - BINJS_MOZ_TRY_DECL(version, readVarU32()); + BINJS_MOZ_TRY_DECL(version, readVarU32()); if (version != MAGIC_FORMAT_VERSION) { return raiseError("Format version not implemented"); } - // TODO: handle `LinkToSharedDictionary` and remaining things here. + decoder_ = BrotliDecoderCreateInstance(/* alloc_func = */ nullptr, + /* free_func = */ nullptr, + /* opaque = */ nullptr); + if (!decoder_) { + return raiseError("Failed to create brotli decoder"); + } + + // TODO: handle strings and models here. return raiseError("Not Yet Implemented"); } @@ -169,13 +242,14 @@ JS::Result BinASTTokenReaderContext::AutoList::done() { // // Encoded as variable length number. -MOZ_MUST_USE JS::Result BinASTTokenReaderContext::readVarU32() { +template +JS::Result BinASTTokenReaderContext::readVarU32() { uint32_t result = 0; uint32_t shift = 0; while (true) { MOZ_ASSERT(shift < 32); uint32_t byte; - MOZ_TRY_VAR(byte, readByte()); + MOZ_TRY_VAR(byte, readByte()); const uint32_t newResult = result | (byte & 0x7f) << shift; if (newResult < result) { @@ -195,6 +269,10 @@ MOZ_MUST_USE JS::Result BinASTTokenReaderContext::readVarU32() { } } +JS::Result BinASTTokenReaderContext::readUnsignedLong(const Context&) { + return readVarU32(); +} + BinASTTokenReaderContext::AutoTaggedTuple::AutoTaggedTuple( BinASTTokenReaderContext& reader) : AutoBase(reader) {} diff --git a/js/src/frontend/BinASTTokenReaderContext.h b/js/src/frontend/BinASTTokenReaderContext.h index be5344dcbe02..7c2148cc6dd4 100644 --- a/js/src/frontend/BinASTTokenReaderContext.h +++ b/js/src/frontend/BinASTTokenReaderContext.h @@ -12,6 +12,8 @@ #include "mozilla/Maybe.h" // mozilla::Maybe +#include // BrotliDecoderState + #include // size_t #include // uint8_t, uint32_t @@ -48,6 +50,8 @@ class ErrorReporter; * - the reader does not support lookahead or pushback. */ class MOZ_STACK_CLASS BinASTTokenReaderContext : public BinASTTokenReaderBase { + using Base = BinASTTokenReaderBase; + public: class AutoList; class AutoTaggedTuple; @@ -82,6 +86,51 @@ class MOZ_STACK_CLASS BinASTTokenReaderContext : public BinASTTokenReaderBase { ~BinASTTokenReaderContext(); + private: + // {readByte, readBuf, readVarU32} are implemented both for uncompressed + // stream and brotli-compressed stream. + // + // Uncompressed variant is for reading the magic header, and compressed + // variant is for reading the remaining part. + // + // Once compressed variant is called, the underlying uncompressed stream is + // buffered and uncompressed variant cannot be called. + enum class Compression { No, Yes }; + + // Buffer that holds already brotli-decoded but not yet used data. + // decodedBuffer[decodedBegin, decodedEnd) holds the data. + static const size_t DECODED_BUFFER_SIZE = 128; + uint8_t decodedBuffer_[DECODED_BUFFER_SIZE]; + size_t decodedBegin_ = 0; + size_t decodedEnd_ = 0; + + // The number of already decoded bytes. + size_t availableDecodedLength() const { return decodedEnd_ - decodedBegin_; } + + // The beginning of decoded buffer. + const uint8_t* decodedBufferBegin() const { + return decodedBuffer_ + decodedBegin_; + } + + // Returns true if the brotli stream finished. + bool isEOF() const; + + /** + * Read a single byte. + */ + template + MOZ_MUST_USE JS::Result readByte(); + + /** + * Read several bytes. + * + * If there is not enough data, or if the tokenizer has previously been + * poisoned, return an error. + */ + template + MOZ_MUST_USE JS::Result readBuf(uint8_t* bytes, uint32_t len); + + public: /** * Read the header of the file. */ @@ -196,14 +245,13 @@ class MOZ_STACK_CLASS BinASTTokenReaderContext : public BinASTTokenReaderBase { /** * Read a single unsigned long. */ - MOZ_MUST_USE JS::Result readUnsignedLong(const Context&) { - return readVarU32(); - } + MOZ_MUST_USE JS::Result readUnsignedLong(const Context&); private: /** * Read a single uint32_t. */ + template MOZ_MUST_USE JS::Result readVarU32(); private: @@ -219,6 +267,8 @@ class MOZ_STACK_CLASS BinASTTokenReaderContext : public BinASTTokenReaderBase { const uint8_t* posBeforeTree_; + BrotliDecoderState* decoder_ = nullptr; + public: BinASTTokenReaderContext(const BinASTTokenReaderContext&) = delete; BinASTTokenReaderContext(BinASTTokenReaderContext&&) = delete;