From 8f667a0d22d372a5d52a1220c93daca618539310 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Fri, 7 Apr 2017 14:48:25 +0900 Subject: [PATCH] Bug 1355661 - Add support for brotli streams in Jar archives. r=aklotz Modern compression algorithms are better than zlib both in terms of space and time. The jar format, used for e.g. omni.ja, addons, etc. could benefit from using such modern algorithms, but the format only allows a limited set of compression algorithms. However, the format in itself is flexible, in that it can be extended with arbitrary compression algorithms. This breaks compatibility with programs like unzip, obviously, but we've never promised the files shipped with Firefox will always remain "valid" zips (which they already aren't, but they currently work with most zip readers). With this change, we allow those archives to contain brotli streams, using an arbitrary large value for the compression type in the Zip local file header. This only allows to read such archives, but not to produce them, and, for now, support for brotli streams is kept Nightly-only, until everything is pieced together and we're happy to ship it. --HG-- extra : rebase_source : fa637251f460ad0d91d5f5bec392c6e59555e80d --- modules/libjar/moz.build | 4 ++ modules/libjar/nsJARInputStream.cpp | 69 ++++++++++++++++++++++++++--- modules/libjar/nsJARInputStream.h | 24 ++++++++-- modules/libjar/nsZipArchive.cpp | 47 +++++++++++++++++++- modules/libjar/nsZipArchive.h | 6 +++ modules/libjar/zipstruct.h | 5 ++- 6 files changed, 143 insertions(+), 12 deletions(-) diff --git a/modules/libjar/moz.build b/modules/libjar/moz.build index 7860db80c5cd..d2f9db971afe 100644 --- a/modules/libjar/moz.build +++ b/modules/libjar/moz.build @@ -44,3 +44,7 @@ UNIFIED_SOURCES += [ include('/ipc/chromium/chromium-config.mozbuild') FINAL_LIBRARY = 'xul' + +LOCAL_INCLUDES += [ + '/modules/brotli/dec', +] diff --git a/modules/libjar/nsJARInputStream.cpp b/modules/libjar/nsJARInputStream.cpp index 086c21ab4fee..d4447315ac92 100644 --- a/modules/libjar/nsJARInputStream.cpp +++ b/modules/libjar/nsJARInputStream.cpp @@ -7,6 +7,9 @@ #include "nsJARInputStream.h" #include "zipstruct.h" // defines ZIP compression codes +#ifdef MOZ_JAR_BROTLI +#include "decode.h" // brotli +#endif #include "nsZipArchive.h" #include "nsEscape.h" @@ -51,6 +54,15 @@ nsJARInputStream::InitFile(nsJAR *aJar, nsZipItem *item) mOutCrc = crc32(0L, Z_NULL, 0); break; +#ifdef MOZ_JAR_BROTLI + case MOZ_JAR_BROTLI: + mBrotliState = BrotliCreateState(nullptr, nullptr, nullptr); + mMode = MODE_BROTLI; + mInCrc = item->CRC32(); + mOutCrc = crc32(0L, Z_NULL, 0); + break; +#endif + default: return NS_ERROR_NOT_IMPLEMENTED; } @@ -166,6 +178,9 @@ nsJARInputStream::Available(uint64_t *_retval) break; case MODE_INFLATE: +#ifdef MOZ_JAR_BROTLI + case MODE_BROTLI: +#endif case MODE_COPY: *_retval = mOutSize - mZs.total_out; break; @@ -195,6 +210,9 @@ MOZ_WIN_MEM_TRY_BEGIN return ReadDirectory(aBuffer, aCount, aBytesRead); case MODE_INFLATE: +#ifdef MOZ_JAR_BROTLI + case MODE_BROTLI: +#endif if (mZs.total_out < mOutSize) { rv = ContinueInflate(aBuffer, aCount, aBytesRead); } @@ -246,6 +264,11 @@ nsJARInputStream::Close() if (mMode == MODE_INFLATE) { inflateEnd(&mZs); } +#ifdef MOZ_JAR_BROTLI + if (mMode == MODE_BROTLI) { + BrotliDestroyState(mBrotliState); + } +#endif mMode = MODE_CLOSED; mFd = nullptr; return NS_OK; @@ -255,6 +278,8 @@ nsresult nsJARInputStream::ContinueInflate(char* aBuffer, uint32_t aCount, uint32_t* aBytesRead) { + bool finished = false; + // No need to check the args, ::Read did that, but assert them at least NS_ASSERTION(aBuffer,"aBuffer parameter must not be null"); NS_ASSERTION(aBytesRead,"aBytesRead parameter must not be null"); @@ -266,11 +291,39 @@ nsJARInputStream::ContinueInflate(char* aBuffer, uint32_t aCount, mZs.avail_out = std::min(aCount, (mOutSize-oldTotalOut)); mZs.next_out = (unsigned char*)aBuffer; - // now inflate - int zerr = inflate(&mZs, Z_SYNC_FLUSH); - if ((zerr != Z_OK) && (zerr != Z_STREAM_END)) { - nsZipArchive::sFileCorruptedReason = "nsJARInputStream: error while inflating"; - return NS_ERROR_FILE_CORRUPTED; +#ifndef MOZ_JAR_BROTLI + MOZ_ASSERT(mMode == MODE_INFLATE); +#endif + if (mMode == MODE_INFLATE) { + // now inflate + int zerr = inflate(&mZs, Z_SYNC_FLUSH); + if ((zerr != Z_OK) && (zerr != Z_STREAM_END)) { + nsZipArchive::sFileCorruptedReason = "nsJARInputStream: error while inflating"; + return NS_ERROR_FILE_CORRUPTED; + } + finished = (zerr == Z_STREAM_END); +#ifdef MOZ_JAR_BROTLI + } else { + MOZ_ASSERT(mMode == MODE_BROTLI); + /* The brotli library wants size_t, but z_stream only contains + * unsigned int for avail_* and unsigned long for total_*. + * So use temporary stack values. */ + size_t avail_in = mZs.avail_in; + size_t avail_out = mZs.avail_out; + size_t total_out = mZs.total_out; + BrotliResult result = BrotliDecompressStream( + &avail_in, const_cast(&mZs.next_in), + &avail_out, &mZs.next_out, &total_out, mBrotliState); + /* We don't need to update avail_out, it's not used outside this + * function. */ + mZs.total_out = total_out; + mZs.avail_in = avail_in; + if (result == BROTLI_RESULT_ERROR) { + nsZipArchive::sFileCorruptedReason = "nsJARInputStream: brotli decompression error"; + return NS_ERROR_FILE_CORRUPTED; + } + finished = (result == BROTLI_RESULT_SUCCESS); +#endif } *aBytesRead = (mZs.total_out - oldTotalOut); @@ -280,8 +333,10 @@ nsJARInputStream::ContinueInflate(char* aBuffer, uint32_t aCount, // be aggressive about ending the inflation // for some reason we don't always get Z_STREAM_END - if (zerr == Z_STREAM_END || mZs.total_out == mOutSize) { - inflateEnd(&mZs); + if (finished || mZs.total_out == mOutSize) { + if (mMode == MODE_INFLATE) { + inflateEnd(&mZs); + } // stop returning valid data as soon as we know we have a bad CRC if (mOutCrc != mInCrc) { diff --git a/modules/libjar/nsJARInputStream.h b/modules/libjar/nsJARInputStream.h index 1c396aa102e8..0affc950644b 100644 --- a/modules/libjar/nsJARInputStream.h +++ b/modules/libjar/nsJARInputStream.h @@ -12,6 +12,10 @@ #include "nsTArray.h" #include "mozilla/Attributes.h" +#ifdef MOZ_JAR_BROTLI +struct BrotliStateStruct; +#endif + /*------------------------------------------------------------------------- * Class nsJARInputStream declaration. This class defines the type of the * object returned by calls to nsJAR::GetInputStream(filename) for the @@ -20,9 +24,17 @@ class nsJARInputStream final : public nsIInputStream { public: - nsJARInputStream() : - mOutSize(0), mInCrc(0), mOutCrc(0), mNameLen(0), - mCurPos(0), mArrPos(0), mMode(MODE_NOTINITED) + nsJARInputStream() + : mOutSize(0) + , mInCrc(0) + , mOutCrc(0) +#ifdef MOZ_JAR_BROTLI + , mBrotliState(nullptr) +#endif + , mNameLen(0) + , mCurPos(0) + , mArrPos(0) + , mMode(MODE_NOTINITED) { memset(&mZs, 0, sizeof(z_stream)); } @@ -45,6 +57,9 @@ class nsJARInputStream final : public nsIInputStream uint32_t mInCrc; // CRC as provided by the zipentry uint32_t mOutCrc; // CRC as calculated by me z_stream mZs; // zip data structure +#ifdef MOZ_JAR_BROTLI + BrotliStateStruct* mBrotliState; // Brotli decoder state +#endif /* For directory reading */ RefPtr mJar; // string reference to zipreader @@ -59,6 +74,9 @@ class nsJARInputStream final : public nsIInputStream MODE_CLOSED, MODE_DIRECTORY, MODE_INFLATE, +#ifdef MOZ_JAR_BROTLI + MODE_BROTLI, +#endif MODE_COPY } JISMode; diff --git a/modules/libjar/nsZipArchive.cpp b/modules/libjar/nsZipArchive.cpp index 7993ebf474c2..938d9f83e1dd 100644 --- a/modules/libjar/nsZipArchive.cpp +++ b/modules/libjar/nsZipArchive.cpp @@ -12,6 +12,9 @@ #define READTYPE int32_t #include "zlib.h" +#ifdef MOZ_JAR_BROTLI +#include "decode.h" // brotli +#endif #include "nsISupportsUtils.h" #include "prio.h" #include "plstr.h" @@ -1168,6 +1171,9 @@ nsZipCursor::nsZipCursor(nsZipItem *item, nsZipArchive *aZip, uint8_t* aBuf, : mItem(item) , mBuf(aBuf) , mBufSize(aBufSize) +#ifdef MOZ_JAR_BROTLI + , mBrotliState(nullptr) +#endif , mCRC(0) , mDoCRC(doCRC) { @@ -1182,6 +1188,12 @@ nsZipCursor::nsZipCursor(nsZipItem *item, nsZipArchive *aZip, uint8_t* aBuf, mZs.avail_in = item->Size(); mZs.next_in = (Bytef*)aZip->GetData(item); + +#ifdef MOZ_JAR_BROTLI + if (mItem->Compression() == MOZ_JAR_BROTLI) { + mBrotliState = BrotliCreateState(nullptr, nullptr, nullptr); + } +#endif if (doCRC) mCRC = crc32(0L, Z_NULL, 0); @@ -1192,6 +1204,11 @@ nsZipCursor::~nsZipCursor() if (mItem->Compression() == DEFLATED) { inflateEnd(&mZs); } +#ifdef MOZ_JAR_BROTLI + if (mItem->Compression() == MOZ_JAR_BROTLI) { + BrotliDestroyState(mBrotliState); + } +#endif } uint8_t* nsZipCursor::ReadOrCopy(uint32_t *aBytesRead, bool aCopy) { @@ -1228,6 +1245,30 @@ MOZ_WIN_MEM_TRY_BEGIN *aBytesRead = mZs.next_out - buf; verifyCRC = (zerr == Z_STREAM_END); break; +#ifdef MOZ_JAR_BROTLI + case MOZ_JAR_BROTLI: { + buf = mBuf; + mZs.next_out = buf; + /* The brotli library wants size_t, but z_stream only contains + * unsigned int for avail_*. So use temporary stack values. */ + size_t avail_out = mBufSize; + size_t avail_in = mZs.avail_in; + BrotliResult result = BrotliDecompressStream( + &avail_in, const_cast(&mZs.next_in), + &avail_out, &mZs.next_out, nullptr, mBrotliState); + /* We don't need to update avail_out, it's not used outside this + * function. */ + mZs.avail_in = avail_in; + + if (result == BROTLI_RESULT_ERROR) { + return nullptr; + } + + *aBytesRead = mZs.next_out - buf; + verifyCRC = (result == BROTLI_RESULT_SUCCESS); + break; + } +#endif default: return nullptr; } @@ -1254,7 +1295,11 @@ nsZipItemPtr_base::nsZipItemPtr_base(nsZipArchive *aZip, return; uint32_t size = 0; - if (item->Compression() == DEFLATED) { + bool compressed = (item->Compression() == DEFLATED); +#ifdef MOZ_JAR_BROTLI + compressed |= (item->Compression() == MOZ_JAR_BROTLI); +#endif + if (compressed) { size = item->RealSize(); mAutoBuf = MakeUniqueFallible(size); if (!mAutoBuf) { diff --git a/modules/libjar/nsZipArchive.h b/modules/libjar/nsZipArchive.h index 373492370468..4ae896e3172b 100644 --- a/modules/libjar/nsZipArchive.h +++ b/modules/libjar/nsZipArchive.h @@ -37,6 +37,9 @@ class nsZipFind; struct PRFileDesc; +#ifdef MOZ_JAR_BROTLI +struct BrotliStateStruct; +#endif /** * This file defines some of the basic structures used by libjar to @@ -314,6 +317,9 @@ private: uint8_t *mBuf; uint32_t mBufSize; z_stream mZs; +#ifdef MOZ_JAR_BROTLI + BrotliStateStruct* mBrotliState; +#endif uint32_t mCRC; bool mDoCRC; }; diff --git a/modules/libjar/zipstruct.h b/modules/libjar/zipstruct.h index f06afe14ea41..257d6d4d3522 100644 --- a/modules/libjar/zipstruct.h +++ b/modules/libjar/zipstruct.h @@ -102,6 +102,9 @@ typedef struct ZipEnd_ #define TOKENIZED 7 #define DEFLATED 8 #define UNSUPPORTED 0xFF - +/* non-standard extension */ +#ifdef NIGHTLY_BUILD +#define MOZ_JAR_BROTLI 0x81 +#endif #endif /* _zipstruct_h */