diff --git a/mozglue/linker/Makefile.in b/mozglue/linker/Makefile.in index 2302adec489a..72b262827a22 100644 --- a/mozglue/linker/Makefile.in +++ b/mozglue/linker/Makefile.in @@ -19,6 +19,17 @@ CPPSRCS = \ $(NULL) ifndef MOZ_OLD_LINKER +HOST_PROGRAM = szip + +HOST_CPPSRCS = \ + szip.cpp \ + Assertions.cpp \ + $(NULL) + +VPATH += $(topsrcdir)/mfbt + +HOST_LIBS = -lz + CPPSRCS += \ ElfLoader.cpp \ CustomElf.cpp \ diff --git a/mozglue/linker/SeekableZStream.h b/mozglue/linker/SeekableZStream.h new file mode 100644 index 000000000000..7659a7964468 --- /dev/null +++ b/mozglue/linker/SeekableZStream.h @@ -0,0 +1,48 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef SeekableZStream_h +#define SeekableZStream_h + +#include "Zip.h" + +/** + * Seekable compressed stream are created by splitting the original + * decompressed data in small chunks and compress these chunks + * individually. + * + * The seekable compressed file format consists in a header defined below, + * followed by a table of 32-bits words containing the offsets for each + * individual compressed chunk, then followed by the compressed chunks. + */ + +#pragma pack(1) +struct SeekableZStreamHeader: public Zip::SignedEntity +{ + SeekableZStreamHeader() + : Zip::SignedEntity(magic) + , totalSize(0), chunkSize(0), nChunks(0), lastChunkSize(0) { } + + /* Reuse Zip::SignedEntity to handle the magic number used in the Seekable + * ZStream file format. The magic number is "SeZz". */ + static const uint32_t magic = 0x7a5a6553; + + /* Total size of the stream, including the 4 magic bytes. */ + le_uint32 totalSize; + + /* Chunk size */ + le_uint32 chunkSize; + + /* Number of chunks */ + le_uint32 nChunks; + + /* Size of last chunk (> 0, <= Chunk size) */ + le_uint32 lastChunkSize; +}; +#pragma pack() + +MOZ_STATIC_ASSERT(sizeof(SeekableZStreamHeader) == 5 * 4, + "SeekableZStreamHeader should be 5 32-bits words"); + +#endif /* SeekableZStream_h */ diff --git a/mozglue/linker/Utils.h b/mozglue/linker/Utils.h index 554187e5bc7e..5f4468213012 100644 --- a/mozglue/linker/Utils.h +++ b/mozglue/linker/Utils.h @@ -16,7 +16,7 @@ * we can use direct type, but on others, we want to have a special class * to handle conversion and alignment issues. */ -#if defined(__i386__) || defined(__x86_64__) +#if !defined(DEBUG) && (defined(__i386__) || defined(__x86_64__)) typedef uint16_t le_uint16; typedef uint32_t le_uint32; #else @@ -29,17 +29,43 @@ template <> struct UInt<16> { typedef uint16_t Type; }; template <> struct UInt<32> { typedef uint32_t Type; }; /** - * Template to read 2 n-bit sized words as a 2*n-bit sized word, doing + * Template to access 2 n-bit sized words as a 2*n-bit sized word, doing * conversion from little endian and avoiding alignment issues. */ template class le_to_cpu { public: - operator typename UInt<16 * sizeof(T)>::Type() const + typedef typename UInt<16 * sizeof(T)>::Type Type; + + operator Type() const { return (b << (sizeof(T) * 8)) | a; } + + const le_to_cpu& operator =(const Type &v) + { + a = v & ((1 << (sizeof(T) * 8)) - 1); + b = v >> (sizeof(T) * 8); + return *this; + } + + le_to_cpu() { } + le_to_cpu(const Type &v) + { + operator =(v); + } + + const le_to_cpu& operator +=(const Type &v) + { + return operator =(operator Type() + v); + } + + const le_to_cpu& operator ++(int) + { + return operator =(operator Type() + 1); + } + private: T a, b; }; diff --git a/mozglue/linker/Zip.h b/mozglue/linker/Zip.h index f85fb7c83f45..0ebcfd894bf7 100644 --- a/mozglue/linker/Zip.h +++ b/mozglue/linker/Zip.h @@ -142,6 +142,7 @@ private: /* All the following types need to be packed */ #pragma pack(1) +public: /** * A Zip archive is an aggregate of entities which all start with a * signature giving their type. This template is to be used as a base @@ -162,10 +163,13 @@ private: return ret; return NULL; } + + SignedEntity(uint32_t magic): signature(magic) { } private: le_uint32 signature; }; +private: /** * Header used to describe a Local File entry. The header is followed by * the file name and an extra field, then by the data stream. diff --git a/mozglue/linker/szip.cpp b/mozglue/linker/szip.cpp new file mode 100644 index 000000000000..213a353381ce --- /dev/null +++ b/mozglue/linker/szip.cpp @@ -0,0 +1,136 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include +#include +#include +#include +#include +#include +#include "mozilla/Assertions.h" +#include "SeekableZStream.h" +#include "Utils.h" +#include "Logging.h" + +static const size_t CHUNK = 16384; + +/* Generate a seekable compressed stream. */ + +int main(int argc, char* argv[]) +{ + if (argc != 3 || !argv[1] || !argv[2] || (strcmp(argv[1], argv[2]) == 0)) { + log("usage: %s file_to_compress out_file", argv[0]); + return 1; + } + + AutoCloseFD origFd = open(argv[1], O_RDONLY); + if (origFd == -1) { + log("Couldn't open %s: %s", argv[1], strerror(errno)); + return 1; + } + + struct stat st; + int ret = fstat(origFd, &st); + if (ret == -1) { + log("Couldn't seek %s: %s", argv[1], strerror(errno)); + return 1; + } + + size_t origSize = st.st_size; + log("Size = %lu", origSize); + if (origSize == 0) { + log("Won't compress %s: it's empty", argv[1]); + return 1; + } + + /* Mmap the original file */ + MappedPtr origBuf; + origBuf.Assign(mmap(NULL, origSize, PROT_READ, MAP_PRIVATE, origFd, 0), origSize); + if (origBuf == MAP_FAILED) { + log("Couldn't mmap %s: %s", argv[1], strerror(errno)); + return 1; + } + + /* Create the compressed file */ + AutoCloseFD outFd = open(argv[2], O_RDWR | O_CREAT | O_TRUNC, 0666); + if (outFd == -1) { + log("Couldn't open %s: %s", argv[2], strerror(errno)); + return 1; + } + + /* Expected total number of chunks */ + size_t nChunks = ((origSize + CHUNK - 1) / CHUNK); + + /* The first chunk is going to be stored after the header and the offset + * table */ + size_t offset = sizeof(SeekableZStreamHeader) + nChunks * sizeof(uint32_t); + + /* Give enough room for the header and the offset table, and map them */ + ret = posix_fallocate(outFd, 0, offset); + MOZ_ASSERT(ret == 0); + MappedPtr headerMap; + headerMap.Assign(mmap(NULL, offset, PROT_READ | PROT_WRITE, MAP_SHARED, + outFd, 0), offset); + if (headerMap == MAP_FAILED) { + log("Couldn't mmap %s: %s", argv[1], strerror(errno)); + return 1; + } + + SeekableZStreamHeader *header = new (headerMap) SeekableZStreamHeader; + le_uint32 *entry = reinterpret_cast(&header[1]); + + /* Initialize header */ + header->chunkSize = CHUNK; + header->totalSize = offset; + + /* Seek at the end of the output file, where we're going to append + * compressed streams */ + lseek(outFd, offset, SEEK_SET); + + /* Initialize zlib structure */ + z_stream zStream; + memset(&zStream, 0, sizeof(zStream)); + + /* Compression buffer */ + AutoDeleteArray outBuf = new Bytef[CHUNK * 2]; + + Bytef *origData = static_cast(origBuf); + size_t avail = 0; + while (origSize) { + avail = std::min(origSize, CHUNK); + + /* Compress chunk */ + ret = deflateInit(&zStream, 9); + MOZ_ASSERT(ret == Z_OK); + zStream.avail_in = avail; + zStream.next_in = origData; + zStream.avail_out = CHUNK * 2; + zStream.next_out = outBuf; + ret = deflate(&zStream, Z_FINISH); + MOZ_ASSERT(ret == Z_STREAM_END); + ret = deflateEnd(&zStream); + MOZ_ASSERT(ret == Z_OK); + MOZ_ASSERT(zStream.avail_out > 0); + + /* Write chunk */ + size_t len = write(outFd, outBuf, 2 * CHUNK - zStream.avail_out); + MOZ_ASSERT(len == 2 * CHUNK - zStream.avail_out); + + /* Adjust headers */ + header->totalSize += len; + *entry++ = offset; + header->nChunks++; + + /* Prepare for next iteration */ + origSize -= avail; + origData += avail; + offset += len; + } + header->lastChunkSize = avail; + + MOZ_ASSERT(header->nChunks == nChunks); + log("Compressed size is %lu", offset); + + return 0; +}