зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1340901 - Update Snappy to version 1.1.8. r=dom-workers-and-storage-reviewers,asuth
Add a static assertion in IndexedDB to detect future updates. Differential Revision: https://phabricator.services.mozilla.com/D56708
This commit is contained in:
ssengupta
Родитель
9b4822e0ff
Коммит
070a49e060
|
|
@ -61,6 +61,8 @@ class nsIFile;
|
|||
|
||||
namespace mozilla::dom::indexedDB {
|
||||
|
||||
static_assert(SNAPPY_VERSION == 0x010108);
|
||||
|
||||
using mozilla::ipc::IsOnBackgroundThread;
|
||||
|
||||
namespace {
|
||||
|
|
|
|||
|
|
@ -4,23 +4,21 @@ Mozilla does not modify the actual snappy source with the exception of the
|
|||
'snappy-stubs-public.h' header. We have replaced its build system with our own.
|
||||
|
||||
Snappy comes from:
|
||||
http://code.google.com/p/snappy/
|
||||
https://github.com/google/snappy
|
||||
|
||||
We are currently using revision: 114
|
||||
We are currently using revision: 1.1.8
|
||||
|
||||
To upgrade to a newer version:
|
||||
1. Check out the new code using subversion.
|
||||
2. Update 'snappy-stubs-public.h' in this directory with any changes that were
|
||||
made to 'snappy-stubs-public.h.in' in the new source.
|
||||
3. Copy the major/minor/patch versions from 'configure.ac' into
|
||||
3. Copy the major/minor/patch versions from 'CMakeLists.txt' into
|
||||
'snappy-stubs-public.h'.
|
||||
4. Copy all source files from the new version into the src subdirectory. The
|
||||
following files are not needed:
|
||||
- 'autom4te.cache' subdirectory
|
||||
- 'm4' subdirectory
|
||||
following are not needed:
|
||||
- 'CMakeLists.txt' file
|
||||
- 'cmake' subdirectory
|
||||
- 'docs' subdirectory
|
||||
- 'testdata' subdirectory
|
||||
- 'autogen.sh'
|
||||
- 'configure.ac'
|
||||
- 'Makefile.am'
|
||||
- 'snappy.pc.in'
|
||||
5. Update the revision stamp in this file.
|
||||
|
||||
|
|
|
|||
|
|
@ -39,14 +39,14 @@
|
|||
#include <stdint.h>
|
||||
|
||||
#if defined IS_BIG_ENDIAN || defined __BIG_ENDIAN__
|
||||
#define WORDS_BIGENDIAN
|
||||
# define WORDS_BIGENDIAN
|
||||
#endif
|
||||
|
||||
#define SNAPPY_MAJOR 1
|
||||
#define SNAPPY_MINOR 1
|
||||
#define SNAPPY_PATCHLEVEL 3
|
||||
#define SNAPPY_PATCHLEVEL 8
|
||||
#define SNAPPY_VERSION \
|
||||
((SNAPPY_MAJOR << 16) | (SNAPPY_MINOR << 8) | SNAPPY_PATCHLEVEL)
|
||||
((SNAPPY_MAJOR << 16) | (SNAPPY_MINOR << 8) | SNAPPY_PATCHLEVEL)
|
||||
|
||||
#include <string>
|
||||
|
||||
|
|
@ -64,22 +64,22 @@ typedef uint64_t uint64;
|
|||
typedef std::string string;
|
||||
|
||||
#ifndef DISALLOW_COPY_AND_ASSIGN
|
||||
#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
|
||||
TypeName(const TypeName&); \
|
||||
void operator=(const TypeName&)
|
||||
# define DISALLOW_COPY_AND_ASSIGN(TypeName) \
|
||||
TypeName(const TypeName&); \
|
||||
void operator=(const TypeName&)
|
||||
#endif
|
||||
|
||||
struct iovec {
|
||||
void* iov_base;
|
||||
size_t iov_len;
|
||||
void* iov_base;
|
||||
size_t iov_len;
|
||||
};
|
||||
|
||||
#if defined(_WIN32) || defined(_WIN64)
|
||||
#if defined(_WIN64)
|
||||
# if defined(_WIN64)
|
||||
typedef __int64 LONG_PTR;
|
||||
#else
|
||||
# else
|
||||
typedef long LONG_PTR;
|
||||
#endif
|
||||
# endif
|
||||
typedef LONG_PTR SSIZE_T;
|
||||
typedef SSIZE_T ssize_t;
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -0,0 +1,26 @@
|
|||
# How to Contribute
|
||||
|
||||
We'd love to accept your patches and contributions to this project. There are
|
||||
just a few small guidelines you need to follow.
|
||||
|
||||
## Contributor License Agreement
|
||||
|
||||
Contributions to this project must be accompanied by a Contributor License
|
||||
Agreement. You (or your employer) retain the copyright to your contribution,
|
||||
this simply gives us permission to use and redistribute your contributions as
|
||||
part of the project. Head over to <https://cla.developers.google.com/> to see
|
||||
your current agreements on file or to sign a new one.
|
||||
|
||||
You generally only need to submit a CLA once, so if you've already submitted one
|
||||
(even if it was for a different project), you probably don't need to do it
|
||||
again.
|
||||
|
||||
## Code reviews
|
||||
|
||||
All submissions, including submissions by project members, require review. We
|
||||
use GitHub pull requests for this purpose. Consult
|
||||
[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
|
||||
information on using pull requests.
|
||||
|
||||
Please make sure that all the automated checks (CLA, AppVeyor, Travis) pass for
|
||||
your pull requests. Pull requests whose checks fail may be ignored.
|
||||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
|
@ -1,3 +1,51 @@
|
|||
Snappy v1.1.8, January 15th 2020:
|
||||
|
||||
* Small performance improvements.
|
||||
|
||||
* Removed snappy::string alias for std::string.
|
||||
|
||||
* Improved CMake configuration.
|
||||
|
||||
Snappy v1.1.7, August 24th 2017:
|
||||
|
||||
* Improved CMake build support for 64-bit Linux distributions.
|
||||
|
||||
* MSVC builds now use MSVC-specific intrinsics that map to clzll.
|
||||
|
||||
* ARM64 (AArch64) builds use the code paths optimized for 64-bit processors.
|
||||
|
||||
Snappy v1.1.6, July 12th 2017:
|
||||
|
||||
This is a re-release of v1.1.5 with proper SONAME / SOVERSION values.
|
||||
|
||||
Snappy v1.1.5, June 28th 2017:
|
||||
|
||||
This release has broken SONAME / SOVERSION values. Users of snappy as a shared
|
||||
library should avoid 1.1.5 and use 1.1.6 instead. SONAME / SOVERSION errors will
|
||||
manifest as the dynamic library loader complaining that it cannot find snappy's
|
||||
shared library file (libsnappy.so / libsnappy.dylib), or that the library it
|
||||
found does not have the required version. 1.1.6 has the same code as 1.1.5, but
|
||||
carries build configuration fixes for the issues above.
|
||||
|
||||
* Add CMake build support. The autoconf build support is now deprecated, and
|
||||
will be removed in the next release.
|
||||
|
||||
* Add AppVeyor configuration, for Windows CI coverage.
|
||||
|
||||
* Small performance improvement on little-endian PowerPC.
|
||||
|
||||
* Small performance improvement on LLVM with position-independent executables.
|
||||
|
||||
* Fix a few issues with various build environments.
|
||||
|
||||
Snappy v1.1.4, January 25th 2017:
|
||||
|
||||
* Fix a 1% performance regression when snappy is used in PIE executables.
|
||||
|
||||
* Improve compression performance by 5%.
|
||||
|
||||
* Improve decompression performance by 20%.
|
||||
|
||||
Snappy v1.1.3, July 6th 2015:
|
||||
|
||||
This is the first release to be done from GitHub, which means that
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ Snappy is intended to be fast. On a single core of a Core i7 processor
|
|||
in 64-bit mode, it compresses at about 250 MB/sec or more and decompresses at
|
||||
about 500 MB/sec or more. (These numbers are for the slowest inputs in our
|
||||
benchmark suite; others are much faster.) In our tests, Snappy usually
|
||||
is faster than algorithms in the same class (e.g. LZO, LZF, FastLZ, QuickLZ,
|
||||
is faster than algorithms in the same class (e.g. LZO, LZF, QuickLZ,
|
||||
etc.) while achieving comparable compression ratios.
|
||||
|
||||
Typical compression ratios (based on the benchmark suite) are about 1.5-1.7x
|
||||
|
|
@ -51,8 +51,8 @@ In particular:
|
|||
|
||||
- Snappy uses 64-bit operations in several places to process more data at
|
||||
once than would otherwise be possible.
|
||||
- Snappy assumes unaligned 32- and 64-bit loads and stores are cheap.
|
||||
On some platforms, these must be emulated with single-byte loads
|
||||
- Snappy assumes unaligned 32 and 64-bit loads and stores are cheap.
|
||||
On some platforms, these must be emulated with single-byte loads
|
||||
and stores, which is much slower.
|
||||
- Snappy assumes little-endian throughout, and needs to byte-swap data in
|
||||
several places if running on a big-endian platform.
|
||||
|
|
@ -62,25 +62,40 @@ Performance optimizations, whether for 64-bit x86 or other platforms,
|
|||
are of course most welcome; see "Contact", below.
|
||||
|
||||
|
||||
Building
|
||||
========
|
||||
|
||||
You need the CMake version specified in [CMakeLists.txt](./CMakeLists.txt)
|
||||
or later to build:
|
||||
|
||||
```bash
|
||||
mkdir build
|
||||
cd build && cmake ../ && make
|
||||
```
|
||||
|
||||
Usage
|
||||
=====
|
||||
|
||||
Note that Snappy, both the implementation and the main interface,
|
||||
is written in C++. However, several third-party bindings to other languages
|
||||
are available; see the home page at http://google.github.io/snappy/
|
||||
for more information. Also, if you want to use Snappy from C code, you can
|
||||
use the included C bindings in snappy-c.h.
|
||||
are available; see the [home page](docs/README.md) for more information.
|
||||
Also, if you want to use Snappy from C code, you can use the included C
|
||||
bindings in snappy-c.h.
|
||||
|
||||
To use Snappy from your own C++ program, include the file "snappy.h" from
|
||||
your calling file, and link against the compiled library.
|
||||
|
||||
There are many ways to call Snappy, but the simplest possible is
|
||||
|
||||
snappy::Compress(input.data(), input.size(), &output);
|
||||
```c++
|
||||
snappy::Compress(input.data(), input.size(), &output);
|
||||
```
|
||||
|
||||
and similarly
|
||||
|
||||
snappy::Uncompress(input.data(), input.size(), &output);
|
||||
```c++
|
||||
snappy::Uncompress(input.data(), input.size(), &output);
|
||||
```
|
||||
|
||||
where "input" and "output" are both instances of std::string.
|
||||
|
||||
|
|
@ -102,12 +117,12 @@ tests to verify you have not broken anything. Note that if you have the
|
|||
Google Test library installed, unit test behavior (especially failures) will be
|
||||
significantly more user-friendly. You can find Google Test at
|
||||
|
||||
http://github.com/google/googletest
|
||||
https://github.com/google/googletest
|
||||
|
||||
You probably also want the gflags library for handling of command-line flags;
|
||||
you can find it at
|
||||
|
||||
http://gflags.github.io/gflags/
|
||||
https://gflags.github.io/gflags/
|
||||
|
||||
In addition to the unit tests, snappy contains microbenchmarks used to
|
||||
tune compression and decompression performance. These are automatically run
|
||||
|
|
@ -116,7 +131,7 @@ before the unit tests, but you can disable them using the flag
|
|||
need to edit the source).
|
||||
|
||||
Finally, snappy can benchmark Snappy against a few other compression libraries
|
||||
(zlib, LZO, LZF, FastLZ and QuickLZ), if they were detected at configure time.
|
||||
(zlib, LZO, LZF, and QuickLZ), if they were detected at configure time.
|
||||
To benchmark using a given file, give the compression algorithm you want to test
|
||||
Snappy against (e.g. --zlib) and then a list of one or more file names on the
|
||||
command line. The testdata/ directory contains the files used by the
|
||||
|
|
@ -130,10 +145,4 @@ Contact
|
|||
=======
|
||||
|
||||
Snappy is distributed through GitHub. For the latest version, a bug tracker,
|
||||
and other information, see
|
||||
|
||||
http://google.github.io/snappy/
|
||||
|
||||
or the repository at
|
||||
|
||||
https://github.com/google/snappy
|
||||
and other information, see https://github.com/google/snappy.
|
||||
|
|
@ -36,21 +36,30 @@
|
|||
namespace snappy {
|
||||
namespace internal {
|
||||
|
||||
// Working memory performs a single allocation to hold all scratch space
|
||||
// required for compression.
|
||||
class WorkingMemory {
|
||||
public:
|
||||
WorkingMemory() : large_table_(NULL) { }
|
||||
~WorkingMemory() { delete[] large_table_; }
|
||||
explicit WorkingMemory(size_t input_size);
|
||||
~WorkingMemory();
|
||||
|
||||
// Allocates and clears a hash table using memory in "*this",
|
||||
// stores the number of buckets in "*table_size" and returns a pointer to
|
||||
// the base of the hash table.
|
||||
uint16* GetHashTable(size_t input_size, int* table_size);
|
||||
uint16* GetHashTable(size_t fragment_size, int* table_size) const;
|
||||
char* GetScratchInput() const { return input_; }
|
||||
char* GetScratchOutput() const { return output_; }
|
||||
|
||||
private:
|
||||
uint16 small_table_[1<<10]; // 2KB
|
||||
uint16* large_table_; // Allocated only when needed
|
||||
char* mem_; // the allocated memory, never nullptr
|
||||
size_t size_; // the size of the allocated memory, never 0
|
||||
uint16* table_; // the pointer to the hashtable
|
||||
char* input_; // the pointer to the input scratch buffer
|
||||
char* output_; // the pointer to the output scratch buffer
|
||||
|
||||
DISALLOW_COPY_AND_ASSIGN(WorkingMemory);
|
||||
// No copying
|
||||
WorkingMemory(const WorkingMemory&);
|
||||
void operator=(const WorkingMemory&);
|
||||
};
|
||||
|
||||
// Flat array compression that does not emit the "uncompressed length"
|
||||
|
|
@ -70,57 +79,72 @@ char* CompressFragment(const char* input,
|
|||
uint16* table,
|
||||
const int table_size);
|
||||
|
||||
// Return the largest n such that
|
||||
// Find the largest n such that
|
||||
//
|
||||
// s1[0,n-1] == s2[0,n-1]
|
||||
// and n <= (s2_limit - s2).
|
||||
//
|
||||
// Return make_pair(n, n < 8).
|
||||
// Does not read *s2_limit or beyond.
|
||||
// Does not read *(s1 + (s2_limit - s2)) or beyond.
|
||||
// Requires that s2_limit >= s2.
|
||||
//
|
||||
// Separate implementation for x86_64, for speed. Uses the fact that
|
||||
// x86_64 is little endian.
|
||||
#if defined(ARCH_K8)
|
||||
static inline int FindMatchLength(const char* s1,
|
||||
const char* s2,
|
||||
const char* s2_limit) {
|
||||
// Separate implementation for 64-bit, little-endian cpus.
|
||||
#if !defined(SNAPPY_IS_BIG_ENDIAN) && \
|
||||
(defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM))
|
||||
static inline std::pair<size_t, bool> FindMatchLength(const char* s1,
|
||||
const char* s2,
|
||||
const char* s2_limit) {
|
||||
assert(s2_limit >= s2);
|
||||
int matched = 0;
|
||||
size_t matched = 0;
|
||||
|
||||
// This block isn't necessary for correctness; we could just start looping
|
||||
// immediately. As an optimization though, it is useful. It creates some not
|
||||
// uncommon code paths that determine, without extra effort, whether the match
|
||||
// length is less than 8. In short, we are hoping to avoid a conditional
|
||||
// branch, and perhaps get better code layout from the C++ compiler.
|
||||
if (SNAPPY_PREDICT_TRUE(s2 <= s2_limit - 8)) {
|
||||
uint64 a1 = UNALIGNED_LOAD64(s1);
|
||||
uint64 a2 = UNALIGNED_LOAD64(s2);
|
||||
if (a1 != a2) {
|
||||
return std::pair<size_t, bool>(Bits::FindLSBSetNonZero64(a1 ^ a2) >> 3,
|
||||
true);
|
||||
} else {
|
||||
matched = 8;
|
||||
s2 += 8;
|
||||
}
|
||||
}
|
||||
|
||||
// Find out how long the match is. We loop over the data 64 bits at a
|
||||
// time until we find a 64-bit block that doesn't match; then we find
|
||||
// the first non-matching bit and use that to calculate the total
|
||||
// length of the match.
|
||||
while (PREDICT_TRUE(s2 <= s2_limit - 8)) {
|
||||
while (SNAPPY_PREDICT_TRUE(s2 <= s2_limit - 8)) {
|
||||
if (UNALIGNED_LOAD64(s2) == UNALIGNED_LOAD64(s1 + matched)) {
|
||||
s2 += 8;
|
||||
matched += 8;
|
||||
} else {
|
||||
// On current (mid-2008) Opteron models there is a 3% more
|
||||
// efficient code sequence to find the first non-matching byte.
|
||||
// However, what follows is ~10% better on Intel Core 2 and newer,
|
||||
// and we expect AMD's bsf instruction to improve.
|
||||
uint64 x = UNALIGNED_LOAD64(s2) ^ UNALIGNED_LOAD64(s1 + matched);
|
||||
int matching_bits = Bits::FindLSBSetNonZero64(x);
|
||||
matched += matching_bits >> 3;
|
||||
return matched;
|
||||
assert(matched >= 8);
|
||||
return std::pair<size_t, bool>(matched, false);
|
||||
}
|
||||
}
|
||||
while (PREDICT_TRUE(s2 < s2_limit)) {
|
||||
while (SNAPPY_PREDICT_TRUE(s2 < s2_limit)) {
|
||||
if (s1[matched] == *s2) {
|
||||
++s2;
|
||||
++matched;
|
||||
} else {
|
||||
return matched;
|
||||
return std::pair<size_t, bool>(matched, matched < 8);
|
||||
}
|
||||
}
|
||||
return matched;
|
||||
return std::pair<size_t, bool>(matched, matched < 8);
|
||||
}
|
||||
#else
|
||||
static inline int FindMatchLength(const char* s1,
|
||||
const char* s2,
|
||||
const char* s2_limit) {
|
||||
static inline std::pair<size_t, bool> FindMatchLength(const char* s1,
|
||||
const char* s2,
|
||||
const char* s2_limit) {
|
||||
// Implementation based on the x86-64 version, above.
|
||||
assert(s2_limit >= s2);
|
||||
int matched = 0;
|
||||
|
|
@ -140,7 +164,7 @@ static inline int FindMatchLength(const char* s1,
|
|||
++matched;
|
||||
}
|
||||
}
|
||||
return matched;
|
||||
return std::pair<size_t, bool>(matched, matched < 8);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -155,11 +179,6 @@ enum {
|
|||
};
|
||||
static const int kMaximumTagLength = 5; // COPY_4_BYTE_OFFSET plus the actual offset.
|
||||
|
||||
// Mapping from i in range [0,4] to a mask to extract the bottom 8*i bits
|
||||
static const uint32 wordmask[] = {
|
||||
0u, 0xffu, 0xffffu, 0xffffffu, 0xffffffffu
|
||||
};
|
||||
|
||||
// Data stored per entry in lookup table:
|
||||
// Range Bits-used Description
|
||||
// ------------------------------------
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@
|
|||
|
||||
namespace snappy {
|
||||
|
||||
void Varint::Append32(string* s, uint32 value) {
|
||||
void Varint::Append32(std::string* s, uint32 value) {
|
||||
char buf[Varint::kMax32];
|
||||
const char* p = Varint::Encode32(buf, value);
|
||||
s->append(buf, p - buf);
|
||||
|
|
|
|||
|
|
@ -45,6 +45,26 @@
|
|||
#include <sys/mman.h>
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_UNISTD_H
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#include <intrin.h>
|
||||
#endif // defined(_MSC_VER)
|
||||
|
||||
#ifndef __has_feature
|
||||
#define __has_feature(x) 0
|
||||
#endif
|
||||
|
||||
#if __has_feature(memory_sanitizer)
|
||||
#include <sanitizer/msan_interface.h>
|
||||
#define SNAPPY_ANNOTATE_MEMORY_IS_INITIALIZED(address, size) \
|
||||
__msan_unpoison((address), (size))
|
||||
#else
|
||||
#define SNAPPY_ANNOTATE_MEMORY_IS_INITIALIZED(address, size) /* empty */
|
||||
#endif // __has_feature(memory_sanitizer)
|
||||
|
||||
#include "snappy-stubs-public.h"
|
||||
|
||||
#if defined(__x86_64__)
|
||||
|
|
@ -52,6 +72,14 @@
|
|||
// Enable 64-bit optimized versions of some routines.
|
||||
#define ARCH_K8 1
|
||||
|
||||
#elif defined(__ppc64__)
|
||||
|
||||
#define ARCH_PPC 1
|
||||
|
||||
#elif defined(__aarch64__)
|
||||
|
||||
#define ARCH_ARM 1
|
||||
|
||||
#endif
|
||||
|
||||
// Needed by OS X, among others.
|
||||
|
|
@ -59,10 +87,6 @@
|
|||
#define MAP_ANONYMOUS MAP_ANON
|
||||
#endif
|
||||
|
||||
// Pull in std::min, std::ostream, and the likes. This is safe because this
|
||||
// header file is never used from any public header files.
|
||||
using namespace std;
|
||||
|
||||
// The size of an array, if known at compile-time.
|
||||
// Will give unexpected results if used on a pointer.
|
||||
// We undefine it first, since some compilers already have a definition.
|
||||
|
|
@ -73,11 +97,11 @@ using namespace std;
|
|||
|
||||
// Static prediction hints.
|
||||
#ifdef HAVE_BUILTIN_EXPECT
|
||||
#define PREDICT_FALSE(x) (__builtin_expect(x, 0))
|
||||
#define PREDICT_TRUE(x) (__builtin_expect(!!(x), 1))
|
||||
#define SNAPPY_PREDICT_FALSE(x) (__builtin_expect(x, 0))
|
||||
#define SNAPPY_PREDICT_TRUE(x) (__builtin_expect(!!(x), 1))
|
||||
#else
|
||||
#define PREDICT_FALSE(x) x
|
||||
#define PREDICT_TRUE(x) x
|
||||
#define SNAPPY_PREDICT_FALSE(x) x
|
||||
#define SNAPPY_PREDICT_TRUE(x) x
|
||||
#endif
|
||||
|
||||
// This is only used for recomputing the tag byte table used during
|
||||
|
|
@ -96,9 +120,10 @@ static const int64 kint64max = static_cast<int64>(0x7FFFFFFFFFFFFFFFLL);
|
|||
|
||||
// Potentially unaligned loads and stores.
|
||||
|
||||
// x86 and PowerPC can simply do these loads and stores native.
|
||||
// x86, PowerPC, and ARM64 can simply do these loads and stores native.
|
||||
|
||||
#if defined(__i386__) || defined(__x86_64__) || defined(__powerpc__)
|
||||
#if defined(__i386__) || defined(__x86_64__) || defined(__powerpc__) || \
|
||||
defined(__aarch64__)
|
||||
|
||||
#define UNALIGNED_LOAD16(_p) (*reinterpret_cast<const uint16 *>(_p))
|
||||
#define UNALIGNED_LOAD32(_p) (*reinterpret_cast<const uint32 *>(_p))
|
||||
|
|
@ -174,7 +199,7 @@ struct Unaligned32Struct {
|
|||
((reinterpret_cast< ::snappy::base::internal::Unaligned32Struct *>(_p))->value = \
|
||||
(_val))
|
||||
|
||||
// TODO(user): NEON supports unaligned 64-bit loads and stores.
|
||||
// TODO: NEON supports unaligned 64-bit loads and stores.
|
||||
// See if that would be more efficient on platforms supporting it,
|
||||
// at least for copies.
|
||||
|
||||
|
|
@ -225,22 +250,8 @@ inline void UNALIGNED_STORE64(void *p, uint64 v) {
|
|||
|
||||
#endif
|
||||
|
||||
// This can be more efficient than UNALIGNED_LOAD64 + UNALIGNED_STORE64
|
||||
// on some platforms, in particular ARM.
|
||||
inline void UnalignedCopy64(const void *src, void *dst) {
|
||||
if (sizeof(void *) == 8) {
|
||||
UNALIGNED_STORE64(dst, UNALIGNED_LOAD64(src));
|
||||
} else {
|
||||
const char *src_char = reinterpret_cast<const char *>(src);
|
||||
char *dst_char = reinterpret_cast<char *>(dst);
|
||||
|
||||
UNALIGNED_STORE32(dst_char, UNALIGNED_LOAD32(src_char));
|
||||
UNALIGNED_STORE32(dst_char + 4, UNALIGNED_LOAD32(src_char + 4));
|
||||
}
|
||||
}
|
||||
|
||||
// The following guarantees declaration of the byte swap functions.
|
||||
#ifdef WORDS_BIGENDIAN
|
||||
#if defined(SNAPPY_IS_BIG_ENDIAN)
|
||||
|
||||
#ifdef HAVE_SYS_BYTEORDER_H
|
||||
#include <sys/byteorder.h>
|
||||
|
|
@ -297,7 +308,7 @@ inline uint64 bswap_64(uint64 x) {
|
|||
|
||||
#endif
|
||||
|
||||
#endif // WORDS_BIGENDIAN
|
||||
#endif // defined(SNAPPY_IS_BIG_ENDIAN)
|
||||
|
||||
// Convert to little-endian storage, opposite of network format.
|
||||
// Convert x from host to little endian: x = LittleEndian.FromHost(x);
|
||||
|
|
@ -311,7 +322,7 @@ inline uint64 bswap_64(uint64 x) {
|
|||
class LittleEndian {
|
||||
public:
|
||||
// Conversion functions.
|
||||
#ifdef WORDS_BIGENDIAN
|
||||
#if defined(SNAPPY_IS_BIG_ENDIAN)
|
||||
|
||||
static uint16 FromHost16(uint16 x) { return bswap_16(x); }
|
||||
static uint16 ToHost16(uint16 x) { return bswap_16(x); }
|
||||
|
|
@ -321,7 +332,7 @@ class LittleEndian {
|
|||
|
||||
static bool IsLittleEndian() { return false; }
|
||||
|
||||
#else // !defined(WORDS_BIGENDIAN)
|
||||
#else // !defined(SNAPPY_IS_BIG_ENDIAN)
|
||||
|
||||
static uint16 FromHost16(uint16 x) { return x; }
|
||||
static uint16 ToHost16(uint16 x) { return x; }
|
||||
|
|
@ -331,7 +342,7 @@ class LittleEndian {
|
|||
|
||||
static bool IsLittleEndian() { return true; }
|
||||
|
||||
#endif // !defined(WORDS_BIGENDIAN)
|
||||
#endif // !defined(SNAPPY_IS_BIG_ENDIAN)
|
||||
|
||||
// Functions to do unaligned loads and stores in little-endian order.
|
||||
static uint16 Load16(const void *p) {
|
||||
|
|
@ -354,6 +365,9 @@ class LittleEndian {
|
|||
// Some bit-manipulation functions.
|
||||
class Bits {
|
||||
public:
|
||||
// Return floor(log2(n)) for positive integer n.
|
||||
static int Log2FloorNonZero(uint32 n);
|
||||
|
||||
// Return floor(log2(n)) for positive integer n. Returns -1 iff n == 0.
|
||||
static int Log2Floor(uint32 n);
|
||||
|
||||
|
|
@ -361,31 +375,85 @@ class Bits {
|
|||
// undefined value if n == 0. FindLSBSetNonZero() is similar to ffs() except
|
||||
// that it's 0-indexed.
|
||||
static int FindLSBSetNonZero(uint32 n);
|
||||
|
||||
#if defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
|
||||
static int FindLSBSetNonZero64(uint64 n);
|
||||
#endif // defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
|
||||
|
||||
private:
|
||||
DISALLOW_COPY_AND_ASSIGN(Bits);
|
||||
// No copying
|
||||
Bits(const Bits&);
|
||||
void operator=(const Bits&);
|
||||
};
|
||||
|
||||
#ifdef HAVE_BUILTIN_CTZ
|
||||
|
||||
inline int Bits::Log2FloorNonZero(uint32 n) {
|
||||
assert(n != 0);
|
||||
// (31 ^ x) is equivalent to (31 - x) for x in [0, 31]. An easy proof
|
||||
// represents subtraction in base 2 and observes that there's no carry.
|
||||
//
|
||||
// GCC and Clang represent __builtin_clz on x86 as 31 ^ _bit_scan_reverse(x).
|
||||
// Using "31 ^" here instead of "31 -" allows the optimizer to strip the
|
||||
// function body down to _bit_scan_reverse(x).
|
||||
return 31 ^ __builtin_clz(n);
|
||||
}
|
||||
|
||||
inline int Bits::Log2Floor(uint32 n) {
|
||||
return n == 0 ? -1 : 31 ^ __builtin_clz(n);
|
||||
return (n == 0) ? -1 : Bits::Log2FloorNonZero(n);
|
||||
}
|
||||
|
||||
inline int Bits::FindLSBSetNonZero(uint32 n) {
|
||||
assert(n != 0);
|
||||
return __builtin_ctz(n);
|
||||
}
|
||||
|
||||
#if defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
|
||||
inline int Bits::FindLSBSetNonZero64(uint64 n) {
|
||||
assert(n != 0);
|
||||
return __builtin_ctzll(n);
|
||||
}
|
||||
#endif // defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
|
||||
|
||||
#elif defined(_MSC_VER)
|
||||
|
||||
inline int Bits::Log2FloorNonZero(uint32 n) {
|
||||
assert(n != 0);
|
||||
unsigned long where;
|
||||
_BitScanReverse(&where, n);
|
||||
return static_cast<int>(where);
|
||||
}
|
||||
|
||||
inline int Bits::Log2Floor(uint32 n) {
|
||||
unsigned long where;
|
||||
if (_BitScanReverse(&where, n))
|
||||
return static_cast<int>(where);
|
||||
return -1;
|
||||
}
|
||||
|
||||
inline int Bits::FindLSBSetNonZero(uint32 n) {
|
||||
assert(n != 0);
|
||||
unsigned long where;
|
||||
if (_BitScanForward(&where, n))
|
||||
return static_cast<int>(where);
|
||||
return 32;
|
||||
}
|
||||
|
||||
#if defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
|
||||
inline int Bits::FindLSBSetNonZero64(uint64 n) {
|
||||
assert(n != 0);
|
||||
unsigned long where;
|
||||
if (_BitScanForward64(&where, n))
|
||||
return static_cast<int>(where);
|
||||
return 64;
|
||||
}
|
||||
#endif // defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
|
||||
|
||||
#else // Portable versions.
|
||||
|
||||
inline int Bits::Log2Floor(uint32 n) {
|
||||
if (n == 0)
|
||||
return -1;
|
||||
inline int Bits::Log2FloorNonZero(uint32 n) {
|
||||
assert(n != 0);
|
||||
|
||||
int log = 0;
|
||||
uint32 value = n;
|
||||
for (int i = 4; i >= 0; --i) {
|
||||
|
|
@ -400,7 +468,13 @@ inline int Bits::Log2Floor(uint32 n) {
|
|||
return log;
|
||||
}
|
||||
|
||||
inline int Bits::Log2Floor(uint32 n) {
|
||||
return (n == 0) ? -1 : Bits::Log2FloorNonZero(n);
|
||||
}
|
||||
|
||||
inline int Bits::FindLSBSetNonZero(uint32 n) {
|
||||
assert(n != 0);
|
||||
|
||||
int rc = 31;
|
||||
for (int i = 4, shift = 1 << 4; i >= 0; --i) {
|
||||
const uint32 x = n << shift;
|
||||
|
|
@ -413,8 +487,11 @@ inline int Bits::FindLSBSetNonZero(uint32 n) {
|
|||
return rc;
|
||||
}
|
||||
|
||||
#if defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
|
||||
// FindLSBSetNonZero64() is defined in terms of FindLSBSetNonZero().
|
||||
inline int Bits::FindLSBSetNonZero64(uint64 n) {
|
||||
assert(n != 0);
|
||||
|
||||
const uint32 bottombits = static_cast<uint32>(n);
|
||||
if (bottombits == 0) {
|
||||
// Bottom bits are zero, so scan in top bits
|
||||
|
|
@ -423,6 +500,7 @@ inline int Bits::FindLSBSetNonZero64(uint64 n) {
|
|||
return FindLSBSetNonZero(bottombits);
|
||||
}
|
||||
}
|
||||
#endif // defined(ARCH_K8) || defined(ARCH_PPC) || defined(ARCH_ARM)
|
||||
|
||||
#endif // End portable versions.
|
||||
|
||||
|
|
@ -446,7 +524,7 @@ class Varint {
|
|||
static char* Encode32(char* ptr, uint32 v);
|
||||
|
||||
// EFFECTS Appends the varint representation of "value" to "*s".
|
||||
static void Append32(string* s, uint32 value);
|
||||
static void Append32(std::string* s, uint32 value);
|
||||
};
|
||||
|
||||
inline const char* Varint::Parse32WithLimit(const char* p,
|
||||
|
|
@ -503,7 +581,7 @@ inline char* Varint::Encode32(char* sptr, uint32 v) {
|
|||
// replace this function with one that resizes the string without
|
||||
// filling the new space with zeros (if applicable) --
|
||||
// it will be non-portable but faster.
|
||||
inline void STLStringResizeUninitialized(string* s, size_t new_size) {
|
||||
inline void STLStringResizeUninitialized(std::string* s, size_t new_size) {
|
||||
s->resize(new_size);
|
||||
}
|
||||
|
||||
|
|
@ -519,7 +597,7 @@ inline void STLStringResizeUninitialized(string* s, size_t new_size) {
|
|||
// (http://www.open-std.org/JTC1/SC22/WG21/docs/lwg-defects.html#530)
|
||||
// proposes this as the method. It will officially be part of the standard
|
||||
// for C++0x. This should already work on all current implementations.
|
||||
inline char* string_as_array(string* str) {
|
||||
inline char* string_as_array(std::string* str) {
|
||||
return str->empty() ? NULL : &*str->begin();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
// Copyright 2011 Google Inc. All Rights Reserved.
|
||||
// Author: sesse@google.com (Steinar H. Gunderson)
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are
|
||||
|
|
@ -36,64 +35,39 @@
|
|||
#ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
|
||||
#define THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
|
||||
|
||||
#if @ac_cv_have_stdint_h@
|
||||
#include <stdint.h>
|
||||
#endif
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
#if @ac_cv_have_stddef_h@
|
||||
#include <stddef.h>
|
||||
#endif
|
||||
|
||||
#if @ac_cv_have_sys_uio_h@
|
||||
#if ${HAVE_SYS_UIO_H_01} // HAVE_SYS_UIO_H
|
||||
#include <sys/uio.h>
|
||||
#endif
|
||||
#endif // HAVE_SYS_UIO_H
|
||||
|
||||
#define SNAPPY_MAJOR @SNAPPY_MAJOR@
|
||||
#define SNAPPY_MINOR @SNAPPY_MINOR@
|
||||
#define SNAPPY_PATCHLEVEL @SNAPPY_PATCHLEVEL@
|
||||
#define SNAPPY_MAJOR ${PROJECT_VERSION_MAJOR}
|
||||
#define SNAPPY_MINOR ${PROJECT_VERSION_MINOR}
|
||||
#define SNAPPY_PATCHLEVEL ${PROJECT_VERSION_PATCH}
|
||||
#define SNAPPY_VERSION \
|
||||
((SNAPPY_MAJOR << 16) | (SNAPPY_MINOR << 8) | SNAPPY_PATCHLEVEL)
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace snappy {
|
||||
|
||||
#if @ac_cv_have_stdint_h@
|
||||
typedef int8_t int8;
|
||||
typedef uint8_t uint8;
|
||||
typedef int16_t int16;
|
||||
typedef uint16_t uint16;
|
||||
typedef int32_t int32;
|
||||
typedef uint32_t uint32;
|
||||
typedef int64_t int64;
|
||||
typedef uint64_t uint64;
|
||||
#else
|
||||
typedef signed char int8;
|
||||
typedef unsigned char uint8;
|
||||
typedef short int16;
|
||||
typedef unsigned short uint16;
|
||||
typedef int int32;
|
||||
typedef unsigned int uint32;
|
||||
typedef long long int64;
|
||||
typedef unsigned long long uint64;
|
||||
#endif
|
||||
using int8 = std::int8_t;
|
||||
using uint8 = std::uint8_t;
|
||||
using int16 = std::int16_t;
|
||||
using uint16 = std::uint16_t;
|
||||
using int32 = std::int32_t;
|
||||
using uint32 = std::uint32_t;
|
||||
using int64 = std::int64_t;
|
||||
using uint64 = std::uint64_t;
|
||||
|
||||
typedef std::string string;
|
||||
|
||||
#ifndef DISALLOW_COPY_AND_ASSIGN
|
||||
#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
|
||||
TypeName(const TypeName&); \
|
||||
void operator=(const TypeName&)
|
||||
#endif
|
||||
|
||||
#if !@ac_cv_have_sys_uio_h@
|
||||
#if !${HAVE_SYS_UIO_H_01} // !HAVE_SYS_UIO_H
|
||||
// Windows does not have an iovec type, yet the concept is universally useful.
|
||||
// It is simple to define it ourselves, so we put it inside our own namespace.
|
||||
struct iovec {
|
||||
void* iov_base;
|
||||
size_t iov_len;
|
||||
void* iov_base;
|
||||
size_t iov_len;
|
||||
};
|
||||
#endif
|
||||
#endif // !HAVE_SYS_UIO_H
|
||||
|
||||
} // namespace snappy
|
||||
|
||||
|
|
|
|||
|
|
@ -33,6 +33,9 @@
|
|||
#endif
|
||||
|
||||
#ifdef HAVE_WINDOWS_H
|
||||
// Needed to be able to use std::max without workarounds in the source code.
|
||||
// https://support.microsoft.com/en-us/help/143208/prb-using-stl-in-windows-program-can-cause-min-max-conflicts
|
||||
#define NOMINMAX
|
||||
#include <windows.h>
|
||||
#endif
|
||||
|
||||
|
|
@ -45,12 +48,12 @@ DEFINE_bool(run_microbenchmarks, true,
|
|||
|
||||
namespace snappy {
|
||||
|
||||
string ReadTestDataFile(const string& base, size_t size_limit) {
|
||||
string contents;
|
||||
std::string ReadTestDataFile(const std::string& base, size_t size_limit) {
|
||||
std::string contents;
|
||||
const char* srcdir = getenv("srcdir"); // This is set by Automake.
|
||||
string prefix;
|
||||
std::string prefix;
|
||||
if (srcdir) {
|
||||
prefix = string(srcdir) + "/";
|
||||
prefix = std::string(srcdir) + "/";
|
||||
}
|
||||
file::GetContents(prefix + "testdata/" + base, &contents, file::Defaults()
|
||||
).CheckSuccess();
|
||||
|
|
@ -60,11 +63,11 @@ string ReadTestDataFile(const string& base, size_t size_limit) {
|
|||
return contents;
|
||||
}
|
||||
|
||||
string ReadTestDataFile(const string& base) {
|
||||
std::string ReadTestDataFile(const std::string& base) {
|
||||
return ReadTestDataFile(base, 0);
|
||||
}
|
||||
|
||||
string StringPrintf(const char* format, ...) {
|
||||
std::string StrFormat(const char* format, ...) {
|
||||
char buf[4096];
|
||||
va_list ap;
|
||||
va_start(ap, format);
|
||||
|
|
@ -76,7 +79,7 @@ string StringPrintf(const char* format, ...) {
|
|||
bool benchmark_running = false;
|
||||
int64 benchmark_real_time_us = 0;
|
||||
int64 benchmark_cpu_time_us = 0;
|
||||
string *benchmark_label = NULL;
|
||||
std::string* benchmark_label = nullptr;
|
||||
int64 benchmark_bytes_processed = 0;
|
||||
|
||||
void ResetBenchmarkTiming() {
|
||||
|
|
@ -160,11 +163,11 @@ void StopBenchmarkTiming() {
|
|||
benchmark_running = false;
|
||||
}
|
||||
|
||||
void SetBenchmarkLabel(const string& str) {
|
||||
void SetBenchmarkLabel(const std::string& str) {
|
||||
if (benchmark_label) {
|
||||
delete benchmark_label;
|
||||
}
|
||||
benchmark_label = new string(str);
|
||||
benchmark_label = new std::string(str);
|
||||
}
|
||||
|
||||
void SetBenchmarkBytesProcessed(int64 bytes) {
|
||||
|
|
@ -201,7 +204,7 @@ void Benchmark::Run() {
|
|||
if (benchmark_real_time_us > 0) {
|
||||
num_iterations = 200000 * kCalibrateIterations / benchmark_real_time_us;
|
||||
}
|
||||
num_iterations = max(num_iterations, kCalibrateIterations);
|
||||
num_iterations = std::max(num_iterations, kCalibrateIterations);
|
||||
BenchmarkRun benchmark_runs[kNumRuns];
|
||||
|
||||
for (int run = 0; run < kNumRuns; ++run) {
|
||||
|
|
@ -214,13 +217,13 @@ void Benchmark::Run() {
|
|||
benchmark_runs[run].cpu_time_us = benchmark_cpu_time_us;
|
||||
}
|
||||
|
||||
string heading = StringPrintf("%s/%d", name_.c_str(), test_case_num);
|
||||
string human_readable_speed;
|
||||
std::string heading = StrFormat("%s/%d", name_.c_str(), test_case_num);
|
||||
std::string human_readable_speed;
|
||||
|
||||
nth_element(benchmark_runs,
|
||||
benchmark_runs + kMedianPos,
|
||||
benchmark_runs + kNumRuns,
|
||||
BenchmarkCompareCPUTime());
|
||||
std::nth_element(benchmark_runs,
|
||||
benchmark_runs + kMedianPos,
|
||||
benchmark_runs + kNumRuns,
|
||||
BenchmarkCompareCPUTime());
|
||||
int64 real_time_us = benchmark_runs[kMedianPos].real_time_us;
|
||||
int64 cpu_time_us = benchmark_runs[kMedianPos].cpu_time_us;
|
||||
if (cpu_time_us <= 0) {
|
||||
|
|
@ -229,15 +232,16 @@ void Benchmark::Run() {
|
|||
int64 bytes_per_second =
|
||||
benchmark_bytes_processed * 1000000 / cpu_time_us;
|
||||
if (bytes_per_second < 1024) {
|
||||
human_readable_speed = StringPrintf("%dB/s", bytes_per_second);
|
||||
human_readable_speed =
|
||||
StrFormat("%dB/s", static_cast<int>(bytes_per_second));
|
||||
} else if (bytes_per_second < 1024 * 1024) {
|
||||
human_readable_speed = StringPrintf(
|
||||
human_readable_speed = StrFormat(
|
||||
"%.1fkB/s", bytes_per_second / 1024.0f);
|
||||
} else if (bytes_per_second < 1024 * 1024 * 1024) {
|
||||
human_readable_speed = StringPrintf(
|
||||
human_readable_speed = StrFormat(
|
||||
"%.1fMB/s", bytes_per_second / (1024.0f * 1024.0f));
|
||||
} else {
|
||||
human_readable_speed = StringPrintf(
|
||||
human_readable_speed = StrFormat(
|
||||
"%.1fGB/s", bytes_per_second / (1024.0f * 1024.0f * 1024.0f));
|
||||
}
|
||||
}
|
||||
|
|
@ -523,8 +527,8 @@ int ZLib::UncompressAtMostOrAll(Bytef *dest, uLongf *destLen,
|
|||
LOG(WARNING)
|
||||
<< "UncompressChunkOrAll: Received some extra data, bytes total: "
|
||||
<< uncomp_stream_.avail_in << " bytes: "
|
||||
<< string(reinterpret_cast<const char *>(uncomp_stream_.next_in),
|
||||
min(int(uncomp_stream_.avail_in), 20));
|
||||
<< std::string(reinterpret_cast<const char *>(uncomp_stream_.next_in),
|
||||
std::min(int(uncomp_stream_.avail_in), 20));
|
||||
UncompressErrorInit();
|
||||
return Z_DATA_ERROR; // what's the extra data for?
|
||||
} else if (err != Z_OK && err != Z_STREAM_END && err != Z_BUF_ERROR) {
|
||||
|
|
|
|||
|
|
@ -55,8 +55,6 @@
|
|||
#include <windows.h>
|
||||
#endif
|
||||
|
||||
#include <string>
|
||||
|
||||
#ifdef HAVE_GTEST
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
|
@ -110,26 +108,8 @@
|
|||
#include "lzo/lzo1x.h"
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_LIBLZF
|
||||
extern "C" {
|
||||
#include "lzf.h"
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_LIBFASTLZ
|
||||
#include "fastlz.h"
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_LIBQUICKLZ
|
||||
#include "quicklz.h"
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
|
||||
namespace File {
|
||||
void Init() { }
|
||||
} // namespace File
|
||||
|
||||
namespace file {
|
||||
int Defaults() { return 0; }
|
||||
|
||||
|
|
@ -138,7 +118,8 @@ namespace file {
|
|||
void CheckSuccess() { }
|
||||
};
|
||||
|
||||
DummyStatus GetContents(const string& filename, string* data, int unused) {
|
||||
DummyStatus GetContents(
|
||||
const std::string& filename, std::string* data, int unused) {
|
||||
FILE* fp = fopen(filename.c_str(), "rb");
|
||||
if (fp == NULL) {
|
||||
perror(filename.c_str());
|
||||
|
|
@ -153,7 +134,7 @@ namespace file {
|
|||
perror("fread");
|
||||
exit(1);
|
||||
}
|
||||
data->append(string(buf, ret));
|
||||
data->append(std::string(buf, ret));
|
||||
}
|
||||
|
||||
fclose(fp);
|
||||
|
|
@ -161,9 +142,8 @@ namespace file {
|
|||
return DummyStatus();
|
||||
}
|
||||
|
||||
DummyStatus SetContents(const string& filename,
|
||||
const string& str,
|
||||
int unused) {
|
||||
inline DummyStatus SetContents(
|
||||
const std::string& filename, const std::string& str, int unused) {
|
||||
FILE* fp = fopen(filename.c_str(), "wb");
|
||||
if (fp == NULL) {
|
||||
perror(filename.c_str());
|
||||
|
|
@ -187,7 +167,7 @@ namespace file {
|
|||
namespace snappy {
|
||||
|
||||
#define FLAGS_test_random_seed 301
|
||||
typedef string TypeParam;
|
||||
using TypeParam = std::string;
|
||||
|
||||
void Test_CorruptedTest_VerifyCorrupted();
|
||||
void Test_Snappy_SimpleTests();
|
||||
|
|
@ -201,63 +181,13 @@ void Test_Snappy_ReadPastEndOfBuffer();
|
|||
void Test_Snappy_FindMatchLength();
|
||||
void Test_Snappy_FindMatchLengthRandom();
|
||||
|
||||
string ReadTestDataFile(const string& base, size_t size_limit);
|
||||
std::string ReadTestDataFile(const std::string& base, size_t size_limit);
|
||||
|
||||
string ReadTestDataFile(const string& base);
|
||||
std::string ReadTestDataFile(const std::string& base);
|
||||
|
||||
// A sprintf() variant that returns a std::string.
|
||||
// Not safe for general use due to truncation issues.
|
||||
string StringPrintf(const char* format, ...);
|
||||
|
||||
// A simple, non-cryptographically-secure random generator.
|
||||
class ACMRandom {
|
||||
public:
|
||||
explicit ACMRandom(uint32 seed) : seed_(seed) {}
|
||||
|
||||
int32 Next();
|
||||
|
||||
int32 Uniform(int32 n) {
|
||||
return Next() % n;
|
||||
}
|
||||
uint8 Rand8() {
|
||||
return static_cast<uint8>((Next() >> 1) & 0x000000ff);
|
||||
}
|
||||
bool OneIn(int X) { return Uniform(X) == 0; }
|
||||
|
||||
// Skewed: pick "base" uniformly from range [0,max_log] and then
|
||||
// return "base" random bits. The effect is to pick a number in the
|
||||
// range [0,2^max_log-1] with bias towards smaller numbers.
|
||||
int32 Skewed(int max_log);
|
||||
|
||||
private:
|
||||
static const uint32 M = 2147483647L; // 2^31-1
|
||||
uint32 seed_;
|
||||
};
|
||||
|
||||
inline int32 ACMRandom::Next() {
|
||||
static const uint64 A = 16807; // bits 14, 8, 7, 5, 2, 1, 0
|
||||
// We are computing
|
||||
// seed_ = (seed_ * A) % M, where M = 2^31-1
|
||||
//
|
||||
// seed_ must not be zero or M, or else all subsequent computed values
|
||||
// will be zero or M respectively. For all other values, seed_ will end
|
||||
// up cycling through every number in [1,M-1]
|
||||
uint64 product = seed_ * A;
|
||||
|
||||
// Compute (product % M) using the fact that ((x << 31) % M) == x.
|
||||
seed_ = (product >> 31) + (product & M);
|
||||
// The first reduction may overflow by 1 bit, so we may need to repeat.
|
||||
// mod == M is not possible; using > allows the faster sign-bit-based test.
|
||||
if (seed_ > M) {
|
||||
seed_ -= M;
|
||||
}
|
||||
return seed_;
|
||||
}
|
||||
|
||||
inline int32 ACMRandom::Skewed(int max_log) {
|
||||
const int32 base = (Next() - 1) % (max_log+1);
|
||||
return (Next() - 1) & ((1u << base)-1);
|
||||
}
|
||||
std::string StrFormat(const char* format, ...);
|
||||
|
||||
// A wall-time clock. This stub is not super-accurate, nor resistant to the
|
||||
// system time changing.
|
||||
|
|
@ -311,8 +241,8 @@ typedef void (*BenchmarkFunction)(int, int);
|
|||
|
||||
class Benchmark {
|
||||
public:
|
||||
Benchmark(const string& name, BenchmarkFunction function) :
|
||||
name_(name), function_(function) {}
|
||||
Benchmark(const std::string& name, BenchmarkFunction function)
|
||||
: name_(name), function_(function) {}
|
||||
|
||||
Benchmark* DenseRange(int start, int stop) {
|
||||
start_ = start;
|
||||
|
|
@ -323,7 +253,7 @@ class Benchmark {
|
|||
void Run();
|
||||
|
||||
private:
|
||||
const string name_;
|
||||
const std::string name_;
|
||||
const BenchmarkFunction function_;
|
||||
int start_, stop_;
|
||||
};
|
||||
|
|
@ -335,11 +265,13 @@ extern Benchmark* Benchmark_BM_UFlat;
|
|||
extern Benchmark* Benchmark_BM_UIOVec;
|
||||
extern Benchmark* Benchmark_BM_UValidate;
|
||||
extern Benchmark* Benchmark_BM_ZFlat;
|
||||
extern Benchmark* Benchmark_BM_ZFlatAll;
|
||||
extern Benchmark* Benchmark_BM_ZFlatIncreasingTableSize;
|
||||
|
||||
void ResetBenchmarkTiming();
|
||||
void StartBenchmarkTiming();
|
||||
void StopBenchmarkTiming();
|
||||
void SetBenchmarkLabel(const string& str);
|
||||
void SetBenchmarkLabel(const std::string& str);
|
||||
void SetBenchmarkBytesProcessed(int64 bytes);
|
||||
|
||||
#ifdef HAVE_LIBZ
|
||||
|
|
@ -467,7 +399,7 @@ class ZLib {
|
|||
|
||||
DECLARE_bool(run_microbenchmarks);
|
||||
|
||||
static void RunSpecifiedBenchmarks() {
|
||||
static inline void RunSpecifiedBenchmarks() {
|
||||
if (!FLAGS_run_microbenchmarks) {
|
||||
return;
|
||||
}
|
||||
|
|
@ -486,6 +418,8 @@ static void RunSpecifiedBenchmarks() {
|
|||
snappy::Benchmark_BM_UIOVec->Run();
|
||||
snappy::Benchmark_BM_UValidate->Run();
|
||||
snappy::Benchmark_BM_ZFlat->Run();
|
||||
snappy::Benchmark_BM_ZFlatAll->Run();
|
||||
snappy::Benchmark_BM_ZFlatIncreasingTableSize->Run();
|
||||
|
||||
fprintf(stderr, "\n");
|
||||
}
|
||||
|
|
@ -515,10 +449,6 @@ static inline int RUN_ALL_TESTS() {
|
|||
// For main().
|
||||
namespace snappy {
|
||||
|
||||
static void CompressFile(const char* fname);
|
||||
static void UncompressFile(const char* fname);
|
||||
static void MeasureFile(const char* fname);
|
||||
|
||||
// Logging.
|
||||
|
||||
#define LOG(level) LogMessage()
|
||||
|
|
@ -529,15 +459,15 @@ class LogMessage {
|
|||
public:
|
||||
LogMessage() { }
|
||||
~LogMessage() {
|
||||
cerr << endl;
|
||||
std::cerr << std::endl;
|
||||
}
|
||||
|
||||
LogMessage& operator<<(const std::string& msg) {
|
||||
cerr << msg;
|
||||
std::cerr << msg;
|
||||
return *this;
|
||||
}
|
||||
LogMessage& operator<<(int x) {
|
||||
cerr << x;
|
||||
std::cerr << x;
|
||||
return *this;
|
||||
}
|
||||
};
|
||||
|
|
@ -546,7 +476,7 @@ class LogMessage {
|
|||
// and ones that are always active.
|
||||
|
||||
#define CRASH_UNLESS(condition) \
|
||||
PREDICT_TRUE(condition) ? (void)0 : \
|
||||
SNAPPY_PREDICT_TRUE(condition) ? (void)0 : \
|
||||
snappy::LogMessageVoidify() & snappy::LogMessageCrash()
|
||||
|
||||
#ifdef _MSC_VER
|
||||
|
|
@ -560,7 +490,7 @@ class LogMessageCrash : public LogMessage {
|
|||
public:
|
||||
LogMessageCrash() { }
|
||||
~LogMessageCrash() {
|
||||
cerr << endl;
|
||||
std::cerr << std::endl;
|
||||
abort();
|
||||
}
|
||||
};
|
||||
|
|
@ -590,10 +520,6 @@ class LogMessageVoidify {
|
|||
#define CHECK_GT(a, b) CRASH_UNLESS((a) > (b))
|
||||
#define CHECK_OK(cond) (cond).CheckSuccess()
|
||||
|
||||
} // namespace
|
||||
|
||||
using snappy::CompressFile;
|
||||
using snappy::UncompressFile;
|
||||
using snappy::MeasureFile;
|
||||
} // namespace snappy
|
||||
|
||||
#endif // THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_TEST_H_
|
||||
|
|
|
|||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
|
@ -39,7 +39,7 @@
|
|||
#ifndef THIRD_PARTY_SNAPPY_SNAPPY_H__
|
||||
#define THIRD_PARTY_SNAPPY_SNAPPY_H__
|
||||
|
||||
#include <stddef.h>
|
||||
#include <cstddef>
|
||||
#include <string>
|
||||
|
||||
#include "snappy-stubs-public.h"
|
||||
|
|
@ -69,11 +69,12 @@ namespace snappy {
|
|||
// Higher-level string based routines (should be sufficient for most users)
|
||||
// ------------------------------------------------------------------------
|
||||
|
||||
// Sets "*output" to the compressed version of "input[0,input_length-1]".
|
||||
// Original contents of *output are lost.
|
||||
// Sets "*compressed" to the compressed version of "input[0,input_length-1]".
|
||||
// Original contents of *compressed are lost.
|
||||
//
|
||||
// REQUIRES: "input[]" is not an alias of "*output".
|
||||
size_t Compress(const char* input, size_t input_length, string* output);
|
||||
// REQUIRES: "input[]" is not an alias of "*compressed".
|
||||
size_t Compress(const char* input, size_t input_length,
|
||||
std::string* compressed);
|
||||
|
||||
// Decompresses "compressed[0,compressed_length-1]" to "*uncompressed".
|
||||
// Original contents of "*uncompressed" are lost.
|
||||
|
|
@ -82,7 +83,7 @@ namespace snappy {
|
|||
//
|
||||
// returns false if the message is corrupted and could not be decompressed
|
||||
bool Uncompress(const char* compressed, size_t compressed_length,
|
||||
string* uncompressed);
|
||||
std::string* uncompressed);
|
||||
|
||||
// Decompresses "compressed" to "*uncompressed".
|
||||
//
|
||||
|
|
@ -193,11 +194,14 @@ namespace snappy {
|
|||
// Note that there might be older data around that is compressed with larger
|
||||
// block sizes, so the decompression code should not rely on the
|
||||
// non-existence of long backreferences.
|
||||
static const int kBlockLog = 16;
|
||||
static const size_t kBlockSize = 1 << kBlockLog;
|
||||
static constexpr int kBlockLog = 16;
|
||||
static constexpr size_t kBlockSize = 1 << kBlockLog;
|
||||
|
||||
static const int kMaxHashTableBits = 14;
|
||||
static const size_t kMaxHashTableSize = 1 << kMaxHashTableBits;
|
||||
static constexpr int kMinHashTableBits = 8;
|
||||
static constexpr size_t kMinHashTableSize = 1 << kMinHashTableBits;
|
||||
|
||||
static constexpr int kMaxHashTableBits = 14;
|
||||
static constexpr size_t kMaxHashTableSize = 1 << kMaxHashTableBits;
|
||||
} // end namespace snappy
|
||||
|
||||
#endif // THIRD_PARTY_SNAPPY_SNAPPY_H__
|
||||
|
|
|
|||
|
|
@ -0,0 +1,59 @@
|
|||
// Copyright 2019 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are
|
||||
// met:
|
||||
//
|
||||
// * Redistributions of source code must retain the above copyright
|
||||
// notice, this list of conditions and the following disclaimer.
|
||||
// * Redistributions in binary form must reproduce the above
|
||||
// copyright notice, this list of conditions and the following disclaimer
|
||||
// in the documentation and/or other materials provided with the
|
||||
// distribution.
|
||||
// * Neither the name of Google Inc. nor the names of its
|
||||
// contributors may be used to endorse or promote products derived from
|
||||
// this software without specific prior written permission.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
//
|
||||
// libFuzzer harness for fuzzing snappy compression code.
|
||||
|
||||
#include <cassert>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
#include "snappy.h"
|
||||
|
||||
// Entry point for LibFuzzer.
|
||||
extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
|
||||
std::string input(reinterpret_cast<const char*>(data), size);
|
||||
|
||||
std::string compressed;
|
||||
size_t compressed_size =
|
||||
snappy::Compress(input.data(), input.size(), &compressed);
|
||||
|
||||
(void)compressed_size; // Variable only used in debug builds.
|
||||
assert(compressed_size == compressed.size());
|
||||
assert(compressed.size() <= snappy::MaxCompressedLength(input.size()));
|
||||
assert(snappy::IsValidCompressedBuffer(compressed.data(), compressed.size()));
|
||||
|
||||
std::string uncompressed_after_compress;
|
||||
bool uncompress_succeeded = snappy::Uncompress(
|
||||
compressed.data(), compressed.size(), &uncompressed_after_compress);
|
||||
|
||||
(void)uncompress_succeeded; // Variable only used in debug builds.
|
||||
assert(uncompress_succeeded);
|
||||
assert(input == uncompressed_after_compress);
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
// Copyright 2019 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are
|
||||
// met:
|
||||
//
|
||||
// * Redistributions of source code must retain the above copyright
|
||||
// notice, this list of conditions and the following disclaimer.
|
||||
// * Redistributions in binary form must reproduce the above
|
||||
// copyright notice, this list of conditions and the following disclaimer
|
||||
// in the documentation and/or other materials provided with the
|
||||
// distribution.
|
||||
// * Neither the name of Google Inc. nor the names of its
|
||||
// contributors may be used to endorse or promote products derived from
|
||||
// this software without specific prior written permission.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
//
|
||||
// libFuzzer harness for fuzzing snappy's decompression code.
|
||||
|
||||
#include <cassert>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
#include "snappy.h"
|
||||
|
||||
// Entry point for LibFuzzer.
|
||||
extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
|
||||
std::string input(reinterpret_cast<const char*>(data), size);
|
||||
|
||||
// Avoid self-crafted decompression bombs.
|
||||
size_t uncompressed_size;
|
||||
constexpr size_t kMaxUncompressedSize = 1 << 20;
|
||||
bool get_uncompressed_length_succeeded = snappy::GetUncompressedLength(
|
||||
input.data(), input.size(), &uncompressed_size);
|
||||
if (!get_uncompressed_length_succeeded ||
|
||||
(uncompressed_size > kMaxUncompressedSize)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::string uncompressed;
|
||||
// The return value of snappy::Uncompress() is ignored because decompression
|
||||
// will fail on invalid inputs.
|
||||
snappy::Uncompress(input.data(), input.size(), &uncompressed);
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -29,9 +29,10 @@
|
|||
#include <math.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
|
||||
#include <algorithm>
|
||||
#include <random>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "snappy.h"
|
||||
|
|
@ -50,13 +51,6 @@ DEFINE_bool(zlib, false,
|
|||
"Run zlib compression (http://www.zlib.net)");
|
||||
DEFINE_bool(lzo, false,
|
||||
"Run LZO compression (http://www.oberhumer.com/opensource/lzo/)");
|
||||
DEFINE_bool(quicklz, false,
|
||||
"Run quickLZ compression (http://www.quicklz.com/)");
|
||||
DEFINE_bool(liblzf, false,
|
||||
"Run libLZF compression "
|
||||
"(http://www.goof.com/pcg/marc/liblzf.html)");
|
||||
DEFINE_bool(fastlz, false,
|
||||
"Run FastLZ compression (http://www.fastlz.org/");
|
||||
DEFINE_bool(snappy, true, "Run snappy compression");
|
||||
|
||||
DEFINE_bool(write_compressed, false,
|
||||
|
|
@ -69,8 +63,7 @@ DEFINE_bool(snappy_dump_decompression_table, false,
|
|||
|
||||
namespace snappy {
|
||||
|
||||
|
||||
#ifdef HAVE_FUNC_MMAP
|
||||
#if defined(HAVE_FUNC_MMAP) && defined(HAVE_FUNC_SYSCONF)
|
||||
|
||||
// To test against code that reads beyond its input, this class copies a
|
||||
// string to a newly allocated group of pages, the last of which
|
||||
|
|
@ -80,8 +73,8 @@ namespace snappy {
|
|||
// be able to read previously allocated memory while doing heap allocations.
|
||||
class DataEndingAtUnreadablePage {
|
||||
public:
|
||||
explicit DataEndingAtUnreadablePage(const string& s) {
|
||||
const size_t page_size = getpagesize();
|
||||
explicit DataEndingAtUnreadablePage(const std::string& s) {
|
||||
const size_t page_size = sysconf(_SC_PAGESIZE);
|
||||
const size_t size = s.size();
|
||||
// Round up space for string to a multiple of page_size.
|
||||
size_t space_for_string = (size + page_size - 1) & ~(page_size - 1);
|
||||
|
|
@ -99,8 +92,9 @@ class DataEndingAtUnreadablePage {
|
|||
}
|
||||
|
||||
~DataEndingAtUnreadablePage() {
|
||||
const size_t page_size = sysconf(_SC_PAGESIZE);
|
||||
// Undo the mprotect.
|
||||
CHECK_EQ(0, mprotect(protected_page_, getpagesize(), PROT_READ|PROT_WRITE));
|
||||
CHECK_EQ(0, mprotect(protected_page_, page_size, PROT_READ|PROT_WRITE));
|
||||
CHECK_EQ(0, munmap(mem_, alloc_size_));
|
||||
}
|
||||
|
||||
|
|
@ -115,19 +109,19 @@ class DataEndingAtUnreadablePage {
|
|||
size_t size_;
|
||||
};
|
||||
|
||||
#else // HAVE_FUNC_MMAP
|
||||
#else // defined(HAVE_FUNC_MMAP) && defined(HAVE_FUNC_SYSCONF)
|
||||
|
||||
// Fallback for systems without mmap.
|
||||
typedef string DataEndingAtUnreadablePage;
|
||||
using DataEndingAtUnreadablePage = std::string;
|
||||
|
||||
#endif
|
||||
|
||||
enum CompressorType {
|
||||
ZLIB, LZO, LIBLZF, QUICKLZ, FASTLZ, SNAPPY
|
||||
ZLIB, LZO, SNAPPY
|
||||
};
|
||||
|
||||
const char* names[] = {
|
||||
"ZLIB", "LZO", "LIBLZF", "QUICKLZ", "FASTLZ", "SNAPPY"
|
||||
"ZLIB", "LZO", "SNAPPY"
|
||||
};
|
||||
|
||||
static size_t MinimumRequiredOutputSpace(size_t input_size,
|
||||
|
|
@ -143,21 +137,6 @@ static size_t MinimumRequiredOutputSpace(size_t input_size,
|
|||
return input_size + input_size/64 + 16 + 3;
|
||||
#endif // LZO_VERSION
|
||||
|
||||
#ifdef LZF_VERSION
|
||||
case LIBLZF:
|
||||
return input_size;
|
||||
#endif // LZF_VERSION
|
||||
|
||||
#ifdef QLZ_VERSION_MAJOR
|
||||
case QUICKLZ:
|
||||
return input_size + 36000; // 36000 is used for scratch.
|
||||
#endif // QLZ_VERSION_MAJOR
|
||||
|
||||
#ifdef FASTLZ_VERSION
|
||||
case FASTLZ:
|
||||
return max(static_cast<int>(ceil(input_size * 1.05)), 66);
|
||||
#endif // FASTLZ_VERSION
|
||||
|
||||
case SNAPPY:
|
||||
return snappy::MaxCompressedLength(input_size);
|
||||
|
||||
|
|
@ -175,7 +154,7 @@ static size_t MinimumRequiredOutputSpace(size_t input_size,
|
|||
// "compressed" must be preinitialized to at least MinCompressbufSize(comp)
|
||||
// number of bytes, and may contain junk bytes at the end after return.
|
||||
static bool Compress(const char* input, size_t input_size, CompressorType comp,
|
||||
string* compressed, bool compressed_is_preallocated) {
|
||||
std::string* compressed, bool compressed_is_preallocated) {
|
||||
if (!compressed_is_preallocated) {
|
||||
compressed->resize(MinimumRequiredOutputSpace(input_size, comp));
|
||||
}
|
||||
|
|
@ -217,58 +196,6 @@ static bool Compress(const char* input, size_t input_size, CompressorType comp,
|
|||
}
|
||||
#endif // LZO_VERSION
|
||||
|
||||
#ifdef LZF_VERSION
|
||||
case LIBLZF: {
|
||||
int destlen = lzf_compress(input,
|
||||
input_size,
|
||||
string_as_array(compressed),
|
||||
input_size);
|
||||
if (destlen == 0) {
|
||||
// lzf *can* cause lots of blowup when compressing, so they
|
||||
// recommend to limit outsize to insize, and just not compress
|
||||
// if it's bigger. Ideally, we'd just swap input and output.
|
||||
compressed->assign(input, input_size);
|
||||
destlen = input_size;
|
||||
}
|
||||
if (!compressed_is_preallocated) {
|
||||
compressed->resize(destlen);
|
||||
}
|
||||
break;
|
||||
}
|
||||
#endif // LZF_VERSION
|
||||
|
||||
#ifdef QLZ_VERSION_MAJOR
|
||||
case QUICKLZ: {
|
||||
qlz_state_compress *state_compress = new qlz_state_compress;
|
||||
int destlen = qlz_compress(input,
|
||||
string_as_array(compressed),
|
||||
input_size,
|
||||
state_compress);
|
||||
delete state_compress;
|
||||
CHECK_NE(0, destlen);
|
||||
if (!compressed_is_preallocated) {
|
||||
compressed->resize(destlen);
|
||||
}
|
||||
break;
|
||||
}
|
||||
#endif // QLZ_VERSION_MAJOR
|
||||
|
||||
#ifdef FASTLZ_VERSION
|
||||
case FASTLZ: {
|
||||
// Use level 1 compression since we mostly care about speed.
|
||||
int destlen = fastlz_compress_level(
|
||||
1,
|
||||
input,
|
||||
input_size,
|
||||
string_as_array(compressed));
|
||||
if (!compressed_is_preallocated) {
|
||||
compressed->resize(destlen);
|
||||
}
|
||||
CHECK_NE(destlen, 0);
|
||||
break;
|
||||
}
|
||||
#endif // FASTLZ_VERSION
|
||||
|
||||
case SNAPPY: {
|
||||
size_t destlen;
|
||||
snappy::RawCompress(input, input_size,
|
||||
|
|
@ -288,8 +215,8 @@ static bool Compress(const char* input, size_t input_size, CompressorType comp,
|
|||
return true;
|
||||
}
|
||||
|
||||
static bool Uncompress(const string& compressed, CompressorType comp,
|
||||
int size, string* output) {
|
||||
static bool Uncompress(const std::string& compressed, CompressorType comp,
|
||||
int size, std::string* output) {
|
||||
switch (comp) {
|
||||
#ifdef ZLIB_VERSION
|
||||
case ZLIB: {
|
||||
|
|
@ -323,49 +250,6 @@ static bool Uncompress(const string& compressed, CompressorType comp,
|
|||
}
|
||||
#endif // LZO_VERSION
|
||||
|
||||
#ifdef LZF_VERSION
|
||||
case LIBLZF: {
|
||||
output->resize(size);
|
||||
int destlen = lzf_decompress(compressed.data(),
|
||||
compressed.size(),
|
||||
string_as_array(output),
|
||||
output->size());
|
||||
if (destlen == 0) {
|
||||
// This error probably means we had decided not to compress,
|
||||
// and thus have stored input in output directly.
|
||||
output->assign(compressed.data(), compressed.size());
|
||||
destlen = compressed.size();
|
||||
}
|
||||
CHECK_EQ(destlen, size);
|
||||
break;
|
||||
}
|
||||
#endif // LZF_VERSION
|
||||
|
||||
#ifdef QLZ_VERSION_MAJOR
|
||||
case QUICKLZ: {
|
||||
output->resize(size);
|
||||
qlz_state_decompress *state_decompress = new qlz_state_decompress;
|
||||
int destlen = qlz_decompress(compressed.data(),
|
||||
string_as_array(output),
|
||||
state_decompress);
|
||||
delete state_decompress;
|
||||
CHECK_EQ(destlen, size);
|
||||
break;
|
||||
}
|
||||
#endif // QLZ_VERSION_MAJOR
|
||||
|
||||
#ifdef FASTLZ_VERSION
|
||||
case FASTLZ: {
|
||||
output->resize(size);
|
||||
int destlen = fastlz_decompress(compressed.data(),
|
||||
compressed.length(),
|
||||
string_as_array(output),
|
||||
size);
|
||||
CHECK_EQ(destlen, size);
|
||||
break;
|
||||
}
|
||||
#endif // FASTLZ_VERSION
|
||||
|
||||
case SNAPPY: {
|
||||
snappy::RawUncompress(compressed.data(), compressed.size(),
|
||||
string_as_array(output));
|
||||
|
|
@ -393,13 +277,13 @@ static void Measure(const char* data,
|
|||
{
|
||||
// Chop the input into blocks
|
||||
int num_blocks = (length + block_size - 1) / block_size;
|
||||
vector<const char*> input(num_blocks);
|
||||
vector<size_t> input_length(num_blocks);
|
||||
vector<string> compressed(num_blocks);
|
||||
vector<string> output(num_blocks);
|
||||
std::vector<const char*> input(num_blocks);
|
||||
std::vector<size_t> input_length(num_blocks);
|
||||
std::vector<std::string> compressed(num_blocks);
|
||||
std::vector<std::string> output(num_blocks);
|
||||
for (int b = 0; b < num_blocks; b++) {
|
||||
int input_start = b * block_size;
|
||||
int input_limit = min<int>((b+1)*block_size, length);
|
||||
int input_limit = std::min<int>((b+1)*block_size, length);
|
||||
input[b] = data+input_start;
|
||||
input_length[b] = input_limit-input_start;
|
||||
|
||||
|
|
@ -454,29 +338,28 @@ static void Measure(const char* data,
|
|||
}
|
||||
}
|
||||
|
||||
sort(ctime, ctime + kRuns);
|
||||
sort(utime, utime + kRuns);
|
||||
std::sort(ctime, ctime + kRuns);
|
||||
std::sort(utime, utime + kRuns);
|
||||
const int med = kRuns/2;
|
||||
|
||||
float comp_rate = (length / ctime[med]) * repeats / 1048576.0;
|
||||
float uncomp_rate = (length / utime[med]) * repeats / 1048576.0;
|
||||
string x = names[comp];
|
||||
std::string x = names[comp];
|
||||
x += ":";
|
||||
string urate = (uncomp_rate >= 0)
|
||||
? StringPrintf("%.1f", uncomp_rate)
|
||||
: string("?");
|
||||
std::string urate = (uncomp_rate >= 0) ? StrFormat("%.1f", uncomp_rate)
|
||||
: std::string("?");
|
||||
printf("%-7s [b %dM] bytes %6d -> %6d %4.1f%% "
|
||||
"comp %5.1f MB/s uncomp %5s MB/s\n",
|
||||
x.c_str(),
|
||||
block_size/(1<<20),
|
||||
static_cast<int>(length), static_cast<uint32>(compressed_size),
|
||||
(compressed_size * 100.0) / max<int>(1, length),
|
||||
(compressed_size * 100.0) / std::max<int>(1, length),
|
||||
comp_rate,
|
||||
urate.c_str());
|
||||
}
|
||||
|
||||
static int VerifyString(const string& input) {
|
||||
string compressed;
|
||||
static int VerifyString(const std::string& input) {
|
||||
std::string compressed;
|
||||
DataEndingAtUnreadablePage i(input);
|
||||
const size_t written = snappy::Compress(i.data(), i.size(), &compressed);
|
||||
CHECK_EQ(written, compressed.size());
|
||||
|
|
@ -484,15 +367,15 @@ static int VerifyString(const string& input) {
|
|||
snappy::MaxCompressedLength(input.size()));
|
||||
CHECK(snappy::IsValidCompressedBuffer(compressed.data(), compressed.size()));
|
||||
|
||||
string uncompressed;
|
||||
std::string uncompressed;
|
||||
DataEndingAtUnreadablePage c(compressed);
|
||||
CHECK(snappy::Uncompress(c.data(), c.size(), &uncompressed));
|
||||
CHECK_EQ(uncompressed, input);
|
||||
return uncompressed.size();
|
||||
}
|
||||
|
||||
static void VerifyStringSink(const string& input) {
|
||||
string compressed;
|
||||
static void VerifyStringSink(const std::string& input) {
|
||||
std::string compressed;
|
||||
DataEndingAtUnreadablePage i(input);
|
||||
const size_t written = snappy::Compress(i.data(), i.size(), &compressed);
|
||||
CHECK_EQ(written, compressed.size());
|
||||
|
|
@ -500,7 +383,7 @@ static void VerifyStringSink(const string& input) {
|
|||
snappy::MaxCompressedLength(input.size()));
|
||||
CHECK(snappy::IsValidCompressedBuffer(compressed.data(), compressed.size()));
|
||||
|
||||
string uncompressed;
|
||||
std::string uncompressed;
|
||||
uncompressed.resize(input.size());
|
||||
snappy::UncheckedByteArraySink sink(string_as_array(&uncompressed));
|
||||
DataEndingAtUnreadablePage c(compressed);
|
||||
|
|
@ -509,8 +392,8 @@ static void VerifyStringSink(const string& input) {
|
|||
CHECK_EQ(uncompressed, input);
|
||||
}
|
||||
|
||||
static void VerifyIOVec(const string& input) {
|
||||
string compressed;
|
||||
static void VerifyIOVec(const std::string& input) {
|
||||
std::string compressed;
|
||||
DataEndingAtUnreadablePage i(input);
|
||||
const size_t written = snappy::Compress(i.data(), i.size(), &compressed);
|
||||
CHECK_EQ(written, compressed.size());
|
||||
|
|
@ -521,23 +404,28 @@ static void VerifyIOVec(const string& input) {
|
|||
// Try uncompressing into an iovec containing a random number of entries
|
||||
// ranging from 1 to 10.
|
||||
char* buf = new char[input.size()];
|
||||
ACMRandom rnd(input.size());
|
||||
size_t num = rnd.Next() % 10 + 1;
|
||||
std::minstd_rand0 rng(input.size());
|
||||
std::uniform_int_distribution<size_t> uniform_1_to_10(1, 10);
|
||||
size_t num = uniform_1_to_10(rng);
|
||||
if (input.size() < num) {
|
||||
num = input.size();
|
||||
}
|
||||
struct iovec* iov = new iovec[num];
|
||||
int used_so_far = 0;
|
||||
std::bernoulli_distribution one_in_five(1.0 / 5);
|
||||
for (size_t i = 0; i < num; ++i) {
|
||||
assert(used_so_far < input.size());
|
||||
iov[i].iov_base = buf + used_so_far;
|
||||
if (i == num - 1) {
|
||||
iov[i].iov_len = input.size() - used_so_far;
|
||||
} else {
|
||||
// Randomly choose to insert a 0 byte entry.
|
||||
if (rnd.OneIn(5)) {
|
||||
if (one_in_five(rng)) {
|
||||
iov[i].iov_len = 0;
|
||||
} else {
|
||||
iov[i].iov_len = rnd.Uniform(input.size());
|
||||
std::uniform_int_distribution<size_t> uniform_not_used_so_far(
|
||||
0, input.size() - used_so_far - 1);
|
||||
iov[i].iov_len = uniform_not_used_so_far(rng);
|
||||
}
|
||||
}
|
||||
used_so_far += iov[i].iov_len;
|
||||
|
|
@ -551,22 +439,22 @@ static void VerifyIOVec(const string& input) {
|
|||
|
||||
// Test that data compressed by a compressor that does not
|
||||
// obey block sizes is uncompressed properly.
|
||||
static void VerifyNonBlockedCompression(const string& input) {
|
||||
static void VerifyNonBlockedCompression(const std::string& input) {
|
||||
if (input.length() > snappy::kBlockSize) {
|
||||
// We cannot test larger blocks than the maximum block size, obviously.
|
||||
return;
|
||||
}
|
||||
|
||||
string prefix;
|
||||
std::string prefix;
|
||||
Varint::Append32(&prefix, input.size());
|
||||
|
||||
// Setup compression table
|
||||
snappy::internal::WorkingMemory wmem;
|
||||
snappy::internal::WorkingMemory wmem(input.size());
|
||||
int table_size;
|
||||
uint16* table = wmem.GetHashTable(input.size(), &table_size);
|
||||
|
||||
// Compress entire input in one shot
|
||||
string compressed;
|
||||
std::string compressed;
|
||||
compressed += prefix;
|
||||
compressed.resize(prefix.size()+snappy::MaxCompressedLength(input.size()));
|
||||
char* dest = string_as_array(&compressed) + prefix.size();
|
||||
|
|
@ -574,13 +462,13 @@ static void VerifyNonBlockedCompression(const string& input) {
|
|||
dest, table, table_size);
|
||||
compressed.resize(end - compressed.data());
|
||||
|
||||
// Uncompress into string
|
||||
string uncomp_str;
|
||||
// Uncompress into std::string
|
||||
std::string uncomp_str;
|
||||
CHECK(snappy::Uncompress(compressed.data(), compressed.size(), &uncomp_str));
|
||||
CHECK_EQ(uncomp_str, input);
|
||||
|
||||
// Uncompress using source/sink
|
||||
string uncomp_str2;
|
||||
std::string uncomp_str2;
|
||||
uncomp_str2.resize(input.size());
|
||||
snappy::UncheckedByteArraySink sink(string_as_array(&uncomp_str2));
|
||||
snappy::ByteArraySource source(compressed.data(), compressed.size());
|
||||
|
|
@ -592,28 +480,28 @@ static void VerifyNonBlockedCompression(const string& input) {
|
|||
static const int kNumBlocks = 10;
|
||||
struct iovec vec[kNumBlocks];
|
||||
const int block_size = 1 + input.size() / kNumBlocks;
|
||||
string iovec_data(block_size * kNumBlocks, 'x');
|
||||
std::string iovec_data(block_size * kNumBlocks, 'x');
|
||||
for (int i = 0; i < kNumBlocks; i++) {
|
||||
vec[i].iov_base = string_as_array(&iovec_data) + i * block_size;
|
||||
vec[i].iov_len = block_size;
|
||||
}
|
||||
CHECK(snappy::RawUncompressToIOVec(compressed.data(), compressed.size(),
|
||||
vec, kNumBlocks));
|
||||
CHECK_EQ(string(iovec_data.data(), input.size()), input);
|
||||
CHECK_EQ(std::string(iovec_data.data(), input.size()), input);
|
||||
}
|
||||
}
|
||||
|
||||
// Expand the input so that it is at least K times as big as block size
|
||||
static string Expand(const string& input) {
|
||||
static std::string Expand(const std::string& input) {
|
||||
static const int K = 3;
|
||||
string data = input;
|
||||
std::string data = input;
|
||||
while (data.size() < K * snappy::kBlockSize) {
|
||||
data += input;
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
static int Verify(const string& input) {
|
||||
static int Verify(const std::string& input) {
|
||||
VLOG(1) << "Verifying input of size " << input.size();
|
||||
|
||||
// Compress using string based routines
|
||||
|
|
@ -625,7 +513,7 @@ static int Verify(const string& input) {
|
|||
VerifyNonBlockedCompression(input);
|
||||
VerifyIOVec(input);
|
||||
if (!input.empty()) {
|
||||
const string expanded = Expand(input);
|
||||
const std::string expanded = Expand(input);
|
||||
VerifyNonBlockedCompression(expanded);
|
||||
VerifyIOVec(input);
|
||||
}
|
||||
|
|
@ -633,21 +521,20 @@ static int Verify(const string& input) {
|
|||
return result;
|
||||
}
|
||||
|
||||
|
||||
static bool IsValidCompressedBuffer(const string& c) {
|
||||
static bool IsValidCompressedBuffer(const std::string& c) {
|
||||
return snappy::IsValidCompressedBuffer(c.data(), c.size());
|
||||
}
|
||||
static bool Uncompress(const string& c, string* u) {
|
||||
static bool Uncompress(const std::string& c, std::string* u) {
|
||||
return snappy::Uncompress(c.data(), c.size(), u);
|
||||
}
|
||||
|
||||
// This test checks to ensure that snappy doesn't coredump if it gets
|
||||
// corrupted data.
|
||||
TEST(CorruptedTest, VerifyCorrupted) {
|
||||
string source = "making sure we don't crash with corrupted input";
|
||||
std::string source = "making sure we don't crash with corrupted input";
|
||||
VLOG(1) << source;
|
||||
string dest;
|
||||
string uncmp;
|
||||
std::string dest;
|
||||
std::string uncmp;
|
||||
snappy::Compress(source.data(), source.size(), &dest);
|
||||
|
||||
// Mess around with the data. It's hard to simulate all possible
|
||||
|
|
@ -694,9 +581,9 @@ TEST(CorruptedTest, VerifyCorrupted) {
|
|||
|
||||
// try reading stuff in from a bad file.
|
||||
for (int i = 1; i <= 3; ++i) {
|
||||
string data = ReadTestDataFile(StringPrintf("baddata%d.snappy", i).c_str(),
|
||||
0);
|
||||
string uncmp;
|
||||
std::string data =
|
||||
ReadTestDataFile(StrFormat("baddata%d.snappy", i).c_str(), 0);
|
||||
std::string uncmp;
|
||||
// check that we don't return a crazy length
|
||||
size_t ulen;
|
||||
CHECK(!snappy::GetUncompressedLength(data.data(), data.size(), &ulen)
|
||||
|
|
@ -714,7 +601,7 @@ TEST(CorruptedTest, VerifyCorrupted) {
|
|||
// These mirror the compression code in snappy.cc, but are copied
|
||||
// here so that we can bypass some limitations in the how snappy.cc
|
||||
// invokes these routines.
|
||||
static void AppendLiteral(string* dst, const string& literal) {
|
||||
static void AppendLiteral(std::string* dst, const std::string& literal) {
|
||||
if (literal.empty()) return;
|
||||
int n = literal.size() - 1;
|
||||
if (n < 60) {
|
||||
|
|
@ -729,12 +616,12 @@ static void AppendLiteral(string* dst, const string& literal) {
|
|||
n >>= 8;
|
||||
}
|
||||
dst->push_back(0 | ((59+count) << 2));
|
||||
*dst += string(number, count);
|
||||
*dst += std::string(number, count);
|
||||
}
|
||||
*dst += literal;
|
||||
}
|
||||
|
||||
static void AppendCopy(string* dst, int offset, int length) {
|
||||
static void AppendCopy(std::string* dst, int offset, int length) {
|
||||
while (length > 0) {
|
||||
// Figure out how much to copy in one shot
|
||||
int to_copy;
|
||||
|
|
@ -771,51 +658,67 @@ TEST(Snappy, SimpleTests) {
|
|||
Verify("ab");
|
||||
Verify("abc");
|
||||
|
||||
Verify("aaaaaaa" + string(16, 'b') + string("aaaaa") + "abc");
|
||||
Verify("aaaaaaa" + string(256, 'b') + string("aaaaa") + "abc");
|
||||
Verify("aaaaaaa" + string(2047, 'b') + string("aaaaa") + "abc");
|
||||
Verify("aaaaaaa" + string(65536, 'b') + string("aaaaa") + "abc");
|
||||
Verify("abcaaaaaaa" + string(65536, 'b') + string("aaaaa") + "abc");
|
||||
Verify("aaaaaaa" + std::string(16, 'b') + std::string("aaaaa") + "abc");
|
||||
Verify("aaaaaaa" + std::string(256, 'b') + std::string("aaaaa") + "abc");
|
||||
Verify("aaaaaaa" + std::string(2047, 'b') + std::string("aaaaa") + "abc");
|
||||
Verify("aaaaaaa" + std::string(65536, 'b') + std::string("aaaaa") + "abc");
|
||||
Verify("abcaaaaaaa" + std::string(65536, 'b') + std::string("aaaaa") + "abc");
|
||||
}
|
||||
|
||||
// Verify max blowup (lots of four-byte copies)
|
||||
TEST(Snappy, MaxBlowup) {
|
||||
string input;
|
||||
for (int i = 0; i < 20000; i++) {
|
||||
ACMRandom rnd(i);
|
||||
uint32 bytes = static_cast<uint32>(rnd.Next());
|
||||
input.append(reinterpret_cast<char*>(&bytes), sizeof(bytes));
|
||||
}
|
||||
for (int i = 19999; i >= 0; i--) {
|
||||
ACMRandom rnd(i);
|
||||
uint32 bytes = static_cast<uint32>(rnd.Next());
|
||||
input.append(reinterpret_cast<char*>(&bytes), sizeof(bytes));
|
||||
std::mt19937 rng;
|
||||
std::uniform_int_distribution<int> uniform_byte(0, 255);
|
||||
std::string input;
|
||||
for (int i = 0; i < 80000; ++i)
|
||||
input.push_back(static_cast<char>(uniform_byte(rng)));
|
||||
|
||||
for (int i = 0; i < 80000; i += 4) {
|
||||
std::string four_bytes(input.end() - i - 4, input.end() - i);
|
||||
input.append(four_bytes);
|
||||
}
|
||||
Verify(input);
|
||||
}
|
||||
|
||||
TEST(Snappy, RandomData) {
|
||||
ACMRandom rnd(FLAGS_test_random_seed);
|
||||
std::minstd_rand0 rng(FLAGS_test_random_seed);
|
||||
std::uniform_int_distribution<int> uniform_0_to_3(0, 3);
|
||||
std::uniform_int_distribution<int> uniform_0_to_8(0, 8);
|
||||
std::uniform_int_distribution<int> uniform_byte(0, 255);
|
||||
std::uniform_int_distribution<size_t> uniform_4k(0, 4095);
|
||||
std::uniform_int_distribution<size_t> uniform_64k(0, 65535);
|
||||
std::bernoulli_distribution one_in_ten(1.0 / 10);
|
||||
|
||||
const int num_ops = 20000;
|
||||
constexpr int num_ops = 20000;
|
||||
for (int i = 0; i < num_ops; i++) {
|
||||
if ((i % 1000) == 0) {
|
||||
VLOG(0) << "Random op " << i << " of " << num_ops;
|
||||
}
|
||||
|
||||
string x;
|
||||
size_t len = rnd.Uniform(4096);
|
||||
std::string x;
|
||||
size_t len = uniform_4k(rng);
|
||||
if (i < 100) {
|
||||
len = 65536 + rnd.Uniform(65536);
|
||||
len = 65536 + uniform_64k(rng);
|
||||
}
|
||||
while (x.size() < len) {
|
||||
int run_len = 1;
|
||||
if (rnd.OneIn(10)) {
|
||||
run_len = rnd.Skewed(8);
|
||||
if (one_in_ten(rng)) {
|
||||
int skewed_bits = uniform_0_to_8(rng);
|
||||
// int is guaranteed to hold at least 16 bits, this uses at most 8 bits.
|
||||
std::uniform_int_distribution<int> skewed_low(0,
|
||||
(1 << skewed_bits) - 1);
|
||||
run_len = skewed_low(rng);
|
||||
}
|
||||
char c = static_cast<char>(uniform_byte(rng));
|
||||
if (i >= 100) {
|
||||
int skewed_bits = uniform_0_to_3(rng);
|
||||
// int is guaranteed to hold at least 16 bits, this uses at most 3 bits.
|
||||
std::uniform_int_distribution<int> skewed_low(0,
|
||||
(1 << skewed_bits) - 1);
|
||||
c = static_cast<char>(skewed_low(rng));
|
||||
}
|
||||
char c = (i < 100) ? rnd.Uniform(256) : rnd.Skewed(3);
|
||||
while (run_len-- > 0 && x.size() < len) {
|
||||
x += c;
|
||||
x.push_back(c);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -829,19 +732,19 @@ TEST(Snappy, FourByteOffset) {
|
|||
// copy manually.
|
||||
|
||||
// The two fragments that make up the input string.
|
||||
string fragment1 = "012345689abcdefghijklmnopqrstuvwxyz";
|
||||
string fragment2 = "some other string";
|
||||
std::string fragment1 = "012345689abcdefghijklmnopqrstuvwxyz";
|
||||
std::string fragment2 = "some other string";
|
||||
|
||||
// How many times each fragment is emitted.
|
||||
const int n1 = 2;
|
||||
const int n2 = 100000 / fragment2.size();
|
||||
const int length = n1 * fragment1.size() + n2 * fragment2.size();
|
||||
|
||||
string compressed;
|
||||
std::string compressed;
|
||||
Varint::Append32(&compressed, length);
|
||||
|
||||
AppendLiteral(&compressed, fragment1);
|
||||
string src = fragment1;
|
||||
std::string src = fragment1;
|
||||
for (int i = 0; i < n2; i++) {
|
||||
AppendLiteral(&compressed, fragment2);
|
||||
src += fragment2;
|
||||
|
|
@ -850,7 +753,7 @@ TEST(Snappy, FourByteOffset) {
|
|||
src += fragment1;
|
||||
CHECK_EQ(length, src.size());
|
||||
|
||||
string uncompressed;
|
||||
std::string uncompressed;
|
||||
CHECK(snappy::IsValidCompressedBuffer(compressed.data(), compressed.size()));
|
||||
CHECK(snappy::Uncompress(compressed.data(), compressed.size(),
|
||||
&uncompressed));
|
||||
|
|
@ -872,7 +775,7 @@ TEST(Snappy, IOVecEdgeCases) {
|
|||
iov[i].iov_len = kLengths[i];
|
||||
}
|
||||
|
||||
string compressed;
|
||||
std::string compressed;
|
||||
Varint::Append32(&compressed, 22);
|
||||
|
||||
// A literal whose output crosses three blocks.
|
||||
|
|
@ -933,7 +836,7 @@ TEST(Snappy, IOVecLiteralOverflow) {
|
|||
iov[i].iov_len = kLengths[i];
|
||||
}
|
||||
|
||||
string compressed;
|
||||
std::string compressed;
|
||||
Varint::Append32(&compressed, 8);
|
||||
|
||||
AppendLiteral(&compressed, "12345678");
|
||||
|
|
@ -955,7 +858,7 @@ TEST(Snappy, IOVecCopyOverflow) {
|
|||
iov[i].iov_len = kLengths[i];
|
||||
}
|
||||
|
||||
string compressed;
|
||||
std::string compressed;
|
||||
Varint::Append32(&compressed, 8);
|
||||
|
||||
AppendLiteral(&compressed, "123");
|
||||
|
|
@ -969,7 +872,7 @@ TEST(Snappy, IOVecCopyOverflow) {
|
|||
}
|
||||
}
|
||||
|
||||
static bool CheckUncompressedLength(const string& compressed,
|
||||
static bool CheckUncompressedLength(const std::string& compressed,
|
||||
size_t* ulength) {
|
||||
const bool result1 = snappy::GetUncompressedLength(compressed.data(),
|
||||
compressed.size(),
|
||||
|
|
@ -983,7 +886,7 @@ static bool CheckUncompressedLength(const string& compressed,
|
|||
}
|
||||
|
||||
TEST(SnappyCorruption, TruncatedVarint) {
|
||||
string compressed, uncompressed;
|
||||
std::string compressed, uncompressed;
|
||||
size_t ulength;
|
||||
compressed.push_back('\xf0');
|
||||
CHECK(!CheckUncompressedLength(compressed, &ulength));
|
||||
|
|
@ -993,7 +896,7 @@ TEST(SnappyCorruption, TruncatedVarint) {
|
|||
}
|
||||
|
||||
TEST(SnappyCorruption, UnterminatedVarint) {
|
||||
string compressed, uncompressed;
|
||||
std::string compressed, uncompressed;
|
||||
size_t ulength;
|
||||
compressed.push_back('\x80');
|
||||
compressed.push_back('\x80');
|
||||
|
|
@ -1008,7 +911,7 @@ TEST(SnappyCorruption, UnterminatedVarint) {
|
|||
}
|
||||
|
||||
TEST(SnappyCorruption, OverflowingVarint) {
|
||||
string compressed, uncompressed;
|
||||
std::string compressed, uncompressed;
|
||||
size_t ulength;
|
||||
compressed.push_back('\xfb');
|
||||
compressed.push_back('\xff');
|
||||
|
|
@ -1025,14 +928,14 @@ TEST(Snappy, ReadPastEndOfBuffer) {
|
|||
// Check that we do not read past end of input
|
||||
|
||||
// Make a compressed string that ends with a single-byte literal
|
||||
string compressed;
|
||||
std::string compressed;
|
||||
Varint::Append32(&compressed, 1);
|
||||
AppendLiteral(&compressed, "x");
|
||||
|
||||
string uncompressed;
|
||||
std::string uncompressed;
|
||||
DataEndingAtUnreadablePage c(compressed);
|
||||
CHECK(snappy::Uncompress(c.data(), c.size(), &uncompressed));
|
||||
CHECK_EQ(uncompressed, string("x"));
|
||||
CHECK_EQ(uncompressed, std::string("x"));
|
||||
}
|
||||
|
||||
// Check for an infinite loop caused by a copy with offset==0
|
||||
|
|
@ -1054,7 +957,10 @@ TEST(Snappy, ZeroOffsetCopyValidation) {
|
|||
namespace {
|
||||
|
||||
int TestFindMatchLength(const char* s1, const char *s2, unsigned length) {
|
||||
return snappy::internal::FindMatchLength(s1, s2, s2 + length);
|
||||
std::pair<size_t, bool> p =
|
||||
snappy::internal::FindMatchLength(s1, s2, s2 + length);
|
||||
CHECK_EQ(p.first < 8, p.second);
|
||||
return p.first;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
|
@ -1150,22 +1056,24 @@ TEST(Snappy, FindMatchLength) {
|
|||
}
|
||||
|
||||
TEST(Snappy, FindMatchLengthRandom) {
|
||||
const int kNumTrials = 10000;
|
||||
const int kTypicalLength = 10;
|
||||
ACMRandom rnd(FLAGS_test_random_seed);
|
||||
constexpr int kNumTrials = 10000;
|
||||
constexpr int kTypicalLength = 10;
|
||||
std::minstd_rand0 rng(FLAGS_test_random_seed);
|
||||
std::uniform_int_distribution<int> uniform_byte(0, 255);
|
||||
std::bernoulli_distribution one_in_two(1.0 / 2);
|
||||
std::bernoulli_distribution one_in_typical_length(1.0 / kTypicalLength);
|
||||
|
||||
for (int i = 0; i < kNumTrials; i++) {
|
||||
string s, t;
|
||||
char a = rnd.Rand8();
|
||||
char b = rnd.Rand8();
|
||||
while (!rnd.OneIn(kTypicalLength)) {
|
||||
s.push_back(rnd.OneIn(2) ? a : b);
|
||||
t.push_back(rnd.OneIn(2) ? a : b);
|
||||
std::string s, t;
|
||||
char a = static_cast<char>(uniform_byte(rng));
|
||||
char b = static_cast<char>(uniform_byte(rng));
|
||||
while (!one_in_typical_length(rng)) {
|
||||
s.push_back(one_in_two(rng) ? a : b);
|
||||
t.push_back(one_in_two(rng) ? a : b);
|
||||
}
|
||||
DataEndingAtUnreadablePage u(s);
|
||||
DataEndingAtUnreadablePage v(t);
|
||||
int matched = snappy::internal::FindMatchLength(
|
||||
u.data(), v.data(), v.data() + t.size());
|
||||
int matched = TestFindMatchLength(u.data(), v.data(), t.size());
|
||||
if (matched == t.size()) {
|
||||
EXPECT_EQ(s, t);
|
||||
} else {
|
||||
|
|
@ -1195,7 +1103,6 @@ TEST(Snappy, VerifyCharTable) {
|
|||
using snappy::internal::COPY_2_BYTE_OFFSET;
|
||||
using snappy::internal::COPY_4_BYTE_OFFSET;
|
||||
using snappy::internal::char_table;
|
||||
using snappy::internal::wordmask;
|
||||
|
||||
uint16 dst[256];
|
||||
|
||||
|
|
@ -1272,49 +1179,46 @@ TEST(Snappy, VerifyCharTable) {
|
|||
}
|
||||
|
||||
static void CompressFile(const char* fname) {
|
||||
string fullinput;
|
||||
std::string fullinput;
|
||||
CHECK_OK(file::GetContents(fname, &fullinput, file::Defaults()));
|
||||
|
||||
string compressed;
|
||||
std::string compressed;
|
||||
Compress(fullinput.data(), fullinput.size(), SNAPPY, &compressed, false);
|
||||
|
||||
CHECK_OK(file::SetContents(string(fname).append(".comp"), compressed,
|
||||
CHECK_OK(file::SetContents(std::string(fname).append(".comp"), compressed,
|
||||
file::Defaults()));
|
||||
}
|
||||
|
||||
static void UncompressFile(const char* fname) {
|
||||
string fullinput;
|
||||
std::string fullinput;
|
||||
CHECK_OK(file::GetContents(fname, &fullinput, file::Defaults()));
|
||||
|
||||
size_t uncompLength;
|
||||
CHECK(CheckUncompressedLength(fullinput, &uncompLength));
|
||||
|
||||
string uncompressed;
|
||||
std::string uncompressed;
|
||||
uncompressed.resize(uncompLength);
|
||||
CHECK(snappy::Uncompress(fullinput.data(), fullinput.size(), &uncompressed));
|
||||
|
||||
CHECK_OK(file::SetContents(string(fname).append(".uncomp"), uncompressed,
|
||||
CHECK_OK(file::SetContents(std::string(fname).append(".uncomp"), uncompressed,
|
||||
file::Defaults()));
|
||||
}
|
||||
|
||||
static void MeasureFile(const char* fname) {
|
||||
string fullinput;
|
||||
std::string fullinput;
|
||||
CHECK_OK(file::GetContents(fname, &fullinput, file::Defaults()));
|
||||
printf("%-40s :\n", fname);
|
||||
|
||||
int start_len = (FLAGS_start_len < 0) ? fullinput.size() : FLAGS_start_len;
|
||||
int end_len = fullinput.size();
|
||||
if (FLAGS_end_len >= 0) {
|
||||
end_len = min<int>(fullinput.size(), FLAGS_end_len);
|
||||
end_len = std::min<int>(fullinput.size(), FLAGS_end_len);
|
||||
}
|
||||
for (int len = start_len; len <= end_len; len++) {
|
||||
const char* const input = fullinput.data();
|
||||
int repeats = (FLAGS_bytes + len) / (len + 1);
|
||||
if (FLAGS_zlib) Measure(input, len, ZLIB, repeats, 1024<<10);
|
||||
if (FLAGS_lzo) Measure(input, len, LZO, repeats, 1024<<10);
|
||||
if (FLAGS_liblzf) Measure(input, len, LIBLZF, repeats, 1024<<10);
|
||||
if (FLAGS_quicklz) Measure(input, len, QUICKLZ, repeats, 1024<<10);
|
||||
if (FLAGS_fastlz) Measure(input, len, FASTLZ, repeats, 1024<<10);
|
||||
if (FLAGS_snappy) Measure(input, len, SNAPPY, repeats, 4096<<10);
|
||||
|
||||
// For block-size based measurements
|
||||
|
|
@ -1354,10 +1258,10 @@ static void BM_UFlat(int iters, int arg) {
|
|||
// Pick file to process based on "arg"
|
||||
CHECK_GE(arg, 0);
|
||||
CHECK_LT(arg, ARRAYSIZE(files));
|
||||
string contents = ReadTestDataFile(files[arg].filename,
|
||||
files[arg].size_limit);
|
||||
std::string contents =
|
||||
ReadTestDataFile(files[arg].filename, files[arg].size_limit);
|
||||
|
||||
string zcontents;
|
||||
std::string zcontents;
|
||||
snappy::Compress(contents.data(), contents.size(), &zcontents);
|
||||
char* dst = new char[contents.size()];
|
||||
|
||||
|
|
@ -1380,10 +1284,10 @@ static void BM_UValidate(int iters, int arg) {
|
|||
// Pick file to process based on "arg"
|
||||
CHECK_GE(arg, 0);
|
||||
CHECK_LT(arg, ARRAYSIZE(files));
|
||||
string contents = ReadTestDataFile(files[arg].filename,
|
||||
files[arg].size_limit);
|
||||
std::string contents =
|
||||
ReadTestDataFile(files[arg].filename, files[arg].size_limit);
|
||||
|
||||
string zcontents;
|
||||
std::string zcontents;
|
||||
snappy::Compress(contents.data(), contents.size(), &zcontents);
|
||||
|
||||
SetBenchmarkBytesProcessed(static_cast<int64>(iters) *
|
||||
|
|
@ -1403,10 +1307,10 @@ static void BM_UIOVec(int iters, int arg) {
|
|||
// Pick file to process based on "arg"
|
||||
CHECK_GE(arg, 0);
|
||||
CHECK_LT(arg, ARRAYSIZE(files));
|
||||
string contents = ReadTestDataFile(files[arg].filename,
|
||||
files[arg].size_limit);
|
||||
std::string contents =
|
||||
ReadTestDataFile(files[arg].filename, files[arg].size_limit);
|
||||
|
||||
string zcontents;
|
||||
std::string zcontents;
|
||||
snappy::Compress(contents.data(), contents.size(), &zcontents);
|
||||
|
||||
// Uncompress into an iovec containing ten entries.
|
||||
|
|
@ -1449,10 +1353,10 @@ static void BM_UFlatSink(int iters, int arg) {
|
|||
// Pick file to process based on "arg"
|
||||
CHECK_GE(arg, 0);
|
||||
CHECK_LT(arg, ARRAYSIZE(files));
|
||||
string contents = ReadTestDataFile(files[arg].filename,
|
||||
files[arg].size_limit);
|
||||
std::string contents =
|
||||
ReadTestDataFile(files[arg].filename, files[arg].size_limit);
|
||||
|
||||
string zcontents;
|
||||
std::string zcontents;
|
||||
snappy::Compress(contents.data(), contents.size(), &zcontents);
|
||||
char* dst = new char[contents.size()];
|
||||
|
||||
|
|
@ -1467,7 +1371,7 @@ static void BM_UFlatSink(int iters, int arg) {
|
|||
}
|
||||
StopBenchmarkTiming();
|
||||
|
||||
string s(dst, contents.size());
|
||||
std::string s(dst, contents.size());
|
||||
CHECK_EQ(contents, s);
|
||||
|
||||
delete[] dst;
|
||||
|
|
@ -1481,8 +1385,8 @@ static void BM_ZFlat(int iters, int arg) {
|
|||
// Pick file to process based on "arg"
|
||||
CHECK_GE(arg, 0);
|
||||
CHECK_LT(arg, ARRAYSIZE(files));
|
||||
string contents = ReadTestDataFile(files[arg].filename,
|
||||
files[arg].size_limit);
|
||||
std::string contents =
|
||||
ReadTestDataFile(files[arg].filename, files[arg].size_limit);
|
||||
|
||||
char* dst = new char[snappy::MaxCompressedLength(contents.size())];
|
||||
|
||||
|
|
@ -1497,16 +1401,89 @@ static void BM_ZFlat(int iters, int arg) {
|
|||
StopBenchmarkTiming();
|
||||
const double compression_ratio =
|
||||
static_cast<double>(zsize) / std::max<size_t>(1, contents.size());
|
||||
SetBenchmarkLabel(StringPrintf("%s (%.2f %%)",
|
||||
files[arg].label, 100.0 * compression_ratio));
|
||||
VLOG(0) << StringPrintf("compression for %s: %zd -> %zd bytes",
|
||||
files[arg].label, contents.size(), zsize);
|
||||
SetBenchmarkLabel(StrFormat("%s (%.2f %%)", files[arg].label,
|
||||
100.0 * compression_ratio));
|
||||
VLOG(0) << StrFormat("compression for %s: %zd -> %zd bytes",
|
||||
files[arg].label, static_cast<int>(contents.size()),
|
||||
static_cast<int>(zsize));
|
||||
delete[] dst;
|
||||
}
|
||||
BENCHMARK(BM_ZFlat)->DenseRange(0, ARRAYSIZE(files) - 1);
|
||||
|
||||
} // namespace snappy
|
||||
static void BM_ZFlatAll(int iters, int arg) {
|
||||
StopBenchmarkTiming();
|
||||
|
||||
CHECK_EQ(arg, 0);
|
||||
const int num_files = ARRAYSIZE(files);
|
||||
|
||||
std::vector<std::string> contents(num_files);
|
||||
std::vector<char*> dst(num_files);
|
||||
|
||||
int64 total_contents_size = 0;
|
||||
for (int i = 0; i < num_files; ++i) {
|
||||
contents[i] = ReadTestDataFile(files[i].filename, files[i].size_limit);
|
||||
dst[i] = new char[snappy::MaxCompressedLength(contents[i].size())];
|
||||
total_contents_size += contents[i].size();
|
||||
}
|
||||
|
||||
SetBenchmarkBytesProcessed(static_cast<int64>(iters) * total_contents_size);
|
||||
StartBenchmarkTiming();
|
||||
|
||||
size_t zsize = 0;
|
||||
while (iters-- > 0) {
|
||||
for (int i = 0; i < num_files; ++i) {
|
||||
snappy::RawCompress(contents[i].data(), contents[i].size(), dst[i],
|
||||
&zsize);
|
||||
}
|
||||
}
|
||||
StopBenchmarkTiming();
|
||||
|
||||
for (int i = 0; i < num_files; ++i) {
|
||||
delete[] dst[i];
|
||||
}
|
||||
SetBenchmarkLabel(StrFormat("%d files", num_files));
|
||||
}
|
||||
BENCHMARK(BM_ZFlatAll)->DenseRange(0, 0);
|
||||
|
||||
static void BM_ZFlatIncreasingTableSize(int iters, int arg) {
|
||||
StopBenchmarkTiming();
|
||||
|
||||
CHECK_EQ(arg, 0);
|
||||
CHECK_GT(ARRAYSIZE(files), 0);
|
||||
const std::string base_content =
|
||||
ReadTestDataFile(files[0].filename, files[0].size_limit);
|
||||
|
||||
std::vector<std::string> contents;
|
||||
std::vector<char*> dst;
|
||||
int64 total_contents_size = 0;
|
||||
for (int table_bits = kMinHashTableBits; table_bits <= kMaxHashTableBits;
|
||||
++table_bits) {
|
||||
std::string content = base_content;
|
||||
content.resize(1 << table_bits);
|
||||
dst.push_back(new char[snappy::MaxCompressedLength(content.size())]);
|
||||
total_contents_size += content.size();
|
||||
contents.push_back(std::move(content));
|
||||
}
|
||||
|
||||
size_t zsize = 0;
|
||||
SetBenchmarkBytesProcessed(static_cast<int64>(iters) * total_contents_size);
|
||||
StartBenchmarkTiming();
|
||||
while (iters-- > 0) {
|
||||
for (int i = 0; i < contents.size(); ++i) {
|
||||
snappy::RawCompress(contents[i].data(), contents[i].size(), dst[i],
|
||||
&zsize);
|
||||
}
|
||||
}
|
||||
StopBenchmarkTiming();
|
||||
|
||||
for (int i = 0; i < dst.size(); ++i) {
|
||||
delete[] dst[i];
|
||||
}
|
||||
SetBenchmarkLabel(StrFormat("%zd tables", contents.size()));
|
||||
}
|
||||
BENCHMARK(BM_ZFlatIncreasingTableSize)->DenseRange(0, 0);
|
||||
|
||||
} // namespace snappy
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
InitGoogle(argv[0], &argc, &argv, true);
|
||||
|
|
@ -1515,11 +1492,11 @@ int main(int argc, char** argv) {
|
|||
if (argc >= 2) {
|
||||
for (int arg = 1; arg < argc; arg++) {
|
||||
if (FLAGS_write_compressed) {
|
||||
CompressFile(argv[arg]);
|
||||
snappy::CompressFile(argv[arg]);
|
||||
} else if (FLAGS_write_uncompressed) {
|
||||
UncompressFile(argv[arg]);
|
||||
snappy::UncompressFile(argv[arg]);
|
||||
} else {
|
||||
MeasureFile(argv[arg]);
|
||||
snappy::MeasureFile(argv[arg]);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче