Capture/Replay: Use zlib to compress data files.

Gives about a 2-3x reduced data file size. Should help reduce the time
we spent transferring trace files on Android.

This feature is implemented as a parameter to the capture/replay tool.
This keeps backwards compatiblity if we ever need to integrate a cpp
replay into an environment that doesn't have access to zlib for
decompression.

We'll need to update the traces to take advantage of the compression.

Bug: angleproject:4484
Change-Id: Id7994efe3c0d529b85fa7e7f1b00444e630dd2cd
Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/2104555
Commit-Queue: Jamie Madill <jmadill@chromium.org>
Reviewed-by: Jamie Madill <jmadill@chromium.org>

BUILD.gn

@@ -744,8 +744,21 @@ angle_source_set("angle_gl_enum_utils") {
   ]
 }
 
+config("angle_compression_config") {
+  include_dirs = [ "//third_party/zlib/google" ]
+}
+
+group("angle_compression") {
+  public_configs = [
+    ":angle_compression_config",
+    "//third_party/zlib:zlib_config",
+  ]
+  public_deps = [ "//third_party/zlib/google:compression_utils_portable" ]
+}
+
 angle_source_set("libANGLE_with_capture") {
   public_deps = [ ":libANGLE_base" ]
+  deps = [ ":angle_compression" ]
   public_configs = [ ":angle_frame_capture_enabled" ]
   sources = libangle_capture_sources
 }

doc/CaptureAndReplay.md

@@ -32,6 +32,8 @@ Some simple environment variables control frame capture:
 
  * `ANGLE_CAPTURE_ENABLED`:
    * Set to `0` to disable capture entirely. Default is `1`.
+ * `ANGLE_CAPTURE_COMPRESSION`:
+   * Set to `0` to disable capture compression. Default is `1`.
  * `ANGLE_CAPTURE_OUT_DIR=<path>`:
    * Can specify an alternate replay output directory.
    * Example: `ANGLE_CAPTURE_OUT_DIR=samples/capture_replay`. Default is the CWD.

src/libANGLE/FrameCapture.cpp

@@ -29,6 +29,9 @@
 #include "libANGLE/queryconversions.h"
 #include "libANGLE/queryutils.h"
 
+#define USE_SYSTEM_ZLIB
+#include "compression_utils_portable.h"
+
 #if !ANGLE_CAPTURE_ENABLED
 #    error Frame capture must be enbled to include this file.
 #endif  // !ANGLE_CAPTURE_ENABLED
@@ -43,6 +46,7 @@ constexpr char kOutDirectoryVarName[] = "ANGLE_CAPTURE_OUT_DIR";
 constexpr char kFrameStartVarName[]   = "ANGLE_CAPTURE_FRAME_START";
 constexpr char kFrameEndVarName[]     = "ANGLE_CAPTURE_FRAME_END";
 constexpr char kCaptureLabel[]        = "ANGLE_CAPTURE_LABEL";
+constexpr char kCompression[]         = "ANGLE_CAPTURE_COMPRESSION";
 
 #if defined(ANGLE_PLATFORM_ANDROID)
 
@@ -51,6 +55,7 @@ constexpr char kAndroidOutDir[]         = "debug.angle.capture.out_dir";
 constexpr char kAndroidFrameStart[]     = "debug.angle.capture.frame_start";
 constexpr char kAndroidFrameEnd[]       = "debug.angle.capture.frame_end";
 constexpr char kAndroidCaptureLabel[]   = "debug.angle.capture.label";
+constexpr char kAndroidCompression[]    = "debug.angle.capture.compression";
 
 constexpr int kStreamSize = 64;
 
@@ -110,9 +115,16 @@ void PrimeAndroidEnvironmentVariables()
     std::string captureLabel = AndroidGetEnvFromProp(kAndroidCaptureLabel);
     if (!captureLabel.empty())
     {
-        INFO() << "Capture label read " << captureLabel << " from " << kAndroidCaptureLabel;
+        INFO() << "Frame capture read " << captureLabel << " from " << kAndroidCaptureLabel;
         setenv(kCaptureLabel, captureLabel.c_str(), 1);
     }
+
+    std::string compression = AndroidGetEnvFromProp(kAndroidCompression);
+    if (!compression.empty())
+    {
+        INFO() << "Frame capture read " << compression << " from " << kAndroidCompression;
+        setenv(kCompression, compression.c_str(), 1);
+    }
 }
 #endif
 
@@ -604,32 +616,65 @@ struct SaveFileHelper
     std::string filePath;
 };
 
-std::string GetBinaryDataFilePath(int contextId, const std::string &captureLabel)
+std::string GetBinaryDataFilePath(bool compression, int contextId, const std::string &captureLabel)
 {
     std::stringstream fnameStream;
     fnameStream << FmtCapturePrefix(contextId, captureLabel) << ".angledata";
+    if (compression)
+    {
+        fnameStream << ".gz";
+    }
     return fnameStream.str();
 }
 
-void SaveBinaryData(const std::string &outDir,
+void SaveBinaryData(bool compression,
+                    const std::string &outDir,
                     int contextId,
                     const std::string &captureLabel,
                     const std::vector<uint8_t> &binaryData)
 {
-    std::string binaryDataFileName = GetBinaryDataFilePath(contextId, captureLabel);
+    std::string binaryDataFileName = GetBinaryDataFilePath(compression, contextId, captureLabel);
     std::string dataFilepath       = outDir + binaryDataFileName;
 
     SaveFileHelper saveData(dataFilepath, std::ios::binary);
-    saveData.ofs.write(reinterpret_cast<const char *>(binaryData.data()), binaryData.size());
+
+    if (compression)
+    {
+        // Save compressed data.
+        uLong uncompressedSize       = static_cast<uLong>(binaryData.size());
+        uLong expectedCompressedSize = zlib_internal::GzipExpectedCompressedSize(uncompressedSize);
+
+        std::vector<uint8_t> compressedData(expectedCompressedSize, 0);
+
+        uLong compressedSize = expectedCompressedSize;
+        int zResult = zlib_internal::GzipCompressHelper(compressedData.data(), &compressedSize,
+                                                        binaryData.data(), uncompressedSize,
+                                                        nullptr, nullptr);
+
+        if (zResult != Z_OK)
+        {
+            FATAL() << "Error compressing binary data: " << zResult;
+        }
+
+        saveData.ofs.write(reinterpret_cast<const char *>(compressedData.data()), compressedSize);
+    }
+    else
+    {
+        saveData.ofs.write(reinterpret_cast<const char *>(binaryData.data()), binaryData.size());
+    }
 }
 
-void WriteLoadBinaryDataCall(std::ostream &out, int contextId, const std::string &captureLabel)
+void WriteLoadBinaryDataCall(bool compression,
+                             std::ostream &out,
+                             int contextId,
+                             const std::string &captureLabel)
 {
-    std::string binaryDataFileName = GetBinaryDataFilePath(contextId, captureLabel);
+    std::string binaryDataFileName = GetBinaryDataFilePath(compression, contextId, captureLabel);
     out << "    LoadBinaryData(\"" << binaryDataFileName << "\");\n";
 }
 
-void WriteCppReplay(const std::string &outDir,
+void WriteCppReplay(bool compression,
+                    const std::string &outDir,
                     int contextId,
                     const std::string &captureLabel,
                     uint32_t frameIndex,
@@ -661,7 +706,7 @@ void WriteCppReplay(const std::string &outDir,
 
         std::stringstream setupCallStream;
 
-        WriteLoadBinaryDataCall(setupCallStream, contextId, captureLabel);
+        WriteLoadBinaryDataCall(compression, setupCallStream, contextId, captureLabel);
 
         for (const CallCapture &call : setupCalls)
         {
@@ -710,7 +755,8 @@ void WriteCppReplay(const std::string &outDir,
     }
 }
 
-void WriteCppReplayIndexFiles(const std::string &outDir,
+void WriteCppReplayIndexFiles(bool compression,
+                              const std::string &outDir,
                               int contextId,
                               const std::string &captureLabel,
                               uint32_t frameStart,
@@ -772,6 +818,11 @@ void WriteCppReplayIndexFiles(const std::string &outDir,
         header << "void " << FmtReplayFunction(contextId, frameIndex) << ";\n";
     }
     header << "\n";
+    header << "constexpr bool kIsBinaryDataCompressed = " << (compression ? "true" : "false")
+           << ";\n";
+    header << "\n";
+    header << "using DecompressCallback = uint8_t *(*)(const std::vector<uint8_t> &);\n";
+    header << "void SetBinaryDataDecompressCallback(DecompressCallback callback);\n";
     header << "void SetBinaryDataDir(const char *dataDir);\n";
     header << "void LoadBinaryData(const char *fileName);\n";
     header << "\n";
@@ -801,6 +852,7 @@ void WriteCppReplayIndexFiles(const std::string &outDir,
     source << "    (*resourceMap)[id] = returnedID;\n";
     source << "}\n";
     source << "\n";
+    source << "DecompressCallback gDecompressCallback;\n";
     source << "const char *gBinaryDataDir = \".\";\n";
     source << "FramebufferChangeCallback gFramebufferChangeCallback;\n";
     source << "void *gFramebufferChangeCallbackUserData;\n";
@@ -879,6 +931,11 @@ void WriteCppReplayIndexFiles(const std::string &outDir,
     source << "    }\n";
     source << "}\n";
     source << "\n";
+    source << "void SetBinaryDataDecompressCallback(DecompressCallback callback)\n";
+    source << "{\n";
+    source << "    gDecompressCallback = callback;\n";
+    source << "}\n";
+    source << "\n";
     source << "void SetBinaryDataDir(const char *dataDir)\n";
     source << "{\n";
     source << "    gBinaryDataDir = dataDir;\n";
@@ -901,8 +958,17 @@ void WriteCppReplayIndexFiles(const std::string &outDir,
     source << "    fseek(fp, 0, SEEK_END);\n";
     source << "    long size = ftell(fp);\n";
     source << "    fseek(fp, 0, SEEK_SET);\n";
-    source << "    gBinaryData = new uint8_t[size];\n";
+    source << "    if (gDecompressCallback)\n";
-    source << "    (void)fread(gBinaryData, 1, size, fp);\n";
+    source << "    {\n";
+    source << "        std::vector<uint8_t> compressedData(size);\n";
+    source << "        (void)fread(compressedData.data(), 1, size, fp);\n";
+    source << "        gBinaryData = gDecompressCallback(compressedData);\n";
+    source << "    }\n";
+    source << "    else\n";
+    source << "    {\n";
+    source << "        gBinaryData = new uint8_t[size];\n";
+    source << "        (void)fread(gBinaryData, 1, size, fp);\n";
+    source << "    }\n";
     source << "    fclose(fp);\n";
     source << "}\n";
 
@@ -2487,6 +2553,7 @@ ReplayContext::~ReplayContext() {}
 
 FrameCapture::FrameCapture()
     : mEnabled(true),
+      mCompression(true),
       mClientVertexArrayMap{},
       mFrameIndex(0),
       mFrameStart(0),
@@ -2541,6 +2608,12 @@ FrameCapture::FrameCapture()
         // Optional label to provide unique file names and namespaces
         mCaptureLabel = labelFromEnv;
     }
+
+    std::string compressionFromEnv = angle::GetEnvironmentVar(kCompression);
+    if (compressionFromEnv == "0")
+    {
+        mCompression = false;
+    }
 }
 
 FrameCapture::~FrameCapture() = default;
@@ -2984,19 +3057,20 @@ void FrameCapture::onEndFrame(const gl::Context *context)
     // Note that we currently capture before the start frame to collect shader and program sources.
     if (!mFrameCalls.empty() && mFrameIndex >= mFrameStart)
     {
-        WriteCppReplay(mOutDirectory, context->id(), mCaptureLabel, mFrameIndex, mFrameCalls,
+        WriteCppReplay(mCompression, mOutDirectory, context->id(), mCaptureLabel, mFrameIndex,
-                       mSetupCalls, &mBinaryData);
+                       mFrameCalls, mSetupCalls, &mBinaryData);
 
         // Save the index files after the last frame.
         if (mFrameIndex == mFrameEnd)
         {
-            WriteCppReplayIndexFiles(mOutDirectory, context->id(), mCaptureLabel, mFrameStart,
+            WriteCppReplayIndexFiles(mCompression, mOutDirectory, context->id(), mCaptureLabel,
-                                     mFrameEnd, mReadBufferSize, mClientArraySizes,
+                                     mFrameStart, mFrameEnd, mReadBufferSize, mClientArraySizes,
                                      mHasResourceType);
 
             if (!mBinaryData.empty())
             {
-                SaveBinaryData(mOutDirectory, context->id(), mCaptureLabel, mBinaryData);
+                SaveBinaryData(mCompression, mOutDirectory, context->id(), mCaptureLabel,
+                               mBinaryData);
                 mBinaryData.clear();
             }
         }

src/libANGLE/FrameCapture.h

@@ -225,6 +225,7 @@ class FrameCapture final : angle::NonCopyable
     bool mEnabled;
     std::string mOutDirectory;
     std::string mCaptureLabel;
+    bool mCompression;
     gl::AttribArray<int> mClientVertexArrayMap;
     uint32_t mFrameIndex;
     uint32_t mFrameStart;

src/tests/BUILD.gn

@@ -289,7 +289,10 @@ if (is_win || is_linux || is_android || is_mac || is_fuchsia) {
       sources = angle_trace_perf_sources
       defines = angle_trace_perf_defines
       data = angle_trace_perf_data
-      deps = [ ":angle_perftests_shared" ]
+      deps = [
+        ":angle_perftests_shared",
+        "$angle_root:angle_compression",
+      ]
       suppressed_configs +=
           [ "$angle_root:constructor_and_destructor_warnings" ]
 

src/tests/perf_tests/TracePerfTest.cpp

@@ -27,6 +27,9 @@
 #include <functional>
 #include <sstream>
 
+#define USE_SYSTEM_ZLIB
+#include "compression_utils_portable.h"
+
 using namespace angle;
 using namespace egl_platform;
 
@@ -34,6 +37,26 @@ namespace
 {
 void FramebufferChangeCallback(void *userData, GLenum target, GLuint framebuffer);
 
+ANGLE_MAYBE_UNUSED uint8_t *DecompressBinaryData(const std::vector<uint8_t> &compressedData)
+{
+    uint32_t uncompressedSize =
+        zlib_internal::GetGzipUncompressedSize(compressedData.data(), compressedData.size());
+
+    std::unique_ptr<uint8_t[]> uncompressedData(new uint8_t[uncompressedSize]);
+    uLong destLen = uncompressedSize;
+    int zResult =
+        zlib_internal::GzipUncompressHelper(uncompressedData.get(), &destLen, compressedData.data(),
+                                            static_cast<uLong>(compressedData.size()));
+
+    if (zResult != Z_OK)
+    {
+        std::cerr << "Failure to decompressed binary data: " << zResult << "\n";
+        return nullptr;
+    }
+
+    return uncompressedData.release();
+}
+
 enum class TracePerfTestID
 {
     Manhattan10,
@@ -154,6 +177,7 @@ TracePerfTest::TracePerfTest()
     : ANGLERenderTest("TracePerf", GetParam()), mStartFrame(0), mEndFrame(0)
 {}
 
+// TODO(jmadill/cnorthrop): Use decompression path. http://anglebug.com/3630
 #define TRACE_TEST_CASE(NAME)                            \
     mStartFrame = NAME::kReplayFrameStart;               \
     mEndFrame   = NAME::kReplayFrameEnd;                 \