Merge pull request #866 from epistor/compress

Add Transfer-Encoding compression support and extensible compression API

Release/CMakeLists.txt

@@ -18,6 +18,7 @@ enable_testing()
 set(WERROR ON CACHE BOOL "Treat Warnings as Errors.")
 set(CPPREST_EXCLUDE_WEBSOCKETS OFF CACHE BOOL "Exclude websockets functionality.")
 set(CPPREST_EXCLUDE_COMPRESSION OFF CACHE BOOL "Exclude compression functionality.")
+set(CPPREST_EXCLUDE_BROTLI ON CACHE BOOL "Exclude Brotli compression functionality.")
 set(CPPREST_EXPORT_DIR cpprestsdk CACHE STRING "Directory to install CMake config files.")
 set(CPPREST_INSTALL_HEADERS ON CACHE BOOL "Install header files.")
 set(CPPREST_INSTALL ON CACHE BOOL "Add install commands.")
@@ -62,6 +63,7 @@ include(cmake/cpprest_find_boost.cmake)
 include(cmake/cpprest_find_zlib.cmake)
 include(cmake/cpprest_find_openssl.cmake)
 include(cmake/cpprest_find_websocketpp.cmake)
+include(cmake/cpprest_find_brotli.cmake)
 include(CheckIncludeFiles)
 include(GNUInstallDirs)
 

Release/cmake/cpprest_find_brotli.cmake

@@ -0,0 +1,10 @@
+function(cpprest_find_brotli)
+  if(TARGET cpprestsdk_brotli_internal)
+    return()
+  endif()
+
+  find_package(unofficial-brotli REQUIRED)
+
+  add_library(cpprestsdk_brotli_internal INTERFACE)
+  target_link_libraries(cpprestsdk_brotli_internal INTERFACE unofficial::brotli::brotlienc unofficial::brotli::brotlidec unofficial::brotli::brotlicommon)
+endfunction()

Release/cmake/cpprestsdk-config.in.cmake

@@ -3,6 +3,10 @@ if(@CPPREST_USES_ZLIB@)
   find_dependency(ZLIB)
 endif()
 
+if(@CPPREST_USES_BROTLI@)
+  find_dependency(unofficial-brotli)
+endif()
+
 if(@CPPREST_USES_OPENSSL@)
   find_dependency(OpenSSL)
 endif()

Release/include/cpprest/asyncrt_utils.h

@@ -503,7 +503,7 @@ _ASYNCRTIMP const std::error_category & __cdecl linux_category();
 
 /// <summary>
 /// Gets the one global instance of the current platform's error category.
-/// <summary>
+/// </summary>
 _ASYNCRTIMP const std::error_category & __cdecl platform_category();
 
 /// <summary>

Release/include/cpprest/details/http_helpers.h

@@ -41,70 +41,4 @@ namespace details
         _ASYNCRTIMP size_t __cdecl add_chunked_delimiters(_Out_writes_(buffer_size) uint8_t *data, _In_ size_t buffer_size, size_t bytes_read);
     }
 
-    namespace compression
-    {
-        enum class compression_algorithm : int 
-        { 
-            deflate = 15,
-            gzip = 31,
-            invalid = 9999
-        };
-
-        using data_buffer = std::vector<uint8_t>;
-
-        class stream_decompressor
-        {
-        public:
-
-            static compression_algorithm to_compression_algorithm(const utility::string_t& alg)
-            {
-                if (_XPLATSTR("gzip") == alg)
-                {
-                    return compression_algorithm::gzip;
-                }
-                else if (_XPLATSTR("deflate") == alg)
-                {
-                    return compression_algorithm::deflate;
-                }
-
-                return compression_algorithm::invalid;
-            }
-
-            static utility::string_t known_algorithms() { return _XPLATSTR("deflate, gzip"); }
-
-            _ASYNCRTIMP static bool __cdecl is_supported();
-
-            _ASYNCRTIMP stream_decompressor(compression_algorithm alg);
-
-            _ASYNCRTIMP data_buffer decompress(const data_buffer& input);
-
-            _ASYNCRTIMP data_buffer decompress(const uint8_t* input, size_t input_size);
-
-            _ASYNCRTIMP bool has_error() const;
-
-        private:
-            class stream_decompressor_impl;
-            std::shared_ptr<stream_decompressor_impl> m_pimpl;
-        };
-
-        class stream_compressor
-        {
-        public:
-
-            _ASYNCRTIMP static bool __cdecl is_supported();
-
-            _ASYNCRTIMP stream_compressor(compression_algorithm alg);
-
-            _ASYNCRTIMP data_buffer compress(const data_buffer& input, bool finish);
-
-            _ASYNCRTIMP data_buffer compress(const uint8_t* input, size_t input_size, bool finish);
-
-            _ASYNCRTIMP bool has_error() const;
-
-        private:
-            class stream_compressor_impl;
-            std::shared_ptr<stream_compressor_impl> m_pimpl;
-        };
-
-    }
 }}}

Release/include/cpprest/http_client.h

@@ -247,20 +247,22 @@ public:
     }
 
     /// <summary>
-    /// Checks if requesting a compressed response is turned on, the default is off.
+    /// Checks if requesting a compressed response using Content-Encoding is turned on, the default is off.
     /// </summary>
-    /// <returns>True if compressed response is enabled, false otherwise</returns>
+    /// <returns>True if a content-encoded compressed response is allowed, false otherwise</returns>
     bool request_compressed_response() const
     {
         return m_request_compressed;
     }
 
     /// <summary>
-    /// Request that the server responds with a compressed body.
-    /// If true, in cases where the server does not support compression, this will have no effect.
+    /// Request that the server respond with a compressed body using Content-Encoding; to use Transfer-Encoding, do not
+    /// set this, and specify a vector of <see cref="web::http::details::comporession::decompress_factory" /> pointers
+    /// to the set_decompress_factories method of the <see cref="web::http::http_request" /> object for the request.
+    /// If true and the server does not support compression, this will have no effect.
     /// The response body is internally decompressed before the consumer receives the data.
     /// </summary>
-    /// <param name="request_compressed">True to turn on response body compression, false otherwise.</param>
+    /// <param name="request_compressed">True to turn on content-encoded response body compression, false otherwise.</param>
     /// <remarks>Please note there is a performance cost due to copying the request data. Currently only supported on Windows and OSX.</remarks>
     void set_request_compressed_response(bool request_compressed)
     {

Release/include/cpprest/http_compression.h

@@ -0,0 +1,319 @@
+/***
+ * Copyright (C) Microsoft. All rights reserved.
+ * Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
+ *
+ * =+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+
+ *
+ * HTTP Library: Compression and decompression interfaces
+ *
+ * For the latest on this and related APIs, please see: https://github.com/Microsoft/cpprestsdk
+ *
+ * =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+ ****/
+#pragma once
+
+namespace web
+{
+namespace http
+{
+namespace compression
+{
+/// <summary>
+/// Hint as to whether a compress or decompress call is meant to be the last for a particular HTTP request or reply
+/// </summary>
+enum operation_hint
+{
+    is_last, // Used for the expected last compress() call, or for an expected single decompress() call
+    has_more // Used when further compress() calls will be made, or when multiple decompress() calls may be required
+};
+
+/// <summary>
+/// Result structure for asynchronous compression and decompression operations
+/// </summary>
+struct operation_result
+{
+    size_t input_bytes_processed; // From the input buffer
+    size_t output_bytes_produced; // To the output buffer
+    bool done; // For compress, set when 'last' is true and there was enough space to complete compression;
+               // for decompress, set if the end of the decompression stream has been reached
+};
+
+/// <summary>
+/// Compression interface for use with HTTP requests
+/// </summary>
+class compress_provider
+{
+public:
+    virtual const utility::string_t& algorithm() const = 0;
+    virtual size_t compress(const uint8_t* input,
+                            size_t input_size,
+                            uint8_t* output,
+                            size_t output_size,
+                            operation_hint hint,
+                            size_t& input_bytes_processed,
+                            bool* done = nullptr) = 0;
+    virtual pplx::task<operation_result> compress(
+        const uint8_t* input, size_t input_size, uint8_t* output, size_t output_size, operation_hint hint) = 0;
+    virtual void reset() = 0;
+    virtual ~compress_provider() = default;
+};
+
+/// <summary>
+/// Decompression interface for use with HTTP requests
+/// </summary>
+class decompress_provider
+{
+public:
+    virtual const utility::string_t& algorithm() const = 0;
+    virtual size_t decompress(const uint8_t* input,
+                              size_t input_size,
+                              uint8_t* output,
+                              size_t output_size,
+                              operation_hint hint,
+                              size_t& input_bytes_processed,
+                              bool* done = nullptr) = 0;
+    virtual pplx::task<operation_result> decompress(
+        const uint8_t* input, size_t input_size, uint8_t* output, size_t output_size, operation_hint hint) = 0;
+    virtual void reset() = 0;
+    virtual ~decompress_provider() = default;
+};
+
+/// <summary>
+/// Factory interface for compressors for use with received HTTP requests
+/// </summary>
+class compress_factory
+{
+public:
+    virtual const utility::string_t& algorithm() const = 0;
+    virtual std::unique_ptr<compress_provider> make_compressor() const = 0;
+    virtual ~compress_factory() = default;
+};
+
+/// <summary>
+/// Factory interface for decompressors for use with HTTP requests
+/// </summary>
+class decompress_factory
+{
+public:
+    virtual const utility::string_t& algorithm() const = 0;
+    virtual const uint16_t weight() const = 0;
+    virtual std::unique_ptr<decompress_provider> make_decompressor() const = 0;
+    virtual ~decompress_factory() = default;
+};
+
+/// <summary>
+/// Built-in compression support
+/// </summary>
+namespace builtin
+{
+/// <summary>
+/// Test whether cpprestsdk was built with built-in compression support
+/// <returns>True if cpprestsdk was built with built-in compression support, and false if not.</returns>
+/// </summary>
+_ASYNCRTIMP bool supported();
+
+/// <summary>
+// String constants for each built-in compression algorithm, for convenient use with the factory functions
+/// </summary>
+namespace algorithm
+{
+constexpr utility::char_t *GZIP = _XPLATSTR("gzip");
+constexpr utility::char_t *DEFLATE = _XPLATSTR("deflate");
+constexpr utility::char_t *BROTLI = _XPLATSTR("br");
+
+/// <summary>
+/// Test whether cpprestsdk was built with built-in compression support and
+/// the supplied string matches a supported built-in algorithm
+/// <param name="algorithm">The name of the algorithm to test for built-in support.</param>
+/// <returns>True if cpprestsdk was built with built-in compression support and
+/// the supplied string matches a supported built-in algorithm, and false if not.</returns>
+/// <summary>
+_ASYNCRTIMP bool supported(const utility::string_t& algorithm);
+};
+
+/// <summary>
+/// Factory function to instantiate a built-in compression provider with default parameters by compression algorithm
+/// name.
+/// </summary>
+/// <param name="algorithm">The name of the algorithm for which to instantiate a provider.</param>
+/// <returns>
+/// A caller-owned pointer to a provider of the requested-type, or to nullptr if no such built-in type exists.
+/// </returns>
+_ASYNCRTIMP std::unique_ptr<compress_provider> make_compressor(const utility::string_t& algorithm);
+
+/// <summary>
+/// Factory function to instantiate a built-in decompression provider with default parameters by compression algorithm
+/// name.
+/// </summary>
+/// <param name="algorithm">The name of the algorithm for which to instantiate a provider.</param>
+/// <returns>
+/// A caller-owned pointer to a provider of the requested-type, or to nullptr if no such built-in type exists.
+/// </returns>
+_ASYNCRTIMP std::unique_ptr<decompress_provider> make_decompressor(const utility::string_t& algorithm);
+
+/// <summary>
+/// Factory function to obtain a pointer to a built-in compression provider factory by compression algorithm name.
+/// </summary>
+/// <param name="algorithm">The name of the algorithm for which to find a factory.</param>
+/// <returns>
+/// A caller-owned pointer to a provider of the requested-type, or to nullptr if no such built-in type exists.
+/// </returns>
+_ASYNCRTIMP std::shared_ptr<compress_factory> get_compress_factory(const utility::string_t& algorithm);
+
+/// <summary>
+/// Factory function to obtain a pointer to a built-in decompression provider factory by compression algorithm name.
+/// </summary>
+/// <param name="algorithm">The name of the algorithm for which to find a factory.</param>
+/// <returns>
+/// A caller-owned pointer to a provider of the requested-type, or to nullptr if no such built-in type exists.
+/// </returns>
+_ASYNCRTIMP std::shared_ptr<decompress_factory> get_decompress_factory(const utility::string_t& algorithm);
+
+/// <summary>
+// Factory function to instantiate a built-in gzip compression provider with caller-selected parameters.
+/// </summary>
+/// <returns>
+/// A caller-owned pointer to a gzip compression provider, or to nullptr if the library was built without built-in
+/// compression support.
+/// </returns>
+_ASYNCRTIMP std::unique_ptr<compress_provider> make_gzip_compressor(int compressionLevel,
+                                                                    int method,
+                                                                    int strategy,
+                                                                    int memLevel);
+
+/// <summary>
+// Factory function to instantiate a built-in deflate compression provider with caller-selected parameters.
+/// </summary>
+/// <returns>
+/// A caller-owned pointer to a deflate compression provider, or to nullptr if the library was built without built-in
+/// compression support..
+/// </returns>
+_ASYNCRTIMP std::unique_ptr<compress_provider> make_deflate_compressor(int compressionLevel,
+                                                                       int method,
+                                                                       int strategy,
+                                                                       int memLevel);
+
+/// <summary>
+// Factory function to instantiate a built-in Brotli compression provider with caller-selected parameters.
+/// </summary>
+/// <returns>
+/// A caller-owned pointer to a Brotli compression provider, or to nullptr if the library was built without built-in
+/// compression support.
+/// </returns>
+_ASYNCRTIMP std::unique_ptr<compress_provider> make_brotli_compressor(uint32_t window, uint32_t quality, uint32_t mode);
+} // namespace builtin
+
+/// <summary>
+/// Factory function to instantiate a compression provider factory by compression algorithm name.
+/// </summary>
+/// <param name="algorithm">The name of the algorithm supported by the factory.  Must match that returned by the
+/// <c>web::http::compression::compress_provider</c> type instantiated by the factory's make_compressor function.
+/// The supplied string is copied, and thus need not remain valid once the call returns.</param>
+/// <param name="make_compressor">A factory function to be used to instantiate a compressor matching the factory's
+/// reported algorithm.</param>
+/// <returns>
+/// A pointer to a generic provider factory implementation configured with the supplied parameters.
+/// </returns>
+/// <remarks>
+/// This method may be used to conveniently instantiate a factory object for a caller-selected <c>compress_provider</c>.
+/// That provider may be of the caller's own design, or it may be one of the built-in types.  As such, this method may
+/// be helpful when a caller wishes to build vectors containing a mix of custom and built-in providers.
+/// </remarks>
+_ASYNCRTIMP std::shared_ptr<compress_factory> make_compress_factory(
+    const utility::string_t& algorithm, std::function<std::unique_ptr<compress_provider>()> make_compressor);
+
+/// <summary>
+/// Factory function to instantiate a decompression provider factory by compression algorithm name.
+/// </summary>
+/// <param name="algorithm">The name of the algorithm supported by the factory.  Must match that returned by the
+/// <c>web::http::compression::decompress_provider</c> type instantiated by the factory's make_decompressor function.
+/// The supplied string is copied, and thus need not remain valid once the call returns.</param>
+/// <param name="weight">A numeric weight for the compression algorithm, times 1000,  for use as a "quality value" when
+/// requesting that the server send a compressed response.  Valid values are between 0 and 1000, inclusive, where higher
+/// values indicate more preferred algorithms, and 0 indicates that the algorithm is not allowed; values greater than
+/// 1000 are treated as 1000.</param>
+/// <param name="make_decompressor">A factory function to be used to instantiate a decompressor matching the factory's
+/// reported algorithm.</param>
+/// <returns>
+/// A pointer to a generic provider factory implementation configured with the supplied parameters.
+/// </returns>
+/// <remarks>
+/// This method may be used to conveniently instantiate a factory object for a caller-selected
+/// <c>decompress_provider</c>.  That provider may be of the caller's own design, or it may be one of the built-in
+/// types.  As such, this method may be helpful when a caller wishes to change the weights of built-in provider types,
+/// to use custom providers without explicitly implementing a <c>decompress_factory</c>, or to build vectors containing
+/// a mix of custom and built-in providers.
+/// </remarks>
+_ASYNCRTIMP std::shared_ptr<decompress_factory> make_decompress_factory(
+    const utility::string_t& algorithm,
+    uint16_t weight,
+    std::function<std::unique_ptr<decompress_provider>()> make_decompressor);
+
+namespace details
+{
+/// <summary>
+/// Header type enum for use with compressor and decompressor header parsing and building functions
+/// </summary>
+enum header_types
+{
+    transfer_encoding,
+    content_encoding,
+    te,
+    accept_encoding
+};
+
+/// <summary>
+/// Factory function to instantiate an appropriate compression provider, if any.
+/// </summary>
+/// <param name="encoding">A TE or Accept-Encoding header to interpret.</param>
+/// <param name="type">Specifies the type of header whose contents are in the encoding parameter; valid values are
+/// <c>header_type::te</c> and <c>header_type::accept_encoding</c>.</param>
+/// <param name="preferred">A compressor object of the caller's preferred (possibly custom) type, which is used if
+/// possible.</param>
+/// <param name="factories">A collection of factory objects for use in construction of an appropriate compressor, if
+/// any. If empty or not supplied, the set of supported built-in compressors is used.</param>
+/// <returns>
+/// A pointer to a compressor object that is acceptable per the supplied header, or to nullptr if no matching
+/// algorithm is found.
+/// </returns>
+_ASYNCRTIMP std::unique_ptr<compress_provider> get_compressor_from_header(
+    const utility::string_t& encoding,
+    header_types type,
+    const std::vector<std::shared_ptr<compress_factory>>& factories = std::vector<std::shared_ptr<compress_factory>>());
+
+/// <summary>
+/// Factory function to instantiate an appropriate decompression provider, if any.
+/// </summary>
+/// <param name="encoding">A Transfer-Encoding or Content-Encoding header to interpret.</param>
+/// <param name="type">Specifies the type of header whose contents are in the encoding parameter; valid values are
+/// <c>header_type::transfer_encoding</c> and <c>header_type::content_encoding</c>.</param>
+/// <param name="factories">A collection of factory objects for use in construction of an appropriate decompressor,
+/// if any. If empty or not supplied, the set of supported built-in compressors is used.</param>
+/// <returns>
+/// A pointer to a decompressor object that is acceptable per the supplied header, or to nullptr if no matching
+/// algorithm is found.
+/// </returns>
+_ASYNCRTIMP std::unique_ptr<decompress_provider> get_decompressor_from_header(
+    const utility::string_t& encoding,
+    header_types type,
+    const std::vector<std::shared_ptr<decompress_factory>>& factories =
+        std::vector<std::shared_ptr<decompress_factory>>());
+
+/// <summary>
+/// Helper function to compose a TE or Accept-Encoding header with supported, and possibly ranked, compression
+/// algorithms.
+/// </summary>
+/// <param name="type">Specifies the type of header to be built; valid values are <c>header_type::te</c> and
+/// <c>header_type::accept_encoding</c>.</param>
+/// <param name="factories">A collection of factory objects for use in header construction. If empty or not
+/// supplied, the set of supported built-in compressors is used.</param>
+/// <returns>
+/// A well-formed header, without the header name, specifying the acceptable ranked compression types.
+/// </returns>
+_ASYNCRTIMP utility::string_t build_supported_header(header_types type,
+                                                     const std::vector<std::shared_ptr<decompress_factory>>& factories =
+                                                         std::vector<std::shared_ptr<decompress_factory>>());
+} // namespace details
+} // namespace compression
+} // namespace http
+} // namespace web

Release/include/cpprest/http_headers.h

@@ -219,7 +219,7 @@ public:
             return false;
         }
 
-        return bind_impl(iter->second, value) || iter->second.empty();
+        return details::bind_impl(iter->second, value) || iter->second.empty();
     }
 
     /// <summary>
@@ -285,9 +285,14 @@ public:
     _ASYNCRTIMP void set_date(const utility::datetime& date);
 
 private:
+    // Headers are stored in a map with case insensitive key.
+    inner_container m_headers;
+};
 
-    template<typename _t>
-    bool bind_impl(const key_type &text, _t &ref) const
+namespace details
+{
+    template<typename key_type, typename _t>
+    bool bind_impl(const key_type &text, _t &ref)
     {
         utility::istringstream_t iss(text);
         iss.imbue(std::locale::classic());
@@ -300,20 +305,18 @@ private:
         return true;
     }
 
-    bool bind_impl(const key_type &text, utf16string &ref) const
+    template<typename key_type>
+    bool bind_impl(const key_type &text, utf16string &ref)
     {
         ref = utility::conversions::to_utf16string(text);
         return true;
     }
 
-    bool bind_impl(const key_type &text, std::string &ref) const
+    template<typename key_type>
+    bool bind_impl(const key_type &text, std::string &ref)
     {
         ref = utility::conversions::to_utf8string(text);
         return true;
     }
-
-    // Headers are stored in a map with case insensitive key.
-    inner_container m_headers;
-};
-
+}
 }}

Release/include/cpprest/http_msg.h

@@ -26,6 +26,7 @@
 #include "cpprest/asyncrt_utils.h"
 #include "cpprest/streams.h"
 #include "cpprest/containerstream.h"
+#include "cpprest/http_compression.h"
 
 namespace web
 {
@@ -338,6 +339,38 @@ public:
     /// </summary>
     const concurrency::streams::ostream & outstream() const { return m_outStream; }
 
+    /// <summary>
+    /// Sets the compressor for the message body
+    /// </summary>
+    void set_compressor(std::unique_ptr<http::compression::compress_provider> compressor)
+    {
+        m_compressor = std::move(compressor);
+    }
+
+    /// <summary>
+    /// Gets the compressor for the message body, if any
+    /// </summary>
+    std::unique_ptr<http::compression::compress_provider> &compressor()
+    {
+        return m_compressor;
+    }
+
+    /// <summary>
+    /// Sets the collection of factory classes for decompressors for use with the message body
+    /// </summary>
+    void set_decompress_factories(const std::vector<std::shared_ptr<http::compression::decompress_factory>> &factories)
+    {
+        m_decompressors = factories;
+    }
+
+    /// <summary>
+    /// Gets the collection of factory classes for decompressors to be used to decompress the message body, if any
+    /// </summary>
+    const std::vector<std::shared_ptr<http::compression::decompress_factory>> &decompress_factories()
+    {
+        return m_decompressors;
+    }
+
     const pplx::task_completion_event<utility::size64_t> & _get_data_available() const { return m_data_available; }
 
     /// <summary>
@@ -345,11 +378,25 @@ public:
     /// </summary>
     _ASYNCRTIMP void _prepare_to_receive_data();
 
+
+    /// <summary>
+    /// Determine the remaining input stream length
+    /// </summary>
+    /// <returns>
+    /// std::numeric_limits<size_t>::max() if the stream's remaining length cannot be determined
+    /// length      if the stream's remaining length (which may be 0) can be determined
+    /// </returns>
+    /// <remarks>
+    /// This routine should only be called after a msg (request/response) has been
+    /// completely constructed.
+    /// </remarks>
+    _ASYNCRTIMP size_t _get_stream_length();
+
     /// <summary>
     /// Determine the content length
     /// </summary>
     /// <returns>
-    /// size_t::max if there is content with unknown length (transfer_encoding:chunked)
+    /// std::numeric_limits<size_t>::max() if there is content with unknown length (transfer_encoding:chunked)
     /// 0           if there is no content
     /// length      if there is content with known length
     /// </returns>
@@ -359,8 +406,27 @@ public:
     /// </remarks>
     _ASYNCRTIMP size_t _get_content_length();
 
+    /// <summary>
+    /// Determine the content length, and, if necessary, manage compression in the Transfer-Encoding header
+    /// </summary>
+    /// <returns>
+    /// std::numeric_limits<size_t>::max() if there is content with unknown length (transfer_encoding:chunked)
+    /// 0           if there is no content
+    /// length      if there is content with known length
+    /// </returns>
+    /// <remarks>
+    /// This routine is like _get_content_length, except that it adds a compression algorithm to
+    /// the Trasfer-Length header if compression is configured.  It throws if a Transfer-Encoding
+    /// header exists and does not match the one it generated.
+    /// </remarks>
+    _ASYNCRTIMP size_t _get_content_length_and_set_compression();
+
 protected:
 
+    std::unique_ptr<http::compression::compress_provider> m_compressor;
+    std::unique_ptr<http::compression::decompress_provider> m_decompressor;
+    std::vector<std::shared_ptr<http::compression::decompress_factory>> m_decompressors;
+
     /// <summary>
     /// Stream to read the message body.
     /// By default this is an invalid stream. The user could set the instream on
@@ -386,6 +452,8 @@ protected:
 
     /// <summary> The TCE is used to signal the availability of the message body. </summary>
     pplx::task_completion_event<utility::size64_t> m_data_available;
+
+    size_t _get_content_length(bool honor_compression);
 };
 
 /// <summary>
@@ -671,7 +739,7 @@ public:
     /// <param name="stream">A readable, open asynchronous stream.</param>
     /// <param name="content_type">A string holding the MIME type of the message body.</param>
     /// <remarks>
-    /// This cannot be used in conjunction with any other means of setting the body of the request.
+    /// This cannot be used in conjunction with any external means of setting the body of the request.
     /// The stream will not be read until the message is sent.
     /// </remarks>
     void set_body(const concurrency::streams::istream &stream, const utility::string_t &content_type = _XPLATSTR("application/octet-stream"))
@@ -687,7 +755,7 @@ public:
     /// <param name="content_length">The size of the data to be sent in the body.</param>
     /// <param name="content_type">A string holding the MIME type of the message body.</param>
     /// <remarks>
-    /// This cannot be used in conjunction with any other means of setting the body of the request.
+    /// This cannot be used in conjunction with any external means of setting the body of the request.
     /// The stream will not be read until the message is sent.
     /// </remarks>
     void set_body(const concurrency::streams::istream &stream, utility::size64_t content_length, const utility::string_t &content_type = _XPLATSTR("application/octet-stream"))
@@ -1149,6 +1217,94 @@ public:
         return _m_impl->set_response_stream(stream);
     }
 
+    /// <summary>
+    /// Sets a compressor that will be used to compress the body of the HTTP message as it is sent.
+    /// </summary>
+    /// <param name="compressor">A pointer to an instantiated compressor of the desired type.</param>
+    /// <remarks>
+    /// This cannot be used in conjunction with any external means of compression.  The Transfer-Encoding
+    /// header will be managed internally, and must not be set by the client.
+    /// </remarks>
+    void set_compressor(std::unique_ptr<http::compression::compress_provider> compressor)
+    {
+        return _m_impl->set_compressor(std::move(compressor));
+    }
+
+    /// <summary>
+    /// Sets a compressor that will be used to compress the body of the HTTP message as it is sent.
+    /// </summary>
+    /// <param name="algorithm">The built-in compression algorithm to use.</param>
+    /// <returns>
+    /// True if a built-in compressor was instantiated, otherwise false.
+    /// </returns>
+    /// <remarks>
+    /// This cannot be used in conjunction with any external means of compression.  The Transfer-Encoding
+    /// header will be managed internally, and must not be set by the client.
+    /// </remarks>
+    bool set_compressor(utility::string_t algorithm)
+    {
+        _m_impl->set_compressor(http::compression::builtin::make_compressor(algorithm));
+        return (bool)_m_impl->compressor();
+    }
+
+    /// <summary>
+    /// Gets the compressor to be used to compress the message body, if any.
+    /// </summary>
+    /// <returns>
+    /// The compressor itself.
+    /// </returns>
+    std::unique_ptr<http::compression::compress_provider> &compressor()
+    {
+        return _m_impl->compressor();
+    }
+
+    /// <summary>
+    /// Sets the default collection of built-in factory classes for decompressors that may be used to
+    /// decompress the body of the HTTP message as it is received, effectively enabling decompression.
+    /// </summary>
+    /// <param name="factories">The collection of factory classes for allowable decompressors. The
+    /// supplied vector itself need not remain valid after the call returns.</param>
+    /// <remarks>
+    /// This default collection is implied if request_compressed_response() is set in the associated
+    /// <c>client::http_client_config</c> and neither overload of this method has been called.
+    ///
+    /// This cannot be used in conjunction with any external means of decompression.  The TE and Accept-Encoding
+    /// headers must not be set by the client, as they will be managed internally as appropriate.
+    /// </remarks>
+    _ASYNCRTIMP void set_decompress_factories();
+
+    /// <summary>
+    /// Sets a collection of factory classes for decompressors that may be used to decompress the
+    /// body of the HTTP message as it is received, effectively enabling decompression.
+    /// </summary>
+    /// <remarks>
+    /// If set, this collection takes the place of the built-in compression providers.  It may contain
+    /// custom factory classes and/or factory classes for built-in providers, and may be used to adjust
+    /// the weights of the built-in providers, which default to 500 (i.e. "q=0.500").
+    ///
+    /// This cannot be used in conjunction with any external means of decompression.  The TE and Accept-Encoding
+    /// headers must not be set by the client, as they will be managed internally as appropriate.
+    /// </remarks>
+    void set_decompress_factories(const std::vector<std::shared_ptr<http::compression::decompress_factory>> &factories)
+    {
+        return _m_impl->set_decompress_factories(factories);
+    }
+
+    /// <summary>
+    /// Gets the collection of factory classes for decompressors to be used to decompress the message body, if any.
+    /// </summary>
+    /// <returns>
+    /// The collection of factory classes itself.
+    /// </returns>
+    /// <remarks>
+    /// This cannot be used in conjunction with any external means of decompression.  The TE
+    /// header must not be set by the client, as it will be managed internally.
+    /// </remarks>
+    const std::vector<std::shared_ptr<http::compression::decompress_factory>> &decompress_factories() const
+    {
+        return _m_impl->decompress_factories();
+    }
+
     /// <summary>
     /// Defines a callback function that will be invoked for every chunk of data uploaded or downloaded
     /// as part of the request.

Release/src/CMakeLists.txt

@@ -21,6 +21,7 @@ set(SOURCES
   http/common/internal_http_helpers.h
   http/common/http_helpers.cpp
   http/common/http_msg.cpp
+  http/common/http_compression.cpp
   http/listener/http_listener.cpp
   http/listener/http_listener_msg.cpp
   http/listener/http_server_api.cpp
@@ -71,10 +72,19 @@ endif()
 
 # Compression component
 if(CPPREST_EXCLUDE_COMPRESSION)
+  if(NOT CPPREST_EXCLUDE_BROTLI)
+    message(FATAL_ERROR "Use of Brotli requires compression to be enabled")
+  endif()
   target_compile_definitions(cpprest PRIVATE -DCPPREST_EXCLUDE_COMPRESSION=1)
 else()
   cpprest_find_zlib()
   target_link_libraries(cpprest PRIVATE cpprestsdk_zlib_internal)
+  if(CPPREST_EXCLUDE_BROTLI)
+    target_compile_definitions(cpprest PRIVATE -DCPPREST_EXCLUDE_BROTLI=1)
+  else()
+    cpprest_find_brotli()
+    target_link_libraries(cpprest PRIVATE cpprestsdk_brotli_internal)
+  endif()
 endif()
 
 # PPLX component
@@ -232,6 +242,7 @@ endif()
 if(CPPREST_INSTALL)
   set(CPPREST_USES_BOOST OFF)
   set(CPPREST_USES_ZLIB OFF)
+  set(CPPREST_USES_BROTLI OFF)
   set(CPPREST_USES_OPENSSL OFF)
 
   set(CPPREST_TARGETS cpprest)
@@ -243,6 +254,10 @@ if(CPPREST_INSTALL)
     list(APPEND CPPREST_TARGETS cpprestsdk_zlib_internal)
     set(CPPREST_USES_ZLIB ON)
   endif()
+  if(TARGET cpprestsdk_brotli_internal)
+    list(APPEND CPPREST_TARGETS cpprestsdk_brotli_internal)
+    set(CPPREST_USES_BROTLI ON)
+  endif()
   if(TARGET cpprestsdk_openssl_internal)
     list(APPEND CPPREST_TARGETS cpprestsdk_openssl_internal)
     set(CPPREST_USES_OPENSSL ON)

Release/src/http/client/http_client.cpp

@@ -96,42 +96,57 @@ void request_context::report_exception(std::exception_ptr exceptionPtr)
     finish();
 }
 
-bool request_context::handle_content_encoding_compression()
+bool request_context::handle_compression()
 {
-    if (web::http::details::compression::stream_decompressor::is_supported() && m_http_client->client_config().request_compressed_response())
+    // If the response body is compressed we will read the encoding header and create a decompressor object which will later decompress the body
+    try
     {
-        // If the response body is compressed we will read the encoding header and create a decompressor object which will later decompress the body
-        auto&& headers = m_response.headers();
-        auto it_ce = headers.find(web::http::header_names::content_encoding);
-        if (it_ce != headers.end())
-        {
-            auto alg = web::http::details::compression::stream_decompressor::to_compression_algorithm(it_ce->second);
+        utility::string_t encoding;
+        http_headers &headers = m_response.headers();
 
-            if (alg != web::http::details::compression::compression_algorithm::invalid)
-            {
-                m_decompressor = utility::details::make_unique<web::http::details::compression::stream_decompressor>(alg);
-            }
-            else
-            {
-                report_exception(
-                    http_exception("Unsupported compression algorithm in the Content-Encoding header: "
-                        + utility::conversions::to_utf8string(it_ce->second)));
-                return false;
-            }
+        // Note that some headers, for example "Transfer-Encoding: chunked", may legitimately not produce a decompressor
+        if (m_http_client->client_config().request_compressed_response() && headers.match(web::http::header_names::content_encoding, encoding))
+        {
+            // Note that, while Transfer-Encoding (chunked only) is valid with Content-Encoding,
+            // we don't need to look for it here because winhttp de-chunks for us in that case
+            m_decompressor = compression::details::get_decompressor_from_header(encoding, compression::details::header_types::content_encoding, m_request.decompress_factories());
+        }
+        else if (!m_request.decompress_factories().empty() && headers.match(web::http::header_names::transfer_encoding, encoding))
+        {
+            m_decompressor = compression::details::get_decompressor_from_header(encoding, compression::details::header_types::transfer_encoding, m_request.decompress_factories());
         }
     }
+    catch (...)
+    {
+        report_exception(std::current_exception());
+        return false;
+    }
+
     return true;
 }
 
-utility::string_t request_context::get_accept_encoding_header() const
+utility::string_t request_context::get_compression_header() const
 {
     utility::string_t headers;
-    // Add the header needed to request a compressed response if supported on this platform and it has been specified in the config
-    if (web::http::details::compression::stream_decompressor::is_supported()
-        && m_http_client->client_config().request_compressed_response())
+
+    // Add the correct header needed to request a compressed response if supported
+    // on this platform and it has been specified in the config and/or request
+    if (m_http_client->client_config().request_compressed_response())
     {
-        headers.append(U("Accept-Encoding: "));
-        headers.append(web::http::details::compression::stream_decompressor::known_algorithms());
+        if (!m_request.decompress_factories().empty() || web::http::compression::builtin::supported())
+        {
+            // Accept-Encoding -- request Content-Encoding from the server
+            headers.append(header_names::accept_encoding + U(": "));
+            headers.append(compression::details::build_supported_header(compression::details::header_types::accept_encoding, m_request.decompress_factories()));
+            headers.append(U("\r\n"));
+        }
+    }
+    else if (!m_request.decompress_factories().empty())
+    {
+        // TE -- request Transfer-Encoding from the server
+        headers.append(header_names::connection + U(": TE\r\n") +   // Required by Section 4.3 of RFC-7230
+                       header_names::te + U(": "));
+        headers.append(compression::details::build_supported_header(compression::details::header_types::te, m_request.decompress_factories()));
         headers.append(U("\r\n"));
     }
 

Release/src/http/client/http_client_asio.cpp

@@ -850,12 +850,12 @@ public:
                 extra_headers.append(ctx->generate_basic_auth_header());
             }
 
-            extra_headers += utility::conversions::to_utf8string(ctx->get_accept_encoding_header());
+            extra_headers += utility::conversions::to_utf8string(ctx->get_compression_header());
 
             // Check user specified transfer-encoding.
             std::string transferencoding;
             if (ctx->m_request.headers().match(header_names::transfer_encoding, transferencoding) &&
-                transferencoding == "chunked")
+                boost::icontains(transferencoding, U("chunked")))
             {
                 ctx->m_needChunked = true;
             }
@@ -1394,7 +1394,7 @@ private:
 
                 if (boost::iequals(name, header_names::transfer_encoding))
                 {
-                    needChunked = boost::iequals(value, U("chunked"));
+                    needChunked = boost::icontains(value, U("chunked"));
                 }
 
                 if (boost::iequals(name, header_names::connection))
@@ -1415,7 +1415,7 @@ private:
                                                                // TCP stream - set it size_t max.
         m_response.headers().match(header_names::content_length, m_content_length);
 
-        if (!this->handle_content_encoding_compression())
+        if (!this->handle_compression())
         {
             // false indicates report_exception was called
             return;
@@ -1518,6 +1518,49 @@ private:
         }
     }
 
+    bool decompress(const uint8_t* input, size_t input_size, std::vector<uint8_t>& output)
+    {
+        // Need to guard against attempting to decompress when we're already finished or encountered an error!
+        if (input == nullptr || input_size == 0)
+        {
+            return false;
+        }
+
+        size_t processed;
+        size_t got;
+        size_t inbytes = 0;
+        size_t outbytes = 0;
+        bool done = false;
+
+        try
+        {
+            output.resize(input_size * 3);
+            do
+            {
+                if (inbytes)
+                {
+                    output.resize(output.size() + std::max(input_size, static_cast<size_t>(1024)));
+                }
+                got = m_decompressor->decompress(input + inbytes,
+                                                 input_size - inbytes,
+                                                 output.data() + outbytes,
+                                                 output.size() - outbytes,
+                                                 web::http::compression::operation_hint::has_more,
+                                                 processed,
+                                                 &done);
+                inbytes += processed;
+                outbytes += got;
+            } while (got && !done);
+            output.resize(outbytes);
+        }
+        catch (...)
+        {
+            return false;
+        }
+
+        return true;
+    }
+
     void handle_chunk(const boost::system::error_code& ec, int to_read)
     {
         if (!ec)
@@ -1550,10 +1593,11 @@ private:
                 const auto this_request = shared_from_this();
                 if (m_decompressor)
                 {
-                    auto decompressed = m_decompressor->decompress(
-                        boost::asio::buffer_cast<const uint8_t*>(m_body_buf.data()), to_read);
+                    std::vector<uint8_t> decompressed;
 
-                    if (m_decompressor->has_error())
+                    bool boo =
+                        decompress(boost::asio::buffer_cast<const uint8_t*>(m_body_buf.data()), to_read, decompressed);
+                    if (!boo)
                     {
                         report_exception(std::runtime_error("Failed to decompress the response body"));
                         return;
@@ -1574,8 +1618,7 @@ private:
                     {
                         // Move the decompressed buffer into a shared_ptr to keep it alive until putn_nocopy completes.
                         // When VS 2013 support is dropped, this should be changed to a unique_ptr plus a move capture.
-                        using web::http::details::compression::data_buffer;
-                        auto shared_decompressed = std::make_shared<data_buffer>(std::move(decompressed));
+                        auto shared_decompressed = std::make_shared<std::vector<uint8_t>>(std::move(decompressed));
 
                         writeBuffer.putn_nocopy(shared_decompressed->data(), shared_decompressed->size())
                             .then([this_request, to_read, shared_decompressed AND_CAPTURE_MEMBER_FUNCTION_POINTERS](
@@ -1670,10 +1713,11 @@ private:
 
             if (m_decompressor)
             {
-                auto decompressed =
-                    m_decompressor->decompress(boost::asio::buffer_cast<const uint8_t*>(m_body_buf.data()), read_size);
+                std::vector<uint8_t> decompressed;
 
-                if (m_decompressor->has_error())
+                bool boo =
+                    decompress(boost::asio::buffer_cast<const uint8_t*>(m_body_buf.data()), read_size, decompressed);
+                if (!boo)
                 {
                     this_request->report_exception(std::runtime_error("Failed to decompress the response body"));
                     return;
@@ -1704,8 +1748,7 @@ private:
                 {
                     // Move the decompressed buffer into a shared_ptr to keep it alive until putn_nocopy completes.
                     // When VS 2013 support is dropped, this should be changed to a unique_ptr plus a move capture.
-                    using web::http::details::compression::data_buffer;
-                    auto shared_decompressed = std::make_shared<data_buffer>(std::move(decompressed));
+                    auto shared_decompressed = std::make_shared<std::vector<uint8_t>>(std::move(decompressed));
 
                     writeBuffer.putn_nocopy(shared_decompressed->data(), shared_decompressed->size())
                         .then([this_request, read_size, shared_decompressed AND_CAPTURE_MEMBER_FUNCTION_POINTERS](
@@ -1715,7 +1758,7 @@ private:
                             {
                                 writtenSize = op.get();
                                 this_request->m_downloaded += static_cast<uint64_t>(read_size);
-                                this_request->m_body_buf.consume(writtenSize);
+                                this_request->m_body_buf.consume(read_size);
                                 this_request->async_read_until_buffersize(
                                     static_cast<size_t>(std::min(
                                         static_cast<uint64_t>(this_request->m_http_client->client_config().chunksize()),

Release/src/http/client/http_client_impl.h

@@ -72,10 +72,10 @@ public:
 
     /// <summary>Set m_decompressor based on the response headers, or call report_exception</summary>
     /// <returns>false on failure</returns>
-    bool handle_content_encoding_compression();
+    bool handle_compression();
 
     /// <summary>Append an Accept-Encoding header if requested by the http_client settings</summary>
-    utility::string_t get_accept_encoding_header() const;
+    utility::string_t get_compression_header() const;
 
     concurrency::streams::streambuf<uint8_t> _get_writebuffer();
 
@@ -95,7 +95,7 @@ public:
     // Registration for cancellation notification if enabled.
     pplx::cancellation_token_registration m_cancellationRegistration;
 
-    std::unique_ptr<web::http::details::compression::stream_decompressor> m_decompressor;
+    std::unique_ptr<web::http::compression::decompress_provider> m_decompressor;
 
 protected:
 

Release/src/http/common/http_helpers.cpp

@@ -12,27 +12,6 @@
 ****/
 
 #include "stdafx.h"
-
-// CPPREST_EXCLUDE_COMPRESSION is set if we're on a platform that supports compression but we want to explicitly disable it.
-// CPPREST_EXCLUDE_WEBSOCKETS is a flag that now essentially means "no external dependencies". TODO: Rename
-
-#if __APPLE__
-#include "TargetConditionals.h"
-#if defined(TARGET_OS_MAC)
-#if !defined(CPPREST_EXCLUDE_COMPRESSION)
-#define CPPREST_HTTP_COMPRESSION
-#endif // !defined(CPPREST_EXCLUDE_COMPRESSION)
-#endif // defined(TARGET_OS_MAC)
-#elif defined(_WIN32) && (!defined(WINAPI_FAMILY) || WINAPI_PARTITION_DESKTOP)
-#if !defined(CPPREST_EXCLUDE_WEBSOCKETS) && !defined(CPPREST_EXCLUDE_COMPRESSION)
-#define CPPREST_HTTP_COMPRESSION
-#endif // !defined(CPPREST_EXCLUDE_WEBSOCKETS) && !defined(CPPREST_EXCLUDE_COMPRESSION)
-#endif
-
-#if defined(CPPREST_HTTP_COMPRESSION)
-#include <zlib.h>
-#endif
-
 #include "internal_http_helpers.h"
 
 using namespace web;
@@ -142,309 +121,5 @@ bool validate_method(const utility::string_t& method)
     return true;
 }
 
-namespace compression
-{
-#if defined(CPPREST_HTTP_COMPRESSION)
-
-    class compression_base_impl
-    {
-    public:
-        compression_base_impl(compression_algorithm alg) : m_alg(alg), m_zLibState(Z_OK)
-        {
-            memset(&m_zLibStream, 0, sizeof(m_zLibStream));
-        }
-
-        size_t read_output(size_t input_offset, size_t available_input, size_t total_out_before, uint8_t* temp_buffer, data_buffer& output)
-        {
-            input_offset += (available_input - stream().avail_in);
-            auto out_length = stream().total_out - total_out_before;
-            output.insert(output.end(), temp_buffer, temp_buffer + out_length);
-
-            return input_offset;
-        }
-
-        bool is_complete() const
-        {
-            return state() == Z_STREAM_END;
-        }
-
-        bool has_error() const
-        {
-            return !is_complete() && state() != Z_OK;
-        }
-
-        int state() const
-        {
-            return m_zLibState;
-        }
-
-        void set_state(int state)
-        {
-            m_zLibState = state;
-        }
-
-        compression_algorithm algorithm() const
-        {
-            return m_alg;
-        }
-
-        z_stream& stream()
-        {
-            return m_zLibStream;
-        }
-
-        int to_zlib_alg(compression_algorithm alg)
-        {
-            return static_cast<int>(alg);
-        }
-
-    private:
-        const compression_algorithm m_alg;
-
-        std::atomic<int> m_zLibState{ Z_OK };
-        z_stream m_zLibStream;
-    };
-
-    class stream_decompressor::stream_decompressor_impl : public compression_base_impl
-    {
-    public:
-        stream_decompressor_impl(compression_algorithm alg) : compression_base_impl(alg)
-        {
-            set_state(inflateInit2(&stream(), to_zlib_alg(alg)));
-        }
-
-        ~stream_decompressor_impl()
-        {
-            inflateEnd(&stream());
-        }
-
-        data_buffer decompress(const uint8_t* input, size_t input_size)
-        {
-            if (input == nullptr || input_size == 0)
-            {
-                set_state(Z_BUF_ERROR);
-                return data_buffer();
-            }
-
-            // Need to guard against attempting to decompress when we're already finished or encountered an error!
-            if (is_complete() || has_error())
-            {
-                set_state(Z_STREAM_ERROR);
-                return data_buffer();
-            }
-
-            const size_t BUFFER_SIZE = 1024;
-            unsigned char temp_buffer[BUFFER_SIZE];
-
-            data_buffer output;
-            output.reserve(input_size * 3);
-
-            size_t input_offset{ 0 };
-
-            while (state() == Z_OK && input_offset < input_size)
-            {
-                auto total_out_before = stream().total_out;
-
-                auto available_input = input_size - input_offset;
-                stream().next_in = const_cast<uint8_t*>(&input[input_offset]);
-                stream().avail_in = static_cast<int>(available_input);
-                stream().next_out = temp_buffer;
-                stream().avail_out = BUFFER_SIZE;
-
-                set_state(inflate(&stream(), Z_PARTIAL_FLUSH));
-
-                if (has_error())
-                {
-                    return data_buffer();
-                }
-
-                input_offset = read_output(input_offset, available_input, total_out_before, temp_buffer, output);
-            }
-
-            return output;
-        }
-    };
-
-    class stream_compressor::stream_compressor_impl : public compression_base_impl
-    {
-    public:
-        stream_compressor_impl(compression_algorithm alg) : compression_base_impl(alg)
-        {
-            const int level = Z_DEFAULT_COMPRESSION;
-            if (alg == compression_algorithm::gzip)
-            {
-                set_state(deflateInit2(&stream(), level, Z_DEFLATED, to_zlib_alg(alg), MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY));
-            }
-            else if (alg == compression_algorithm::deflate)
-            {
-                set_state(deflateInit(&stream(), level));
-            }
-        }
-
-        web::http::details::compression::data_buffer compress(const uint8_t* input, size_t input_size, bool finish)
-        {
-            if (input == nullptr || input_size == 0)
-            {
-                set_state(Z_BUF_ERROR);
-                return data_buffer();
-            }
-
-            if (state() != Z_OK)
-            {
-                set_state(Z_STREAM_ERROR);
-                return data_buffer();
-            }
-
-            data_buffer output;
-            output.reserve(input_size);
-
-            const size_t BUFFER_SIZE = 1024;
-            uint8_t temp_buffer[BUFFER_SIZE];
-
-            size_t input_offset{ 0 };
-            auto flush = Z_NO_FLUSH;
-
-            while (flush == Z_NO_FLUSH)
-            {
-                auto total_out_before = stream().total_out;
-                auto available_input = input_size - input_offset;
-
-                if (available_input == 0)
-                {
-                    flush = finish ? Z_FINISH : Z_PARTIAL_FLUSH;
-                }
-                else
-                {
-                    stream().avail_in = static_cast<int>(available_input);
-                    stream().next_in = const_cast<uint8_t*>(&input[input_offset]);
-                }
-
-                do
-                {
-                    stream().next_out = temp_buffer;
-                    stream().avail_out = BUFFER_SIZE;
-
-                    set_state(deflate(&stream(), flush));
-
-                    if (has_error())
-                    {
-                        return data_buffer();
-                    }
-
-                    input_offset = read_output(input_offset, available_input, total_out_before, temp_buffer, output);
-
-                } while (stream().avail_out == 0);
-            }
-
-            return output;
-        }
-
-        ~stream_compressor_impl()
-        {
-            deflateEnd(&stream());
-        }
-    };
-#else // Stub impl for when compression is not supported
-
-    class compression_base_impl
-    {
-    public:
-        bool has_error() const
-        {
-            return true;
-        }
-    };
-
-    class stream_compressor::stream_compressor_impl : public compression_base_impl
-    {
-    public:
-        stream_compressor_impl(compression_algorithm) {}
-        compression::data_buffer compress(const uint8_t* data, size_t size, bool)
-        {
-            return data_buffer(data, data + size);
-        }
-    };
-
-    class stream_decompressor::stream_decompressor_impl : public compression_base_impl
-    {
-    public:
-        stream_decompressor_impl(compression_algorithm) {}
-        compression::data_buffer decompress(const uint8_t* data, size_t size) 
-        {
-            return data_buffer(data, data + size);
-        }
-    };
-#endif
-
-    bool __cdecl stream_decompressor::is_supported()
-    {
-#if !defined(CPPREST_HTTP_COMPRESSION)
-    return false;
-#else
-    return true;
-#endif
-    }
-
-    stream_decompressor::stream_decompressor(compression_algorithm alg)
-        : m_pimpl(std::make_shared<stream_decompressor::stream_decompressor_impl>(alg))
-    {
-    }
-
-    compression::data_buffer stream_decompressor::decompress(const data_buffer& input)
-    {
-        if (input.empty())
-        {
-            return data_buffer();
-        }
-
-        return m_pimpl->decompress(&input[0], input.size());
-    }
-
-    web::http::details::compression::data_buffer stream_decompressor::decompress(const uint8_t* input, size_t input_size)
-    {
-        return m_pimpl->decompress(input, input_size);
-    }
-
-    bool stream_decompressor::has_error() const
-    {
-        return m_pimpl->has_error();
-    }
-
-    bool __cdecl stream_compressor::is_supported()
-    {
-#if !defined(CPPREST_HTTP_COMPRESSION)
-        return false;
-#else
-        return true;
-#endif
-    }
-
-    stream_compressor::stream_compressor(compression_algorithm alg)
-        : m_pimpl(std::make_shared<stream_compressor::stream_compressor_impl>(alg))
-    {
-
-    }
-
-    compression::data_buffer stream_compressor::compress(const data_buffer& input, bool finish)
-    {
-        if (input.empty())
-        {
-            return compression::data_buffer();
-        }
-
-        return m_pimpl->compress(&input[0], input.size(), finish);
-    }
-
-    web::http::details::compression::data_buffer stream_compressor::compress(const uint8_t* input, size_t input_size, bool finish)
-    {
-        return m_pimpl->compress(input, input_size, finish);
-    }
-    
-    bool stream_compressor::has_error() const
-    {
-        return m_pimpl->has_error();
-    }
-
-} // namespace compression
 } // namespace details
 }} // namespace web::http

Release/src/http/common/http_msg.cpp

@@ -310,36 +310,95 @@ void http_msg_base::_prepare_to_receive_data()
     // or media (like file) that the user can read from...
 }
 
-size_t http_msg_base::_get_content_length()
+size_t http_msg_base::_get_stream_length()
+{
+    auto &stream = instream();
+
+    if (stream.can_seek())
+    {
+        auto offset = stream.tell();
+        auto end = stream.seek(0, std::ios_base::end);
+        stream.seek(offset);
+        return static_cast<utility::size64_t>(end - offset);
+    }
+
+    return std::numeric_limits<size_t>::max();
+}
+
+size_t http_msg_base::_get_content_length(bool honor_compression)
 {
     // An invalid response_stream indicates that there is no body
     if ((bool)instream())
     {
-        size_t content_length = 0;
+        size_t content_length;
         utility::string_t transfer_encoding;
 
-        bool has_cnt_length = headers().match(header_names::content_length, content_length);
-        bool has_xfr_encode = headers().match(header_names::transfer_encoding, transfer_encoding);
-
-        if (has_xfr_encode)
+        if (headers().match(header_names::transfer_encoding, transfer_encoding))
         {
+            // Transfer encoding is set; it trumps any content length that may or may not be present
+            if (honor_compression && m_compressor)
+            {
+                http::http_headers tmp;
+
+                // Build a header for comparison with the existing one
+                tmp.add(header_names::transfer_encoding, m_compressor->algorithm());
+                tmp.add(header_names::transfer_encoding, _XPLATSTR("chunked"));
+
+                if (!utility::details::str_iequal(transfer_encoding, tmp[header_names::transfer_encoding]))
+                {
+                    // Some external entity added this header, and it doesn't match our
+                    // expectations; bail out, since the caller's intentions are not clear
+                    throw http_exception("Transfer-Encoding header is internally managed when compressing");
+                }
+            }
+
             return std::numeric_limits<size_t>::max();
         }
 
-        if (has_cnt_length)
+        if (honor_compression && m_compressor)
         {
+            // A compressor is set; this implies transfer encoding, since we don't know the compressed length
+            // up front for content encoding.  We return the uncompressed length if we can figure it out.
+            headers().add(header_names::transfer_encoding, m_compressor->algorithm());
+            headers().add(header_names::transfer_encoding, _XPLATSTR("chunked"));
+            return std::numeric_limits<size_t>::max();
+        }
+
+        if (headers().match(header_names::content_length, content_length))
+        {
+            // An explicit content length is set; trust it, since we
+            // may not be required to send the stream's entire contents
             return content_length;
         }
 
-        // Neither is set. Assume transfer-encoding for now (until we have the ability to determine
-        // the length of the stream).
+        content_length = _get_stream_length();
+        if (content_length != std::numeric_limits<size_t>::max())
+        {
+            // The content length wasn't explcitly set, but we figured it out;
+            // use it, since sending this way is more efficient than chunking
+            headers().add(header_names::content_length, content_length);
+            return content_length;
+        }
+
+        // We don't know the content length; we'll chunk the stream
         headers().add(header_names::transfer_encoding, _XPLATSTR("chunked"));
         return std::numeric_limits<size_t>::max();
     }
 
+    // There is no content
     return 0;
 }
 
+size_t http_msg_base::_get_content_length_and_set_compression()
+{
+    return _get_content_length(true);
+}
+
+size_t http_msg_base::_get_content_length()
+{
+    return _get_content_length(false);
+}
+
 // Helper function to inline continuation if possible.
 struct inline_continuation
 {
@@ -1041,6 +1100,11 @@ details::_http_request::_http_request(std::unique_ptr<http::details::_http_serve
 {
 }
 
+void http_request::set_decompress_factories()
+{
+    return _m_impl->set_decompress_factories(compression::details::builtin::get_decompress_factories());
+}
+
 const http_version http_versions::HTTP_0_9 = { 0, 9 };
 const http_version http_versions::HTTP_1_0 = { 1, 0 };
 const http_version http_versions::HTTP_1_1 = { 1, 1 };

Release/src/http/common/internal_http_helpers.h

@@ -31,3 +31,19 @@ void trim_whitespace(std::basic_string<Char> &str)
 bool validate_method(const utility::string_t& method);
 
 }}}
+
+namespace web { namespace http { namespace compression {
+
+class compression::decompress_factory;
+
+namespace details { namespace builtin {
+
+/// <summary>
+/// Helper function to get the set of built-in decompress factories
+/// </summary>
+
+const std::vector<std::shared_ptr<web::http::compression::decompress_factory>> get_decompress_factories();
+
+}}
+
+}}}

Release/src/http/listener/http_server_httpsys.cpp

@@ -85,7 +85,7 @@ static utility::string_t HttpServerAPIKnownHeaders[] =
     U("Proxy-Authorization"),
     U("Referer"),
     U("Range"),
-    U("Te"),
+    U("TE"),
     U("Translate"),
     U("User-Agent"),
     U("Request-Maximum"),
@@ -539,6 +539,7 @@ void windows_request_context::read_headers_io_completion(DWORD error_code, DWORD
     }
     else
     {
+        utility::string_t header;
         std::string badRequestMsg;
         try
         {
@@ -557,6 +558,66 @@ void windows_request_context::read_headers_io_completion(DWORD error_code, DWORD
         m_msg.set_method(parse_request_method(m_request));
         parse_http_headers(m_request->Headers, m_msg.headers());
 
+        // See if we need to compress or decompress the incoming request body, and if so, prepare for it
+        try
+        {
+            if (m_msg.headers().match(header_names::transfer_encoding, header))
+            {
+                try
+                {
+                    m_decompressor = http::compression::details::get_decompressor_from_header(header, http::compression::details::header_types::transfer_encoding);
+                }
+                catch (http_exception &e)
+                {
+                    if (e.error_code().value() != status_codes::NotImplemented)
+                    {
+                        // Something is wrong with the header; we'll fail here
+                        throw;
+                    }
+                    // We could not find a decompressor; we'll see if the user's handler adds one later
+                    m_decompress_header_type = http::compression::details::header_types::transfer_encoding;
+                    m_decompress_header = std::move(header);
+                }
+            }
+            else if (m_msg.headers().match(header_names::content_encoding, header))
+            {
+                try
+                {
+                    m_decompressor = http::compression::details::get_decompressor_from_header(header, http::compression::details::header_types::content_encoding);
+                }
+                catch (http_exception &e)
+                {
+                    if (e.error_code().value() != status_codes::UnsupportedMediaType)
+                    {
+                        // Something is wrong with the header; we'll fail here
+                        throw;
+                    }
+                    // We could not find a decompressor; we'll see if the user's handler adds one later
+                    m_decompress_header_type = http::compression::details::header_types::content_encoding;
+                    m_decompress_header = std::move(header);
+                }
+            }
+            else if (m_msg.headers().match(header_names::te, header))
+            {
+                // Note that init_response_headers throws away m_msg, so we need to set our compressor here.  If
+                // the header contains all unsupported algorithms, it's not an error -- we just won't compress
+                m_compressor = http::compression::details::get_compressor_from_header(header, http::compression::details::header_types::te);
+            }
+            else if (m_msg.headers().match(header_names::accept_encoding, header))
+            {
+                // This would require pre-compression of the input stream, since we MUST send Content-Length, so we'll (legally) ignore it
+                //m_compressor = http::compression::details::get_compressor_from_header(header, http::compression::details::header_types:accept_encoding);
+            }
+        }
+        catch (http_exception &e)
+        {
+            if (badRequestMsg.empty())
+            {
+                // Respond with a reasonable message
+                badRequestMsg = e.what();
+            }
+        }
+
         m_msg._get_impl()->_set_http_version({ (uint8_t)m_request->Version.MajorVersion, (uint8_t)m_request->Version.MinorVersion });
 
         // Retrieve the remote IP address
@@ -601,12 +662,24 @@ void windows_request_context::read_headers_io_completion(DWORD error_code, DWORD
 void windows_request_context::read_request_body_chunk()
 {
     auto *pServer = static_cast<http_windows_server *>(http_server_api::server_api());
+    PVOID body;
 
     // The read_body_io_completion callback function
     m_overlapped.set_http_io_completion([this](DWORD error, DWORD nBytes){ read_body_io_completion(error, nBytes);});
 
     auto request_body_buf = m_msg._get_impl()->outstream().streambuf();
-    auto body = request_body_buf.alloc(CHUNK_SIZE);
+    if (!m_decompressor)
+    {
+        body = request_body_buf.alloc(CHUNK_SIZE);
+    }
+    else
+    {
+        if (m_compress_buffer.size() < CHUNK_SIZE)
+        {
+            m_compress_buffer.resize(CHUNK_SIZE);
+        }
+        body = m_compress_buffer.data();
+    }
 
     // Once we allow users to set the output stream the following assert could fail.
     // At that time we would need compensation code that would allocate a buffer from the heap instead.
@@ -626,7 +699,10 @@ void windows_request_context::read_request_body_chunk()
     {
         // There was no more data to read.
         CancelThreadpoolIo(pServer->m_threadpool_io);
-        request_body_buf.commit(0);
+        if (!m_decompressor)
+        {
+            request_body_buf.commit(0);
+        }
         if(error_code == ERROR_HANDLE_EOF)
         {
             m_msg._get_impl()->_complete(request_body_buf.in_avail());
@@ -647,17 +723,49 @@ void windows_request_context::read_body_io_completion(DWORD error_code, DWORD by
 
     if (error_code == NO_ERROR)
     {
-        request_body_buf.commit(bytes_read);
+        if (!m_decompressor)
+        {
+            request_body_buf.commit(bytes_read);
+        }
+        else
+        {
+            size_t got;
+            size_t used;
+            size_t total_used = 0;
+
+            do
+            {
+                auto body = request_body_buf.alloc(CHUNK_SIZE);
+                try
+                {
+                    got = m_decompressor->decompress(m_compress_buffer.data()+total_used, bytes_read-total_used, body, CHUNK_SIZE, http::compression::operation_hint::has_more, used, NULL);
+                }
+                catch (...)
+                {
+                    request_body_buf.commit(0);
+                    m_msg._get_impl()->_complete(0, std::current_exception());
+                    return;
+                }
+                request_body_buf.commit(got);
+                total_used += used;
+            } while (total_used != bytes_read);
+        }
         read_request_body_chunk();
     }
     else if (error_code == ERROR_HANDLE_EOF)
     {
-        request_body_buf.commit(0);
+        if (!m_decompressor)
+        {
+            request_body_buf.commit(0);
+        }
         m_msg._get_impl()->_complete(request_body_buf.in_avail());
     }
     else
     {
-        request_body_buf.commit(0);
+        if (!m_decompressor)
+        {
+            request_body_buf.commit(0);
+        }
         m_msg._get_impl()->_complete(0, std::make_exception_ptr(http_exception(error_code)));
     }
 }
@@ -800,8 +908,53 @@ void windows_request_context::async_process_response()
     const std::string reason = utf16_to_utf8(m_response.reason_phrase());
     win_api_response.pReason = reason.c_str();
     win_api_response.ReasonLength = (USHORT)reason.size();
+    size_t content_length;
 
-    size_t content_length = m_response._get_impl()->_get_content_length();
+    if (m_compressor || m_response._get_impl()->compressor())
+    {
+        if (m_response.headers().has(header_names::content_length))
+        {
+            // Content-Length should not be sent with Transfer-Encoding
+            m_response.headers().remove(header_names::content_length);
+        }
+        if (!m_response._get_impl()->compressor())
+        {
+            // Temporarily move the compressor to the reponse, so _get_content_length() will honor it
+            m_response._get_impl()->set_compressor(std::move(m_compressor));
+        } // else one was already set from a callback, and we'll (blindly) use it
+        content_length = m_response._get_impl()->_get_content_length_and_set_compression();
+        m_compressor = std::move(m_response._get_impl()->compressor());
+        m_response._get_impl()->set_compressor(nullptr);
+    }
+    else
+    {
+        if (!m_decompress_header.empty())
+        {
+            auto factories = m_response._get_impl()->decompress_factories();
+            try
+            {
+                m_decompressor = http::compression::details::get_decompressor_from_header(m_decompress_header, m_decompress_header_type, factories);
+                m_decompress_header.clear();
+                if (!m_decompressor)
+                {
+                    http::status_code code = http::status_codes::NotImplemented;
+                    if (m_decompress_header_type == http::compression::details::header_types::content_encoding)
+                    {
+                        code = status_codes::UnsupportedMediaType;
+                    }
+                    throw http_exception(code);
+                }
+            }
+            catch (http_exception &e)
+            {
+                // No matching decompressor was supplied via callback
+                CancelThreadpoolIo(pServer->m_threadpool_io);
+                cancel_request(std::make_exception_ptr(e));
+                return;
+            }
+        }
+        content_length = m_response._get_impl()->_get_content_length();
+    }
 
     m_headers = std::unique_ptr<HTTP_UNKNOWN_HEADER []>(new HTTP_UNKNOWN_HEADER[msl::safeint3::SafeInt<size_t>(m_response.headers().size())]);
     m_headers_buffer.resize(msl::safeint3::SafeInt<size_t>(m_response.headers().size()) * 2);
@@ -822,8 +975,6 @@ void windows_request_context::async_process_response()
     // Send response callback function
     m_overlapped.set_http_io_completion([this](DWORD error, DWORD nBytes){ send_response_io_completion(error, nBytes);});
 
-    m_remaining_to_write = content_length;
-
     // Figure out how to send the entity body of the message.
     if (content_length == 0)
     {
@@ -854,6 +1005,12 @@ void windows_request_context::async_process_response()
     _ASSERTE(content_length > 0);
     m_sending_in_chunks = (content_length != std::numeric_limits<size_t>::max());
     m_transfer_encoding = (content_length == std::numeric_limits<size_t>::max());
+    m_remaining_to_write = content_length;
+    if (content_length == std::numeric_limits<size_t>::max())
+    {
+        // Attempt to figure out the remaining length of the input stream
+        m_remaining_to_write = m_response._get_impl()->_get_stream_length();
+    }
 
     StartThreadpoolIo(pServer->m_threadpool_io);
     const unsigned long error_code = HttpSendHttpResponse(
@@ -901,11 +1058,12 @@ void windows_request_context::transmit_body()
         return;
     }
 
+    msl::safeint3::SafeInt<size_t> safeCount = m_remaining_to_write;
+    size_t next_chunk_size = safeCount.Min(CHUNK_SIZE);
+
     // In both cases here we could perform optimizations to try and use acquire on the streams to avoid an extra copy.
     if ( m_sending_in_chunks )
     {
-        msl::safeint3::SafeInt<size_t> safeCount = m_remaining_to_write;
-        size_t next_chunk_size = safeCount.Min(CHUNK_SIZE);
         m_body_data.resize(CHUNK_SIZE);
 
         streams::rawptr_buffer<unsigned char> buf(&m_body_data[0], next_chunk_size);
@@ -937,33 +1095,109 @@ void windows_request_context::transmit_body()
     else
     {
         // We're transfer-encoding...
-        const size_t body_data_length = CHUNK_SIZE+http::details::chunked_encoding::additional_encoding_space;
-        m_body_data.resize(body_data_length);
-
-        streams::rawptr_buffer<unsigned char> buf(&m_body_data[http::details::chunked_encoding::data_offset], body_data_length);
-
-        m_response.body().read(buf, CHUNK_SIZE).then([this, body_data_length](pplx::task<size_t> op)
+        if (m_compressor)
         {
-            size_t bytes_read = 0;
+            // ...and compressing.  For simplicity, we allocate a buffer that's "too large to fail" while compressing.
+            const size_t body_data_length = 2*CHUNK_SIZE + http::details::chunked_encoding::additional_encoding_space;
+            m_body_data.resize(body_data_length);
 
-            // If an exception occurs surface the error to user on the server side
-            // and cancel the request so the client sees the error.
-            try
+            // We'll read into a temporary buffer before compressing
+            if (m_compress_buffer.capacity() < next_chunk_size)
             {
-                bytes_read = op.get();
-            } catch (...)
-            {
-                cancel_request(std::current_exception());
-                return;
+                m_compress_buffer.reserve(next_chunk_size);
             }
 
-            // Check whether this is the last one to send...
-            m_transfer_encoding = (bytes_read > 0);
-            size_t offset = http::details::chunked_encoding::add_chunked_delimiters(&m_body_data[0], body_data_length, bytes_read);
+            streams::rawptr_buffer<unsigned char> buf(m_compress_buffer.data(), next_chunk_size);
 
-            auto data_length = bytes_read + (http::details::chunked_encoding::additional_encoding_space-offset);
-            send_entity_body(&m_body_data[offset], data_length);
-        });
+            m_response.body().read(buf, next_chunk_size).then([this, body_data_length](pplx::task<size_t> op)
+            {
+                size_t bytes_read = 0;
+
+                // If an exception occurs surface the error to user on the server side
+                // and cancel the request so the client sees the error.
+                try
+                {
+                    bytes_read = op.get();
+                }
+                catch (...)
+                {
+                    cancel_request(std::current_exception());
+                    return;
+                }
+                _ASSERTE(bytes_read >= 0);
+
+                // Compress this chunk; if we read no data, allow the compressor to finalize its stream
+                http::compression::operation_hint hint = http::compression::operation_hint::has_more;
+                if (!bytes_read)
+                {
+                    hint = http::compression::operation_hint::is_last;
+                }
+                m_compressor->compress(m_compress_buffer.data(), bytes_read, &m_body_data[http::details::chunked_encoding::data_offset], body_data_length, hint)
+                    .then([this, bytes_read, body_data_length](pplx::task<http::compression::operation_result> op)
+                {
+                    http::compression::operation_result r;
+
+                    try
+                    {
+                        r = op.get();
+                    }
+                    catch (...)
+                    {
+                        cancel_request(std::current_exception());
+                        return;
+                    }
+
+                    if (r.input_bytes_processed != bytes_read ||
+                        r.output_bytes_produced == body_data_length - http::details::chunked_encoding::additional_encoding_space ||
+                        r.done != !bytes_read)
+                    {
+                        // We chose our parameters so that compression should
+                        // never overflow body_data_length; fail if it does
+                        cancel_request(std::make_exception_ptr(std::exception("Compressed data exceeds internal buffer size.")));
+                        return;
+                    }
+
+                    // Check whether this is the last one to send; note that this is a
+                    // few lines of near-duplicate code with the non-compression path
+                    _ASSERTE(bytes_read <= m_remaining_to_write);
+                    m_remaining_to_write -= bytes_read;
+                    m_transfer_encoding = (r.output_bytes_produced > 0);
+                    size_t offset = http::details::chunked_encoding::add_chunked_delimiters(&m_body_data[0], body_data_length, r.output_bytes_produced);
+                    send_entity_body(&m_body_data[offset], r.output_bytes_produced + http::details::chunked_encoding::additional_encoding_space - offset);
+                });
+            });
+        }
+        else
+        {
+            const size_t body_data_length = CHUNK_SIZE + http::details::chunked_encoding::additional_encoding_space;
+            m_body_data.resize(body_data_length);
+
+            streams::rawptr_buffer<unsigned char> buf(&m_body_data[http::details::chunked_encoding::data_offset], body_data_length);
+
+            m_response.body().read(buf, next_chunk_size).then([this, body_data_length](pplx::task<size_t> op)
+            {
+                size_t bytes_read = 0;
+
+                // If an exception occurs surface the error to user on the server side
+                // and cancel the request so the client sees the error.
+                try
+                {
+                    bytes_read = op.get();
+                }
+                catch (...)
+                {
+                    cancel_request(std::current_exception());
+                    return;
+                }
+
+                // Check whether this is the last one to send...
+                m_transfer_encoding = (bytes_read > 0);
+                size_t offset = http::details::chunked_encoding::add_chunked_delimiters(&m_body_data[0], body_data_length, bytes_read);
+
+                auto data_length = bytes_read + (http::details::chunked_encoding::additional_encoding_space - offset);
+                send_entity_body(&m_body_data[offset], data_length);
+            });
+        }
     }
 }
 

Release/src/http/listener/http_server_httpsys.h

@@ -143,6 +143,13 @@ struct windows_request_context : http::details::_http_server_context
     http_response m_response;
 
     std::exception_ptr m_except_ptr;
+
+    std::vector<uint8_t> m_compress_buffer;
+    std::unique_ptr<web::http::compression::compress_provider> m_compressor;
+    std::unique_ptr<web::http::compression::decompress_provider> m_decompressor;
+    utility::string_t m_decompress_header;
+    http::compression::details::header_types m_decompress_header_type;
+
 private:
     windows_request_context(const windows_request_context &);
     windows_request_context& operator=(const windows_request_context &);

Release/tests/functional/http/client/CMakeLists.txt

@@ -21,6 +21,7 @@ set(SOURCES
   status_code_reason_phrase_tests.cpp
   to_string_tests.cpp
   http_client_fuzz_tests.cpp
+  compression_tests.cpp
   stdafx.cpp
 )
 

Release/tests/functional/http/client/outside_tests.cpp

@@ -58,7 +58,7 @@ TEST_FIXTURE(uri_address, outside_cnn_dot_com)
 
 TEST_FIXTURE(uri_address, outside_wikipedia_compressed_http_response)
 {
-    if (web::http::details::compression::stream_decompressor::is_supported() == false)
+    if (web::http::compression::builtin::supported() == false)
     {
         // On platforms which do not support compressed http, nothing to check.
         return;

Release/tests/functional/http/client/request_helper_tests.cpp

@@ -45,7 +45,7 @@ TEST_FIXTURE(uri_address, do_not_fail_on_content_encoding_when_not_requested)
 
 TEST_FIXTURE(uri_address, fail_on_content_encoding_if_unsupported)
 {
-    if (web::http::details::compression::stream_compressor::is_supported())
+    if (web::http::compression::builtin::supported())
     {
         test_http_server::scoped_server scoped(m_uri);
         auto& server = *scoped.server();
@@ -63,7 +63,7 @@ TEST_FIXTURE(uri_address, fail_on_content_encoding_if_unsupported)
 
 TEST_FIXTURE(uri_address, send_accept_encoding)
 {
-    if (web::http::details::compression::stream_compressor::is_supported())
+    if (web::http::compression::builtin::supported())
     {
         test_http_server::scoped_server scoped(m_uri);
         auto& server = *scoped.server();
@@ -93,7 +93,11 @@ TEST_FIXTURE(uri_address, do_not_send_accept_encoding)
     std::atomic<bool> found_accept_encoding(true);
 
     server.next_request().then([&found_accept_encoding](test_request *p_request) {
-        found_accept_encoding = p_request->m_headers.find(header_names::accept_encoding) != p_request->m_headers.end();
+        utility::string_t header;
+
+        // On Windows, someone along the way (not us!) adds "Accept-Encoding: peerdist"
+        found_accept_encoding =
+            p_request->match_header(header_names::accept_encoding, header) && header != _XPLATSTR("peerdist");
         p_request->reply(200, U("OK"));
     });
 
@@ -102,55 +106,6 @@ TEST_FIXTURE(uri_address, do_not_send_accept_encoding)
     VERIFY_IS_FALSE(found_accept_encoding);
 }
 
-TEST_FIXTURE(uri_address, compress_and_decompress)
-{
-    if (web::http::details::compression::stream_compressor::is_supported())
-    {
-        auto compress_and_decompress = [](web::http::details::compression::compression_algorithm alg)
-        {
-            auto compressor = std::make_shared<web::http::details::compression::stream_compressor>(alg);
-            auto decompressor = std::make_shared<web::http::details::compression::stream_decompressor>(alg);
-
-            const size_t buffer_size = 100;
-            const size_t split_pos = buffer_size / 2;
-
-            web::http::details::compression::data_buffer input_buffer;
-            input_buffer.reserve(buffer_size);
-
-            for (size_t i = 0; i < buffer_size; ++i)
-            {
-                input_buffer.push_back(static_cast<uint8_t>(i));
-            }
-
-            web::http::details::compression::data_buffer buffer1(input_buffer.begin(), input_buffer.begin() + split_pos);
-            web::http::details::compression::data_buffer buffer2(input_buffer.begin() + split_pos, input_buffer.end());
-
-            auto compressed_data1 = compressor->compress(buffer1, false);
-            VERIFY_IS_FALSE(compressed_data1.empty());
-            VERIFY_IS_FALSE(compressor->has_error());
-
-            auto compressed_data2 = compressor->compress(buffer2, true);
-            VERIFY_IS_FALSE(compressed_data2.empty());
-            VERIFY_IS_FALSE(compressor->has_error());
-
-            auto decompressed_data1 = decompressor->decompress(compressed_data1);
-            VERIFY_IS_FALSE(decompressed_data1.empty());
-            VERIFY_IS_FALSE(decompressor->has_error());
-
-            auto decompressed_data2 = decompressor->decompress(compressed_data2);
-            VERIFY_IS_FALSE(decompressed_data2.empty());
-            VERIFY_IS_FALSE(decompressor->has_error());
-
-            decompressed_data1.insert(decompressed_data1.end(), decompressed_data2.begin(), decompressed_data2.end());
-
-            VERIFY_ARE_EQUAL(input_buffer, decompressed_data1);
-        };
-
-        compress_and_decompress(web::http::details::compression::compression_algorithm::gzip);
-        compress_and_decompress(web::http::details::compression::compression_algorithm::deflate);
-    }
-}
-
 TEST_FIXTURE(uri_address, non_rvalue_bodies)
 {
     test_http_server::scoped_server scoped(m_uri);

Release/tests/functional/http/utilities/include/test_http_server.h

@@ -46,6 +46,15 @@ public:
         return reply_impl(status_code, reason_phrase, headers, (void *)&data[0], data.size() * sizeof(utf8char));
     }
 
+    unsigned long reply(
+        const unsigned short status_code,
+        const utility::string_t &reason_phrase,
+        const std::map<utility::string_t, utility::string_t> &headers,
+        const std::vector<uint8_t> &data)
+    {
+        return reply_impl(status_code, reason_phrase, headers, (void *)&data[0], data.size());
+    }
+
     unsigned long reply(
         const unsigned short status_code,
         const utility::string_t &reason_phrase,
@@ -60,20 +69,12 @@ public:
     bool match_header(const utility::string_t & header_name, T & header_value)
     {
         auto iter = m_headers.find(header_name);
-        if (iter != m_headers.end())
-        {
-            utility::istringstream_t iss(iter->second);
-            iss >> header_value;
-            if (iss.fail() || !iss.eof())
-            {
-                return false;
-            }
-            return true;
-        }
-        else
-        {
-            return false;
-        }
+        if (iter == m_headers.end())
+         {
+             return false;
+         }
+
+        return web::http::details::bind_impl(iter->second, header_value) || iter->second.empty();
     }
 
     // Request data.

Release/tests/functional/http/utilities/test_http_server.cpp

@@ -160,7 +160,7 @@ static utility::string_t HttpServerAPIKnownHeaders[] =
     U("Proxy-Authorization"),
     U("Referer"),
     U("Range"),
-    U("Te"),
+    U("TE"),
     U("Translate"),
     U("User-Agent"),
     U("Request-Maximum"),
@@ -345,7 +345,7 @@ public:
 
         utility::string_t transfer_encoding;
         const bool has_transfer_encoding = p_test_request->match_header(U("Transfer-Encoding"), transfer_encoding);
-        if (has_transfer_encoding && transfer_encoding == U("chunked"))
+        if (has_transfer_encoding && transfer_encoding.find(U("chunked")) != std::string::npos)
         {
             content_length = 0;
             char buf[4096];