Enable Gzip Response Decompression (#815)

* decompression WIP

* comments...custom write functions

* adding test file

* attempting to push

* switch to main

* preparing to push

* wip

* Adding test json

* Revert "wip"

This reverts commit 723cebe905.

* Remove TestContent-2.json

* removing comments

* removing comments

* remove comments

* renaming HCHttpCallSetCompressedResponse

* attempting to fix build

* attempt to fix iOS build

* adding logic to handle custom write callbacks

* Minor Edits

* Minor Edits

* address nit

* removing Temporary callback apis, renaming callback holder fields

* moving new apis to bottom of .exp and .def files

* removing newlines

* removing spaces

* adding comment to httpcall.h

* invoking custom response callback

* adding newline brackets

* renaming api, fixing build

* removing secret key

* minor edit

* update custom callback reset comments

* added test for custom write flow

* removing my title secret key

* minor edit

* fixing build

* fixing build, adding custom write function wrapper

* minor edits
This commit is contained in:
Fernando C 2024-05-01 16:29:11 -07:00 коммит произвёл GitHub
Родитель 80b7da9edd
Коммит 7323b4e707
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: B5690EEEBB952194
11 изменённых файлов: 239 добавлений и 42 удалений

Просмотреть файл

@ -110,3 +110,4 @@ EXPORTS
HCWebSocketSetProxyUri
HCWinHttpResume
HCWinHttpSuspend
HCHttpCallResponseSetGzipCompressed

Просмотреть файл

@ -130,4 +130,5 @@ EXPORTS
XTaskQueueSubmitDelayedCallback
XTaskQueueTerminate
XTaskQueueUnregisterMonitor
XTaskQueueUnregisterWaiter
XTaskQueueUnregisterWaiter
HCHttpCallResponseSetGzipCompressed

Просмотреть файл

@ -398,6 +398,18 @@ STDAPI HCHttpCallRequestEnableGzipCompression(
_In_ HCCompressionLevel level
) noexcept;
/// <summary>
/// Enable GZIP compression on the expected response.
/// </summary>
/// <param name="call">The handle of the HTTP call.</param>
/// <param name="level">Boolean denoting whether a compressed response is expected.</param>
/// <returns>Result code for this API operation. Possible values are S_OK, E_INVALIDARG, or E_HC_NOT_INITIALISED.</returns>
/// <remarks>This must be called prior to calling HCHttpCallPerformAsync.</remarks>
STDAPI HCHttpCallResponseSetGzipCompressed(
_In_ HCCallHandle call,
_In_ bool compress
) noexcept;
/// <summary>
/// The callback definition used by an HTTP call to read the request body. This callback will be invoked
/// on an unspecified background thread which is platform dependent.

Просмотреть файл

@ -176,19 +176,45 @@ struct SampleHttpCallAsyncContext
HCCallHandle call;
bool isJson;
std::string filePath;
std::vector<uint8_t> response;
bool isCustom;
};
void DoHttpCall(std::string url, std::string requestBody, bool isJson, std::string filePath, bool enableGzipCompression)
HRESULT CustomResponseBodyWrite(HCCallHandle call, const uint8_t* source, size_t bytesAvailable, void* context)
{
SampleHttpCallAsyncContext* customContext = static_cast<SampleHttpCallAsyncContext*> (context);
customContext->response.insert(customContext->response.end(), source, source + bytesAvailable);
return S_OK;
}
void DoHttpCall(std::string url, std::string requestBody, bool isJson, std::string filePath, bool enableGzipCompression, bool enableGzipResponseCompression, bool customWrite)
{
std::string method = "GET";
bool retryAllowed = true;
std::vector<std::vector<std::string>> headers;
std::vector< std::string > header;
if (enableGzipResponseCompression)
{
method = "POST";
header.push_back("X-SecretKey");
header.push_back("");
headers.push_back(header);
header.clear();
header.push_back("Accept-Encoding");
header.push_back("application/gzip");
headers.push_back(header);
header.clear();
header.push_back("Content-Type");
header.push_back("application/json");
headers.push_back(header);
}
header.clear();
header.push_back("TestHeader");
header.push_back("1.0");
headers.push_back(header);
HCCallHandle call = nullptr;
@ -197,7 +223,12 @@ void DoHttpCall(std::string url, std::string requestBody, bool isJson, std::stri
HCHttpCallRequestSetRequestBodyString(call, requestBody.c_str());
HCHttpCallRequestSetRetryAllowed(call, retryAllowed);
if (enableGzipCompression)
if (enableGzipResponseCompression)
{
HCHttpCallResponseSetGzipCompressed(call, true);
}
if (enableGzipCompression)
{
HCHttpCallRequestEnableGzipCompression(call, HCCompressionLevel::Medium);
}
@ -211,11 +242,21 @@ void DoHttpCall(std::string url, std::string requestBody, bool isJson, std::stri
printf_s("Calling %s %s\r\n", method.c_str(), url.c_str());
SampleHttpCallAsyncContext* hcContext = new SampleHttpCallAsyncContext{ call, isJson, filePath };
std::vector<uint8_t> buffer;
SampleHttpCallAsyncContext* hcContext = new SampleHttpCallAsyncContext{ call, isJson, filePath, buffer, customWrite};
XAsyncBlock* asyncBlock = new XAsyncBlock;
ZeroMemory(asyncBlock, sizeof(XAsyncBlock));
asyncBlock->context = hcContext;
asyncBlock->queue = g_queue;
if (customWrite)
{
HCHttpCallResponseBodyWriteFunction customWriteWrapper = [](HCCallHandle call, const uint8_t* source, size_t bytesAvailable, void* context) -> HRESULT
{
return CustomResponseBodyWrite(call, source, bytesAvailable, context);
};
HCHttpCallResponseSetResponseBodyWriteFunction(call, customWriteWrapper, asyncBlock->context);
}
asyncBlock->callback = [](XAsyncBlock* asyncBlock)
{
const char* str;
@ -229,7 +270,9 @@ void DoHttpCall(std::string url, std::string requestBody, bool isJson, std::stri
HCCallHandle call = hcContext->call;
bool isJson = hcContext->isJson;
std::string filePath = hcContext->filePath;
std::vector<uint8_t> readBuffer = hcContext->response;
readBuffer.push_back('\0');
bool customWriteUsed = hcContext->isCustom;
HRESULT hr = XAsyncGetStatus(asyncBlock, false);
if (FAILED(hr))
{
@ -241,24 +284,27 @@ void DoHttpCall(std::string url, std::string requestBody, bool isJson, std::stri
HCHttpCallResponseGetNetworkErrorCode(call, &networkErrorCode, &platErrCode);
HCHttpCallResponseGetStatusCode(call, &statusCode);
HCHttpCallResponseGetResponseString(call, &str);
if (str != nullptr) responseString = str;
std::vector<std::vector<std::string>> headers = ExtractAllHeaders(call);
if (!isJson)
if (!customWriteUsed)
{
size_t bufferSize = 0;
HCHttpCallResponseGetResponseBodyBytesSize(call, &bufferSize);
uint8_t* buffer = new uint8_t[bufferSize];
size_t bufferUsed = 0;
HCHttpCallResponseGetResponseBodyBytes(call, bufferSize, buffer, &bufferUsed);
HANDLE hFile = CreateFileA(filePath.c_str(), GENERIC_WRITE, 0, NULL, CREATE_NEW, FILE_ATTRIBUTE_NORMAL, NULL);
DWORD bufferWritten = 0;
WriteFile(hFile, buffer, (DWORD)bufferUsed, &bufferWritten, NULL);
CloseHandle(hFile);
delete[] buffer;
HCHttpCallResponseGetResponseString(call, &str);
if (str != nullptr) responseString = str;
if (!isJson)
{
size_t bufferSize = 0;
HCHttpCallResponseGetResponseBodyBytesSize(call, &bufferSize);
uint8_t* buffer = new uint8_t[bufferSize];
size_t bufferUsed = 0;
HCHttpCallResponseGetResponseBodyBytes(call, bufferSize, buffer, &bufferUsed);
HANDLE hFile = CreateFileA(filePath.c_str(), GENERIC_WRITE, 0, NULL, CREATE_NEW, FILE_ATTRIBUTE_NORMAL, NULL);
DWORD bufferWritten = 0;
WriteFile(hFile, buffer, (DWORD)bufferUsed, &bufferWritten, NULL);
CloseHandle(hFile);
delete[] buffer;
}
}
std::vector<std::vector<std::string>> headers = ExtractAllHeaders(call);
HCHttpCallCloseHandle(call);
printf_s("HTTP call done\r\n");
@ -272,32 +318,34 @@ void DoHttpCall(std::string url, std::string requestBody, bool isJson, std::stri
i++;
}
if (isJson && responseString.length() > 0)
if (!customWriteUsed)
{
// Returned string starts with a BOM strip it out.
uint8_t BOM[] = { 0xef, 0xbb, 0xbf, 0x0 };
if (responseString.find(reinterpret_cast<char*>(BOM)) == 0)
if (isJson && responseString.length() > 0)
{
responseString = responseString.substr(3);
// Returned string starts with a BOM strip it out.
uint8_t BOM[] = { 0xef, 0xbb, 0xbf, 0x0 };
if (responseString.find(reinterpret_cast<char*>(BOM)) == 0)
{
responseString = responseString.substr(3);
}
web::json::value json = web::json::value::parse(utility::conversions::to_string_t(responseString));;
}
web::json::value json = web::json::value::parse(utility::conversions::to_string_t(responseString));;
}
if (responseString.length() > 200)
{
std::string subResponseString = responseString.substr(0, 200);
printf_s("Response string:\r\n%s...\r\n", subResponseString.c_str());
}
else
{
printf_s("Response string:\r\n%s\r\n", responseString.c_str());
}
else
{
readBuffer.push_back('\0');
const char* responseStr = reinterpret_cast<const char*>(readBuffer.data());
printf_s("Response string: %s\n", responseStr);
}
SetEvent(g_exampleTaskDone.get());
delete asyncBlock;
};
HCHttpCallPerformAsync(call, asyncBlock);
HCHttpCallPerformAsync(call, asyncBlock);
WaitForSingleObject(g_exampleTaskDone.get(), INFINITE);
}
@ -312,11 +360,15 @@ int main()
StartBackgroundThread();
std::string url1 = "https://raw.githubusercontent.com/Microsoft/libHttpClient/master/Samples/Win32-Http/TestContent.json";
DoHttpCall(url1, "{\"test\":\"value\"},{\"test2\":\"value\"},{\"test3\":\"value\"},{\"test4\":\"value\"},{\"test5\":\"value\"},{\"test6\":\"value\"},{\"test7\":\"value\"}", true, "", false);
DoHttpCall(url1, "{\"test\":\"value\"},{\"test2\":\"value\"},{\"test3\":\"value\"},{\"test4\":\"value\"},{\"test5\":\"value\"},{\"test6\":\"value\"},{\"test7\":\"value\"}", true, "", true);
DoHttpCall(url1, "{\"test\":\"value\"},{\"test2\":\"value\"},{\"test3\":\"value\"},{\"test4\":\"value\"},{\"test5\":\"value\"},{\"test6\":\"value\"},{\"test7\":\"value\"}", true, "", false, false, false);
DoHttpCall(url1, "{\"test\":\"value\"},{\"test2\":\"value\"},{\"test3\":\"value\"},{\"test4\":\"value\"},{\"test5\":\"value\"},{\"test6\":\"value\"},{\"test7\":\"value\"}", true, "", true, false, false);
std::string url2 = "https://github.com/Microsoft/libHttpClient/raw/master/Samples/XDK-Http/Assets/SplashScreen.png";
DoHttpCall(url2, "", false, "SplashScreen.png", false);
DoHttpCall(url2, "", false, "SplashScreen.png", false, false, false);
std::string url3 = "https://80996.playfabapi.com/authentication/GetEntityToken";
DoHttpCall(url3, "", false, "", false, true, false);
DoHttpCall(url3, "", false, "", false, true, true);
HCCleanup();
ShutdownActiveThreads();

Просмотреть файл

@ -66,6 +66,52 @@ void Compression::CompressToGzip(uint8_t* inData, size_t inDataSize, HCCompressi
deflateEnd(&stream);
}
void Compression::DecompressFromGzip(uint8_t* inData, size_t inDataSize, http_internal_vector<uint8_t>& outData)
{
z_stream stream;
stream.zalloc = Z_NULL;
stream.zfree = Z_NULL;
stream.opaque = Z_NULL;
// WINDOWBITS | GZIP_ENCODING - add 16 to decode only the gzip format
inflateInit2(&stream, WINDOWBITS | GZIP_ENCODING);
stream.next_in = inData;
stream.avail_in = static_cast<uInt>(inDataSize);
int ret;
do {
outData.resize(outData.size() + CHUNK);
stream.avail_out = CHUNK;
stream.next_out = outData.data() + outData.size() - CHUNK;
ret = inflate(&stream, Z_NO_FLUSH);
if (ret == Z_OK || ret == Z_BUF_ERROR)
{
// Z_BUF_ERROR -> no progress was possible or there was not enough room in the output buffer
// Z_OK -> some progress has been made
continue;
}
else if (ret != Z_STREAM_END)
{
// Handle error
// All dynamically allocated data structures for this stream are freed
inflateEnd(&stream);
// Clear output data since it may contain incomplete or corrupted data
outData.clear();
return;
}
outData.resize(outData.size() - stream.avail_out);
} while (ret != Z_STREAM_END); // Z_STREAM_END if the end of the compressed data has been reached
inflateEnd(&stream);
}
NAMESPACE_XBOX_HTTP_CLIENT_END
#else
@ -82,6 +128,11 @@ void Compression::CompressToGzip(uint8_t*, size_t, HCCompressionLevel, http_inte
assert(false);
}
void Compression::DecompressFromGzip(uint8_t*, size_t, http_internal_vector<uint8_t>&)
{
assert(false);
}
NAMESPACE_XBOX_HTTP_CLIENT_END
#endif // !HC_NOZLIB

Просмотреть файл

@ -12,6 +12,7 @@ public:
static bool Available() noexcept;
static void CompressToGzip(uint8_t* inData, size_t inDataSize, HCCompressionLevel compressionLevel, http_internal_vector<uint8_t>& outData);
static void DecompressFromGzip(uint8_t* inData, size_t inDataSize, http_internal_vector<uint8_t>& outData);
private:
Compression() = delete;

Просмотреть файл

@ -101,6 +101,17 @@ HRESULT CALLBACK HC_CALL::PerfomAsyncProvider(XAsyncOp op, XAsyncProviderData co
}
else
{
// If Custom ReponseWriteFunction is specified and compressedResponse is specified reset to default response body write callback
if (call->responseBodyWriteFunction != HC_CALL::ResponseBodyWrite && call->compressedResponse)
{
// Store custom response write callback
call->clientResponseBodyWriteFunction = call->responseBodyWriteFunction;
call->clientResponseBodyWriteContext = call->responseBodyWriteFunctionContext;
// Set response write callback to HC_CALL::ResponseBodyWrite
call->responseBodyWriteFunction = HC_CALL::ResponseBodyWrite;
call->responseBodyWriteFunctionContext = nullptr;
}
// Compress body before call if applicable
if (Compression::Available() && call->compressionLevel != HCCompressionLevel::None)
{
@ -131,6 +142,7 @@ HRESULT CALLBACK HC_CALL::PerfomAsyncProvider(XAsyncOp op, XAsyncProviderData co
}
case XAsyncOp::Cleanup:
{
if (call->traceCall)
{
HC_TRACE_INFORMATION(HTTPCLIENT, "HC_CALL::PerfomAsyncProvider Cleanup [ID %llu]", TO_ULL(call->id));
@ -204,7 +216,8 @@ void CALLBACK HC_CALL::CompressRequestBody(void* c, bool canceled)
http_internal_vector<uint8_t> compressedRequestBodyBuffer;
Compression::CompressToGzip(uncompressedRequestyBodyBuffer.data(), requestBodySize, call->compressionLevel, compressedRequestBodyBuffer);
Compression::CompressToGzip(uncompressedRequestyBodyBuffer.data(), requestBodySize, call->compressionLevel,
compressedRequestBodyBuffer);
// Setting back to default read request body callback to be invoked by Platform-specific code
call->requestBodyReadFunction = HC_CALL::ReadRequestBody;
@ -353,6 +366,36 @@ void HC_CALL::PerformSingleRequestComplete(XAsyncBlock* async)
}
}
// Decompress Response Bytes
if (Compression::Available() && call->compressedResponse == true)
{
http_internal_vector<uint8_t> uncompressedResponseBodyBuffer;
Compression::DecompressFromGzip(
call->responseBodyBytes.data(),
call->responseBodyBytes.size(),
uncompressedResponseBodyBuffer);
call->responseBodyBytes.resize(uncompressedResponseBodyBuffer.size());
call->responseBodyBytes = std::move(uncompressedResponseBodyBuffer);
}
// Check if we 'reset' the custom response write callback before decompressing the response
HCHttpCallResponseBodyWriteFunction temporaryWriteFunction = call->clientResponseBodyWriteFunction;
// call->clientResponseBodyWriteFunction should remain uninitialized if we did not 'reset' a call's custom response write callback
if (temporaryWriteFunction != nullptr)
{
// Invoke custom response write callback
temporaryWriteFunction(call, reinterpret_cast<uint8_t*>(call->responseBodyBytes.data()), call->responseBodyBytes.size(), call->clientResponseBodyWriteContext);
// Set responseBodyWriteFunction to call->clientResponseBodyWriteFunction
call->responseBodyWriteFunction = call->clientResponseBodyWriteFunction;
call->responseBodyWriteFunctionContext = call->clientResponseBodyWriteContext;
call->clientResponseBodyWriteFunction = nullptr;
call->clientResponseBodyWriteContext = nullptr;
}
// Complete perform if we aren't retrying or if there were any XAsync failures
XAsyncComplete(context->asyncBlock, hr, 0);
}

Просмотреть файл

@ -66,6 +66,15 @@ public:
xbox::httpclient::HttpHeaders responseHeaders{};
HCHttpCallResponseBodyWriteFunction responseBodyWriteFunction{ HC_CALL::ResponseBodyWrite };
void* responseBodyWriteFunctionContext{ nullptr };
// Response Compression
// If a custom write callback is set and a compressed response is expected then we would need
// to 'reset' responseBodyWriteFunction to HC_CALL::ResponseBodyWrite prior to decompression.
// This field will hold the custom response write callback until decompression is completed
HCHttpCallResponseBodyWriteFunction clientResponseBodyWriteFunction{ nullptr };
// Hold a write function context that may be provided when a custom write callback
// is set and a compressed response is expected
void* clientResponseBodyWriteContext{ nullptr };
bool compressedResponse{ false };
// Request metadata
bool performCalled{ false };

Просмотреть файл

@ -122,7 +122,32 @@ try
return S_OK;
}
CATCH_RETURN()
CATCH_RETURN()
STDAPI
HCHttpCallResponseSetGzipCompressed(
_In_ HCCallHandle call,
_In_ bool compressed
) noexcept
try
{
if (call == nullptr)
{
return E_INVALIDARG;
}
RETURN_IF_PERFORM_CALLED(call);
auto httpSingleton = get_http_singleton();
if (nullptr == httpSingleton)
return E_HC_NOT_INITIALISED;
call->compressedResponse = compressed;
if (call->traceCall) { HC_TRACE_INFORMATION(HTTPCLIENT, "HCHttpCallResponseSetGzipCompressed [ID %llu]", TO_ULL(call->id)); }
return S_OK;
} CATCH_RETURN()
STDAPI
HCHttpCallRequestSetRequestBodyString(

Просмотреть файл

@ -40,6 +40,7 @@ _HCHttpCallRequestSetRequestBodyReadFunction
_HCHttpCallRequestGetRequestBodyReadFunction
_HCHttpCallResponseSetResponseBodyWriteFunction
_HCHttpCallResponseGetResponseBodyWriteFunction
_HCHttpCallResponseSetGzipCompressed
_HCWebSocketCreate
_HCWebSocketSetProxyUri

Просмотреть файл

@ -40,6 +40,7 @@ _HCHttpCallRequestSetRequestBodyReadFunction
_HCHttpCallRequestGetRequestBodyReadFunction
_HCHttpCallResponseSetResponseBodyWriteFunction
_HCHttpCallResponseGetResponseBodyWriteFunction
_HCHttpCallResponseSetGzipCompressed
#
# httpProvider.h