fix #121: parsing of chunked transfer encoding

This commit is contained in:
Tomasz Janczuk 2011-12-16 14:24:43 -08:00
Родитель 337a0f2697
Коммит a304bdada9
11 изменённых файлов: 323 добавлений и 89 удалений

Просмотреть файл

@ -270,6 +270,113 @@ Error:
return hr;
}
HRESULT CHttpProtocol::ParseChunkHeader(CNodeHttpStoredContext* context)
{
HRESULT hr;
char* data = (char*)context->GetBuffer() + context->GetParsingOffset();
char* current;
char* chunkHeaderStart;
DWORD dataSize = context->GetDataSize() - context->GetParsingOffset();
ULONG chunkLength = 0;
ULONG totalChunkLength = 0;
// attempt to parse as many response body chunks as there are buffered in memory
current = data;
do
{
// parse chunk length
chunkHeaderStart = current;
chunkLength = 0;
while (true)
{
ErrorIf((current - data) >= dataSize, ERROR_MORE_DATA);
if (*current >= 'A' && *current <= 'F')
{
chunkLength <<= 4;
chunkLength += *current - 'A' + 10;
}
else if (*current >= 'a' && *current <= 'f')
{
chunkLength <<= 4;
chunkLength += *current - 'a' + 10;
}
else if (*current >= '0' && *current <= '9')
{
chunkLength <<= 4;
chunkLength += *current - '0';
}
else
{
ErrorIf(current == chunkHeaderStart, ERROR_BAD_FORMAT); // no hex digits found
break;
}
current++;
}
// skip optional extensions
while (true)
{
ErrorIf((current - data) >= dataSize, ERROR_MORE_DATA);
if (*current == 0x0D)
{
break;
}
current++;
}
// LF
current++;
ErrorIf((current - data) >= dataSize, ERROR_MORE_DATA);
ErrorIf(*current != 0x0A, ERROR_BAD_FORMAT);
current++;
// remember total length of all parsed chunks before attempting to parse subsequent chunk header
// set total chunk length to include current chunk content length, previously parsed chunks (with headers),
// plus the CRLF following the current chunk content
totalChunkLength = chunkLength + (ULONG)(current - data) + 2;
current += chunkLength + 2; // chunk content length + CRLF
} while (chunkLength != 0); // exit when last chunk has been detected
// if we are here, current buffer contains the header of the last chunk of the response
context->SetChunkLength(totalChunkLength);
context->SetIsLastChunk(TRUE);
context->SetChunkTransmitted(0);
return S_OK;
Error:
if (ERROR_MORE_DATA != hr)
{
context->GetNodeApplication()->GetApplicationManager()->GetEventProvider()->Log(
L"iisnode failed to parse response body chunk header", WINEVENT_LEVEL_ERROR, context->GetActivityId());
return hr;
}
else if (0 < totalChunkLength)
{
// at least one response chunk has been successfuly parsed, but more chunks remain
context->SetChunkLength(totalChunkLength);
context->SetIsLastChunk(FALSE);
context->SetChunkTransmitted(0);
return S_OK;
}
return hr;
}
HRESULT CHttpProtocol::ParseResponseHeaders(CNodeHttpStoredContext* context)
{
HRESULT hr;

Просмотреть файл

@ -15,6 +15,7 @@ public:
static HRESULT SerializeRequestHeaders(CNodeHttpStoredContext* ctx, void** result, DWORD* resultSize, DWORD* resultLength);
static HRESULT ParseResponseStatusLine(CNodeHttpStoredContext* context);
static HRESULT ParseResponseHeaders(CNodeHttpStoredContext* context);
static HRESULT ParseChunkHeader(CNodeHttpStoredContext* context);
};
#endif

Просмотреть файл

@ -2,7 +2,7 @@
CNodeHttpStoredContext::CNodeHttpStoredContext(CNodeApplication* nodeApplication, IHttpContext* context)
: nodeApplication(nodeApplication), context(context), process(NULL), buffer(NULL), bufferSize(0), dataSize(0), parsingOffset(0),
responseContentLength(0), responseContentTransmitted(0), pipe(INVALID_HANDLE_VALUE), result(S_OK),
chunkLength(0), chunkTransmitted(0), isChunked(FALSE), pipe(INVALID_HANDLE_VALUE), result(S_OK), isLastChunk(FALSE),
requestNotificationStatus(RQ_NOTIFICATION_PENDING), connectionRetryCount(0), pendingAsyncOperationCount(1),
targetUrl(NULL), targetUrlLength(0), childContext(NULL)
{
@ -153,24 +153,44 @@ void CNodeHttpStoredContext::SetParsingOffset(DWORD parsingOffset)
this->parsingOffset = parsingOffset;
}
LONGLONG CNodeHttpStoredContext::GetResponseContentTransmitted()
LONGLONG CNodeHttpStoredContext::GetChunkTransmitted()
{
return this->responseContentTransmitted;
return this->chunkTransmitted;
}
LONGLONG CNodeHttpStoredContext::GetResponseContentLength()
LONGLONG CNodeHttpStoredContext::GetChunkLength()
{
return this->responseContentLength;
return this->chunkLength;
}
void CNodeHttpStoredContext::SetResponseContentTransmitted(LONGLONG length)
void CNodeHttpStoredContext::SetChunkTransmitted(LONGLONG length)
{
this->responseContentTransmitted = length;
this->chunkTransmitted = length;
}
void CNodeHttpStoredContext::SetResponseContentLength(LONGLONG length)
void CNodeHttpStoredContext::SetChunkLength(LONGLONG length)
{
this->responseContentLength = length;
this->chunkLength = length;
}
BOOL CNodeHttpStoredContext::GetIsChunked()
{
return this->isChunked;
}
void CNodeHttpStoredContext::SetIsChunked(BOOL chunked)
{
this->isChunked = chunked;
}
void CNodeHttpStoredContext::SetIsLastChunk(BOOL lastChunk)
{
this->isLastChunk = lastChunk;
}
BOOL CNodeHttpStoredContext::GetIsLastChunk()
{
return this->isLastChunk;
}
HRESULT CNodeHttpStoredContext::GetHresult()

Просмотреть файл

@ -19,14 +19,16 @@ private:
DWORD bufferSize;
DWORD dataSize;
DWORD parsingOffset;
LONGLONG responseContentTransmitted;
LONGLONG responseContentLength;
LONGLONG chunkTransmitted;
LONGLONG chunkLength;
BOOL isChunked;
HRESULT result;
REQUEST_NOTIFICATION_STATUS requestNotificationStatus;
long pendingAsyncOperationCount;
PCSTR targetUrl;
DWORD targetUrlLength;
IHttpContext* childContext;
BOOL isLastChunk;
public:
@ -47,8 +49,11 @@ public:
DWORD* GetBufferSizeRef();
DWORD GetDataSize();
DWORD GetParsingOffset();
LONGLONG GetResponseContentTransmitted();
LONGLONG GetResponseContentLength();
LONGLONG GetChunkTransmitted();
LONGLONG GetChunkLength();
BOOL GetIsChunked();
void SetIsLastChunk(BOOL lastChunk);
BOOL GetIsLastChunk();
HRESULT GetHresult();
REQUEST_NOTIFICATION_STATUS GetRequestNotificationStatus();
GUID* GetActivityId();
@ -66,8 +71,9 @@ public:
void SetBufferSize(DWORD bufferSize);
void SetDataSize(DWORD dataSize);
void SetParsingOffset(DWORD parsingOffet);
void SetResponseContentTransmitted(LONGLONG length);
void SetResponseContentLength(LONGLONG length);
void SetChunkTransmitted(LONGLONG length);
void SetChunkLength(LONGLONG length);
void SetIsChunked(BOOL chunked);
void SetHresult(HRESULT result);
void SetRequestNotificationStatus(REQUEST_NOTIFICATION_STATUS status);
LPOVERLAPPED InitializeOverlapped();

Просмотреть файл

@ -758,18 +758,6 @@ void CProtocolBridge::ContinueReadResponse(CNodeHttpStoredContext* context)
WINEVENT_LEVEL_VERBOSE,
&activityId);
}
else if (context->GetResponseContentLength() == -1)
{
// connection termination with chunked transfer encoding indicates end of response
// since we have sent Connection: close HTTP request header to node from SendHttpRequestHeaders
etw->Log(L"iisnode iniatiated reading http response chunk and synchronously detected the end of the http response",
WINEVENT_LEVEL_VERBOSE,
&activityId);
// CR: narrow down this condition to orderly pipe closure
CProtocolBridge::FinalizeResponse(context);
}
else
{
// error
@ -844,7 +832,8 @@ void WINAPI CProtocolBridge::ProcessResponseHeaders(DWORD error, DWORD bytesTran
contentLength = ctx->GetHttpContext()->GetResponse()->GetHeader(HttpHeaderContentLength, &contentLengthLength);
if (0 == contentLengthLength)
{
ctx->SetResponseContentLength(-1);
ctx->SetIsChunked(TRUE);
ctx->SetNextProcessor(CProtocolBridge::ProcessChunkHeader);
}
else
{
@ -858,14 +847,16 @@ void WINAPI CProtocolBridge::ProcessResponseHeaders(DWORD error, DWORD bytesTran
while (i < contentLengthLength && contentLength[i] >= '0' && contentLength[i] <= '9')
length = length * 10 + contentLength[i++] - '0';
ctx->SetResponseContentLength(length);
ctx->SetIsChunked(FALSE);
ctx->SetIsLastChunk(TRUE);
ctx->SetChunkLength(length);
ctx->SetNextProcessor(CProtocolBridge::ProcessResponseBody);
}
ctx->GetNodeApplication()->GetApplicationManager()->GetEventProvider()->Log(
L"iisnode finished processing http response headers", WINEVENT_LEVEL_VERBOSE, ctx->GetActivityId());
ctx->SetNextProcessor(CProtocolBridge::ProcessResponseBody);
CProtocolBridge::ProcessResponseBody(S_OK, 0, ctx->GetOverlapped());
ctx->GetAsyncContext()->completionProcessor(S_OK, 0, ctx->GetOverlapped());
return;
Error:
@ -884,6 +875,43 @@ Error:
return;
}
void WINAPI CProtocolBridge::ProcessChunkHeader(DWORD error, DWORD bytesTransfered, LPOVERLAPPED overlapped)
{
HRESULT hr;
CNodeHttpStoredContext* ctx = CNodeHttpStoredContext::Get(overlapped);
ctx->GetNodeApplication()->GetApplicationManager()->GetEventProvider()->Log(
L"iisnode starting to process http response body chunk header", WINEVENT_LEVEL_VERBOSE, ctx->GetActivityId());
CheckError(error);
ctx->SetDataSize(ctx->GetDataSize() + bytesTransfered);
CheckError(CHttpProtocol::ParseChunkHeader(ctx));
ctx->GetNodeApplication()->GetApplicationManager()->GetEventProvider()->Log(
L"iisnode finished processing http response body chunk header", WINEVENT_LEVEL_VERBOSE, ctx->GetActivityId());
ctx->SetNextProcessor(CProtocolBridge::ProcessResponseBody);
CProtocolBridge::ProcessResponseBody(S_OK, 0, ctx->GetOverlapped());
return;
Error:
if (ERROR_MORE_DATA == hr)
{
CProtocolBridge::ContinueReadResponse(ctx);
}
else
{
ctx->GetNodeApplication()->GetApplicationManager()->GetEventProvider()->Log(
L"iisnode failed to process response body chunk header", WINEVENT_LEVEL_ERROR, ctx->GetActivityId());
CProtocolBridge::SendEmptyResponse(ctx, 500, _T("Internal Server Error"), hr);
}
return;
}
void WINAPI CProtocolBridge::ProcessResponseBody(DWORD error, DWORD bytesTransfered, LPOVERLAPPED overlapped)
{
HRESULT hr;
@ -895,64 +923,73 @@ void WINAPI CProtocolBridge::ProcessResponseBody(DWORD error, DWORD bytesTransfe
ctx->GetNodeApplication()->GetApplicationManager()->GetEventProvider()->Log(
L"iisnode starting to process http response body", WINEVENT_LEVEL_VERBOSE, ctx->GetActivityId());
if (S_OK != error)
{
if (ctx->GetResponseContentLength() == -1)
{
// connection termination with chunked transfer encoding indicates end of response
// since we have sent Connection: close HTTP request header to node from SendHttpRequestHeaders
ctx->GetNodeApplication()->GetApplicationManager()->GetEventProvider()->Log(
L"iisnode detected the end of the http response", WINEVENT_LEVEL_VERBOSE, ctx->GetActivityId());
// CR: check the other commend for finalizing response
CProtocolBridge::FinalizeResponse(ctx);
}
else
{
ctx->GetNodeApplication()->GetApplicationManager()->GetEventProvider()->Log(
L"iisnode failed to read http response body", WINEVENT_LEVEL_ERROR, ctx->GetActivityId());
CProtocolBridge::SendEmptyResponse(ctx, 500, _T("Internal Server Error"), error);
}
return;
}
CheckError(error);
ctx->SetDataSize(ctx->GetDataSize() + bytesTransfered);
if (ctx->GetDataSize() > ctx->GetParsingOffset())
{
// send body data to client
// there is response body data in the buffer
// CR: consider using malloc here (memory can be released after Flush)
ErrorIf(NULL == (chunk = (HTTP_DATA_CHUNK*) ctx->GetHttpContext()->AllocateRequestMemory(sizeof HTTP_DATA_CHUNK)), ERROR_NOT_ENOUGH_MEMORY);
chunk->DataChunkType = HttpDataChunkFromMemory;
chunk->FromMemory.BufferLength = ctx->GetDataSize() - ctx->GetParsingOffset();
ErrorIf(NULL == (chunk->FromMemory.pBuffer = ctx->GetHttpContext()->AllocateRequestMemory(chunk->FromMemory.BufferLength)), ERROR_NOT_ENOUGH_MEMORY);
memcpy(chunk->FromMemory.pBuffer, (char*)ctx->GetBuffer() + ctx->GetParsingOffset(), chunk->FromMemory.BufferLength);
ctx->SetDataSize(0);
ctx->SetParsingOffset(0);
ctx->SetNextProcessor(CProtocolBridge::SendResponseBodyCompleted);
CheckError(ctx->GetHttpContext()->GetResponse()->WriteEntityChunks(
chunk,
1,
TRUE,
ctx->GetResponseContentLength() == -1 || ctx->GetResponseContentLength() > (ctx->GetResponseContentTransmitted() + chunk->FromMemory.BufferLength),
&bytesSent,
&completionExpected));
ctx->GetNodeApplication()->GetApplicationManager()->GetEventProvider()->Log(
L"iisnode started sending http response body chunk", WINEVENT_LEVEL_VERBOSE, ctx->GetActivityId());
if (!completionExpected)
if (ctx->GetChunkLength() > ctx->GetChunkTransmitted())
{
CProtocolBridge::SendResponseBodyCompleted(S_OK, chunk->FromMemory.BufferLength, ctx->GetOverlapped());
// send the smaller of the rest of the current chunk or the data available in the buffer to the client
DWORD dataInBuffer = ctx->GetDataSize() - ctx->GetParsingOffset();
DWORD remainingChunkSize = ctx->GetChunkLength() - ctx->GetChunkTransmitted();
DWORD bytesToSend = dataInBuffer < remainingChunkSize ? dataInBuffer : remainingChunkSize;
// CR: consider using malloc here (memory can be released after Flush)
ErrorIf(NULL == (chunk = (HTTP_DATA_CHUNK*) ctx->GetHttpContext()->AllocateRequestMemory(sizeof HTTP_DATA_CHUNK)), ERROR_NOT_ENOUGH_MEMORY);
chunk->DataChunkType = HttpDataChunkFromMemory;
chunk->FromMemory.BufferLength = bytesToSend;
ErrorIf(NULL == (chunk->FromMemory.pBuffer = ctx->GetHttpContext()->AllocateRequestMemory(chunk->FromMemory.BufferLength)), ERROR_NOT_ENOUGH_MEMORY);
memcpy(chunk->FromMemory.pBuffer, (char*)ctx->GetBuffer() + ctx->GetParsingOffset(), chunk->FromMemory.BufferLength);
if (bytesToSend == dataInBuffer)
{
ctx->SetDataSize(0);
ctx->SetParsingOffset(0);
}
else
{
ctx->SetParsingOffset(ctx->GetParsingOffset() + bytesToSend);
}
ctx->SetNextProcessor(CProtocolBridge::SendResponseBodyCompleted);
CheckError(ctx->GetHttpContext()->GetResponse()->WriteEntityChunks(
chunk,
1,
TRUE,
!ctx->GetIsLastChunk(),
&bytesSent,
&completionExpected));
ctx->GetNodeApplication()->GetApplicationManager()->GetEventProvider()->Log(
L"iisnode started sending http response body chunk", WINEVENT_LEVEL_VERBOSE, ctx->GetActivityId());
if (!completionExpected)
{
CProtocolBridge::SendResponseBodyCompleted(S_OK, chunk->FromMemory.BufferLength, ctx->GetOverlapped());
}
}
else if (ctx->GetIsChunked())
{
// process next chunk of the chunked encoding
ctx->SetNextProcessor(CProtocolBridge::ProcessChunkHeader);
CProtocolBridge::ProcessChunkHeader(S_OK, 0, ctx->GetOverlapped());
}
else
{
// response data detected beyond the body length declared with Content-Length
CheckError(ERROR_BAD_FORMAT);
}
}
else if (-1 == ctx->GetResponseContentLength() || ctx->GetResponseContentLength() > ctx->GetResponseContentTransmitted())
else if (ctx->GetIsChunked() || ctx->GetChunkLength() > ctx->GetChunkTransmitted())
{
// read more body data
@ -983,11 +1020,15 @@ void WINAPI CProtocolBridge::SendResponseBodyCompleted(DWORD error, DWORD bytesT
BOOL completionExpected = FALSE;
CheckError(error);
ctx->SetResponseContentTransmitted(ctx->GetResponseContentTransmitted() + bytesTransfered);
ctx->SetChunkTransmitted(ctx->GetChunkTransmitted() + bytesTransfered);
if (ctx->GetResponseContentLength() == -1 || ctx->GetResponseContentLength() > ctx->GetResponseContentTransmitted())
if (ctx->GetIsLastChunk() && ctx->GetChunkLength() == ctx->GetChunkTransmitted())
{
if (ctx->GetResponseContentLength() == -1 && CModuleConfiguration::GetFlushResponse(ctx->GetHttpContext()))
CProtocolBridge::FinalizeResponse(ctx);
}
else
{
if (ctx->GetIsChunked() && CModuleConfiguration::GetFlushResponse(ctx->GetHttpContext()))
{
// Flushing of chunked responses is enabled
@ -1002,10 +1043,6 @@ void WINAPI CProtocolBridge::SendResponseBodyCompleted(DWORD error, DWORD bytesT
CProtocolBridge::ContinueProcessResponseBodyAfterPartialFlush(S_OK, 0, ctx->GetOverlapped());
}
}
else
{
CProtocolBridge::FinalizeResponse(ctx);
}
return;
Error:
@ -1024,7 +1061,7 @@ void WINAPI CProtocolBridge::ContinueProcessResponseBodyAfterPartialFlush(DWORD
CheckError(error);
ctx->SetNextProcessor(CProtocolBridge::ProcessResponseBody);
CProtocolBridge::ContinueReadResponse(ctx);
CProtocolBridge::ProcessResponseBody(S_OK, 0, ctx->GetOverlapped());
return;
Error:

Просмотреть файл

@ -30,7 +30,9 @@ private:
static void ContinueReadResponse(CNodeHttpStoredContext* context);
static void WINAPI ProcessResponseStatusLine(DWORD error, DWORD bytesTransfered, LPOVERLAPPED overlapped);
static void WINAPI ProcessResponseHeaders(DWORD error, DWORD bytesTransfered, LPOVERLAPPED overlapped);
static void WINAPI ProcessResponseBody(DWORD error, DWORD bytesTransfered, LPOVERLAPPED overlapped);
static void WINAPI ProcessChunkHeader(DWORD error, DWORD bytesTransfered, LPOVERLAPPED overlapped);
static void WINAPI ProcessResponseBody(DWORD error, DWORD bytesTransfered, LPOVERLAPPED overlapped);
static void WINAPI SendResponseBodyCompleted(DWORD error, DWORD bytesTransfered, LPOVERLAPPED overlapped);
static void WINAPI ContinueProcessResponseBodyAfterPartialFlush(DWORD error, DWORD bytesTransfered, LPOVERLAPPED overlapped);

Просмотреть файл

@ -238,6 +238,7 @@ copy /y $(ProjectDir)\..\config\* $(ProjectDir)\..\..\build\$(Configuration)\$(P
<None Include="..\..\test\functional\tests\110_log_size_quota.js" />
<None Include="..\..\test\functional\tests\111_node_env.js" />
<None Include="..\..\test\functional\tests\112_dev_errors.js" />
<None Include="..\..\test\functional\tests\113_encoding.js" />
<None Include="..\..\test\functional\tests\200_samples.bat" />
<None Include="..\..\test\functional\tests\node_modules\iisnodeassert.js" />
<None Include="..\..\test\functional\tests\parts\106_autoupdate_first.js" />
@ -273,6 +274,8 @@ copy /y $(ProjectDir)\..\config\* $(ProjectDir)\..\..\build\$(Configuration)\$(P
<None Include="..\..\test\functional\www\112_dev_errors\off\web.config" />
<None Include="..\..\test\functional\www\112_dev_errors\on\hello.js" />
<None Include="..\..\test\functional\www\112_dev_errors\on\web.config" />
<None Include="..\..\test\functional\www\113_encoding\hello.js" />
<None Include="..\..\test\functional\www\113_encoding\web.config" />
<None Include="..\..\test\performance\client.bat" />
<None Include="..\..\test\performance\localRun.bat" />
<None Include="..\..\test\performance\readme.txt" />

Просмотреть файл

@ -108,6 +108,9 @@
<Filter Include="Tests\performance\www\default">
<UniqueIdentifier>{6b173a34-4fbb-49ac-9e03-33cde5a6e50f}</UniqueIdentifier>
</Filter>
<Filter Include="Tests\functional\www\113_encoding">
<UniqueIdentifier>{5e0f95ef-1256-4223-a412-f5146a7713cc}</UniqueIdentifier>
</Filter>
</ItemGroup>
<ItemGroup>
<ClCompile Include="main.cpp">
@ -462,5 +465,14 @@
<None Include="..\..\test\performance\localRun.bat">
<Filter>Tests\performance</Filter>
</None>
<None Include="..\..\test\functional\www\113_encoding\hello.js">
<Filter>Tests\functional\www\113_encoding</Filter>
</None>
<None Include="..\..\test\functional\www\113_encoding\web.config">
<Filter>Tests\functional\www\113_encoding</Filter>
</None>
<None Include="..\..\test\functional\tests\113_encoding.js">
<Filter>Tests\functional\tests</Filter>
</None>
</ItemGroup>
</Project>

Просмотреть файл

@ -0,0 +1,11 @@
/*
Fixed length and chunked transfer encoding responses are received.
*/
var iisnodeassert = require("iisnodeassert");
iisnodeassert.sequence([
iisnodeassert.get(10000, "/113_encoding/hello.js", 200, "content-length response"),
iisnodeassert.get(2000, "/113_encoding/hello.js?onechunk", 200, "chunked response"),
iisnodeassert.get(4000, "/113_encoding/hello.js?tenchunks", 200, "0123456789")
]);

Просмотреть файл

@ -0,0 +1,28 @@
var http = require('http');
http.createServer(function (req, res) {
var query = require('url').parse(req.url).query;
if (query === 'onechunk') { // chunked transfer encoding, one chunk
res.writeHead(200, { 'Content-Type': 'text/html' });
res.end('chunked response');
}
else if (query === 'tenchunks') { // chunked transfer encoding, ten chunks
res.writeHead(200, { 'Content-Type': 'text/html' });
var n = 0;
function writeOne() {
if (n < 9) {
res.write(n.toString());
n++;
setTimeout(writeOne, 200);
}
else {
res.end(n.toString());
}
}
writeOne();
}
else { // fixed response length with Content-Length
res.writeHead(200, { 'Content-Type': 'text/html', 'Content-Length': '23' });
res.end('content-length response');
}
}).listen(process.env.PORT);

Просмотреть файл

@ -0,0 +1,7 @@
<configuration>
<system.webServer>
<handlers>
<add name="iisnode" path="hello.js" verb="*" modules="iisnode" />
</handlers>
</system.webServer>
</configuration>