fixes bug 82873 "improved support for HTTP trailers" r=gagan, sr=mscott

netwerk/protocol/http/src/nsHttp.h

@@ -76,6 +76,9 @@ struct nsHttpAtom
     operator const char *() { return _val; }
     const char *get() { return _val; }
 
+    void operator=(const char *v) { _val = v; }
+    void operator=(const nsHttpAtom &a) { _val = a._val; }
+
     // private
     const char *_val;
 };

netwerk/protocol/http/src/nsHttpChunkedDecoder.cpp

@@ -36,6 +36,27 @@ nsHttpChunkedDecoder::HandleChunkedContent(char *buf,
     LOG(("nsHttpChunkedDecoder::HandleChunkedContent [count=%u]\n", count));
 
     *countRead = 0;
+    
+    // from RFC2617 section 3.6.1, the chunked transfer coding is defined as:
+    //
+    //   Chunked-Body    = *chunk
+    //                     last-chunk
+    //                     trailer
+    //                     CRLF
+    //   chunk           = chunk-size [ chunk-extension ] CRLF
+    //                     chunk-data CRLF
+    //   chunk-size      = 1*HEX
+    //   last-chunk      = 1*("0") [ chunk-extension ] CRLF
+    //       
+    //   chunk-extension = *( ";" chunk-ext-name [ "=" chunk-ext-val ] )
+    //   chunk-ext-name  = token
+    //   chunk-ext-val   = token | quoted-string
+    //   chunk-data      = chunk-size(OCTET)
+    //   trailer         = *(entity-header CRLF)
+    //
+    // the chunk-size field is a string of hex digits indicating the size of the
+    // chunk.  the chunked encoding is ended by any chunk whose size is zero, 
+    // followed by the trailer, which is terminated by an empty line.
 
     while (count) {
         if (mChunkRemaining) {
@@ -47,8 +68,9 @@ nsHttpChunkedDecoder::HandleChunkedContent(char *buf,
             *countRead += amt;
             buf += amt;
         }
-
+        else if (mReachedEOF)
-        if (count) {
+            break; // done
+        else {
             PRUint32 bytesConsumed = 0;
 
             nsresult rv = ParseChunkRemaining(buf, count, &bytesConsumed);
@@ -83,41 +105,59 @@ nsHttpChunkedDecoder::ParseChunkRemaining(char *buf,
     char *p = NS_STATIC_CAST(char *, memchr(buf, '\n', count));
     if (p) {
         *p = 0;
+        if ((p > buf) && (*(p-1) == '\r')) // eliminate a preceding CR
+            *(p-1) = 0;
         *bytesConsumed = p - buf + 1;
-        if (mLineBuf.IsEmpty())
+
-            p = buf;
+        // make buf point to the full line buffer to parse
-        else {
+        if (!mLineBuf.IsEmpty()) {
-            // append to the line buf and use that as the buf to parse.
             mLineBuf.Append(buf);
-            p = (char *) mLineBuf.get();
+            buf = (char *) mLineBuf.get();
         }
+
+        if (mWaitEOF) {
+            if (*buf) {
+                LOG(("got trailer: %s\n", buf));
+                // allocate a header array for the trailers on demand
+                if (!mTrailers) {
+                    mTrailers = new nsHttpHeaderArray();
+                    if (!mTrailers)
+                        return NS_ERROR_OUT_OF_MEMORY;
+                }
+                mTrailers->ParseHeaderLine(buf);
+            }
+            else {
+                mWaitEOF = PR_FALSE;
+                mReachedEOF = PR_TRUE;
+                LOG(("reached end of chunked-body\n"));
+            }
+        }
+        else if (*buf) {
+            // ignore any chunk-extensions
+            if ((p = PL_strchr(buf, ';')) != nsnull)
+                *p = 0;
+
+            if (!sscanf(buf, "%x", &mChunkRemaining)) {
+                LOG(("sscanf failed parsing hex on string [%s]\n", buf));
+                return NS_ERROR_UNEXPECTED;
+            }
+
+            // we've discovered the last chunk
+            if (mChunkRemaining == 0)
+                mWaitEOF = PR_TRUE;
+        }
+
+        // ensure that the line buffer is clear
+        mLineBuf.Truncate();
     }
     else {
-        mLineBuf.Append(buf, count);
+        // save the partial line; wait for more data
         *bytesConsumed = count;
-        return NS_OK;
+        // ignore a trailing CR
+        if (buf[count-1] == '\r')
+            count--;
+        mLineBuf.Append(buf, count);
     }
 
-    if (*p && *p != '\r') {
-        buf = p;
-
-        // ignore any chunk-extensions
-        if ((p = PL_strchr(buf, ';')) != nsnull)
-            *p = 0;
-
-        if (!sscanf(buf, "%x", &mChunkRemaining)) {
-            LOG(("sscanf failed parsing hex on string [%s]\n", buf));
-            return NS_ERROR_UNEXPECTED;
-        }
-
-        // XXX need to add code to consume "trailer" headers
-        if ((mChunkRemaining == 0) && (*buf != 0))
-            mReachedEOF = PR_TRUE;
-    }
-
-    // clear the line buffer if it is not already empty
-    if (!mLineBuf.IsEmpty())
-        mLineBuf.SetLength(0);
-
     return NS_OK;
 }

netwerk/protocol/http/src/nsHttpChunkedDecoder.h

@@ -26,28 +26,41 @@
 
 #include "nsError.h"
 #include "nsString.h"
+#include "nsHttpHeaderArray.h"
 
 class nsHttpChunkedDecoder
 {
 public:
-    nsHttpChunkedDecoder() : mChunkRemaining(0), mReachedEOF(0) {}
+    nsHttpChunkedDecoder() : mTrailers(nsnull)
-   ~nsHttpChunkedDecoder() {}
+                           , mChunkRemaining(0)
+                           , mReachedEOF(PR_FALSE)
+                           , mWaitEOF(PR_FALSE) {}
+   ~nsHttpChunkedDecoder() { delete mTrailers; }
 
     PRBool ReachedEOF() { return mReachedEOF; }
 
-    // Called by the transaction to handle chunked content.
+    // called by the transaction to handle chunked content.
     nsresult HandleChunkedContent(char *buf,
                                   PRUint32 count,
                                   PRUint32 *countRead);
+
+    nsHttpHeaderArray *Trailers() { return mTrailers; }
+
+    nsHttpHeaderArray *TakeTrailers() { nsHttpHeaderArray *h = mTrailers;
+                                        mTrailers = nsnull;
+                                        return h; }
+
 private:
     nsresult ParseChunkRemaining(char *buf,
                                  PRUint32 count,
                                  PRUint32 *countRead);
 
 private:
-    PRUint32     mChunkRemaining;
+    nsHttpHeaderArray *mTrailers;
-    nsCString    mLineBuf; // may hold a partial line
+    PRUint32           mChunkRemaining;
-    PRPackedBool mReachedEOF;
+    nsCString          mLineBuf; // may hold a partial line
+    PRPackedBool       mReachedEOF;
+    PRPackedBool       mWaitEOF;
 };
 
 #endif

netwerk/protocol/http/src/nsHttpHeaderArray.cpp

@@ -102,6 +102,62 @@ nsHttpHeaderArray::VisitHeaders(nsIHttpHeaderVisitor *visitor)
     return NS_OK;
 }
 
+void
+nsHttpHeaderArray::ParseHeaderLine(char *line, nsHttpAtom *hdr, char **val)
+{
+    char *p = PL_strchr(line, ':'), *p2;
+
+    // the header is malformed... but, there are malformed headers in the
+    // world.  search for ' ' and '\t' to simulate 4.x/IE behavior.
+    if (!p) {
+        p = PL_strchr(line, ' ');
+        if (!p) {
+            p = PL_strchr(line, '\t');
+            if (!p) {
+                // some broken cgi scripts even use '=' as a delimiter!!
+                p = PL_strchr(line, '=');
+            }
+        }
+    }
+
+    if (p) {
+        // ignore whitespace between header name and colon
+        p2 = p;
+        while (--p2 >= line && ((*p2 == ' ') || (*p2 == '\t')))
+            ;
+        *++p2= 0; // overwrite first char after header name
+
+        nsHttpAtom atom = nsHttp::ResolveAtom(line);
+        if (atom) {
+            // skip over whitespace
+            do {
+                ++p;
+            } while ((*p == ' ') || (*p == '\t'));
+
+            // trim trailing whitespace - bug 86608
+            p2 = p + PL_strlen(p);
+            do {
+                --p2;
+            } while (p2 >= p && ((*p2 == ' ') || (*p2 == '\t')));
+            *++p2 = 0;
+
+            // assign return values
+            if (hdr) *hdr = atom;
+            if (val) *val = p;
+
+            // assign response header
+            SetHeader(atom, p);
+        }
+        else
+            LOG(("unknown header; skipping\n"));
+    }
+    else
+        LOG(("malformed header\n"));
+
+    // We ignore mal-formed headers in the hope that we'll still be able
+    // to do something useful with the response.
+}
+
 void
 nsHttpHeaderArray::Flatten(nsACString &buf)
 {

netwerk/protocol/http/src/nsHttpHeaderArray.h

@@ -43,7 +43,11 @@ public:
 
     nsresult VisitHeaders(nsIHttpHeaderVisitor *visitor);
 
-    void     Flatten(nsACString &);
+    // parse a header line, return the header atom and a pointer to the 
+    // header value (the substring of the header line -- do not free).
+    void ParseHeaderLine(char *line, nsHttpAtom *header=nsnull, char **value=nsnull);
+
+    void Flatten(nsACString &);
 
     PRUint32 Count() { return (PRUint32) mHeaders.Count(); }
 

netwerk/protocol/http/src/nsHttpResponseHead.cpp

@@ -169,59 +169,16 @@ nsHttpResponseHead::ParseStatusLine(char *line)
 void
 nsHttpResponseHead::ParseHeaderLine(char *line)
 {
-    char *p = PL_strchr(line, ':'), *p2;
+    nsHttpAtom hdr;
+    char *val;
 
-    // the header is malformed... but, there are malformed headers in the
+    mHeaders.ParseHeaderLine(line, &hdr, &val);
-    // world.  search for ' ' and '\t' to simulate 4.x/IE behavior.
-    if (!p) {
-        p = PL_strchr(line, ' ');
-        if (!p) {
-            p = PL_strchr(line, '\t');
-            if (!p) {
-                // some broken cgi scripts even use '=' as a delimiter!!
-                p = PL_strchr(line, '=');
-            }
-        }
-    }
 
-    if (p) {
+    // handle some special case headers...
-        // ignore whitespace between header name and colon
+    if (hdr == nsHttp::Content_Length)
-        p2 = p;
+        mContentLength = atoi(val);
-        while (--p2 >= line && ((*p2 == ' ') || (*p2 == '\t')))
+    else if (hdr == nsHttp::Content_Type)
-            ;
+        ParseContentType(val);
-        *++p2= 0; // overwrite first char after header name
-
-        nsHttpAtom atom = nsHttp::ResolveAtom(line);
-        if (atom) {
-            // skip over whitespace
-            do {
-                ++p;
-            } while ((*p == ' ') || (*p == '\t'));
-
-            // trim trailing whitespace - bug 86608
-            p2 = p + PL_strlen(p);
-            do {
-                --p2;
-            } while (p2 >= p && ((*p2 == ' ') || (*p2 == '\t')));
-            *++p2 = 0;
-
-            // assign response header
-            mHeaders.SetHeader(atom, p);
-
-            // handle some special case headers...
-            if (atom == nsHttp::Content_Length)
-                mContentLength = atoi(p);
-            else if (atom == nsHttp::Content_Type)
-                ParseContentType(p);
-        }
-        else
-            LOG(("unknown header; skipping\n"));
-    }
-    else
-        LOG(("malformed header\n"));
-
-    // We ignore mal-formed headers in the hope that we'll still be able
-    // to do something useful with the response.
 }
 
 // From section 13.2.3 of RFC2616, we compute the current age of a cached

netwerk/protocol/http/src/nsHttpTransaction.cpp

@@ -503,14 +503,6 @@ nsHttpTransaction::HandleContentStart()
                 if (!mChunkedDecoder)
                     return NS_ERROR_OUT_OF_MEMORY;
                 LOG(("chunked decoder created\n"));
-
-                val = mResponseHead->PeekHeader(nsHttp::Trailer);
-                if (val) {
-                    LOG(("response contains a Trailer header\n"));
-                    // FIXME we should at least eat the trailer headers so this
-                    // connection could be reused.
-                    mConnection->DontReuse();
-                }
             }
 #if defined(PR_LOGGING)
             else if (mContentLength == -1)