known bug #46: chunked-encoded CONNECT responses from a http proxy now works.

Added test case 1008 to verify. Note that #47 is still there.

CHANGES

@@ -6,6 +6,9 @@
 
                                   Changelog
 
+Daniel S (2 October 2007)
+- libcurl now handles chunked-encoded CONNECT responses
+
 Daniel S (1 October 2007)
 - Alex Fishman reported a curl_easy_escape() problem that was made the
   function do wrong on all input bytes that are >= 0x80 (decimal 128) due to a

RELEASE-NOTES

@@ -29,6 +29,7 @@ This release includes the following bugfixes:
    before SIZE
  o re-used handle transfers with SFTP
  o curl_easy_escape() problem with byte values >= 128
+ o handles chunked-encoded CONNECT responses
 
 This release includes the following known bugs:
 

docs/KNOWN_BUGS

@@ -7,9 +7,6 @@ may have been fixed since this was written!
   to be kept alive, the function will return prematurely and will confuse the
   rest of the HTTP protocol code.
 
-46. If a CONNECT response is chunked-encoded, the function may return
-  prematurely and will confuse the rest of the HTTP protocol code.
-
 45. libcurl built to support ipv6 uses getaddrinfo() to resolve host names.
   getaddrinfo() sorts the response list which effectively kills how libcurl
   deals with round-robin DNS entries. All details:

lib/http.c

@@ -1143,6 +1143,7 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,
   curl_socket_t tunnelsocket = conn->sock[sockindex];
   curl_off_t cl=0;
   bool closeConnection = FALSE;
+  bool chunked_encoding = FALSE;
   long check;
 
 #define SELECT_OK      0
@@ -1340,15 +1341,35 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,
             nread += gotbytes;
 
             if(keepon > TRUE) {
-              /* This means we are currently ignoring a response-body, so we
+              /* This means we are currently ignoring a response-body */
-                 simply count down our counter and make sure to break out of
+              if(cl) {
-                 the loop when we're done! */
+                /* A Content-Length based body: simply count down the counter
+                   and make sure to break out of the loop when we're done! */
                 cl -= gotbytes;
                 if(cl<=0) {
                   keepon = FALSE;
                   break;
                 }
               }
+              else {
+                /* chunked-encoded body, so we need to do the chunked dance
+                   properly to know when the end of the body is reached */
+                CHUNKcode r;
+                ssize_t tookcareof=0;
+
+                /* now parse the chunked piece of data so that we can
+                   properly tell when the stream ends */
+                r = Curl_httpchunk_read(conn, ptr, gotbytes, &tookcareof);
+                if(r == CHUNKE_STOP) {
+                  /* we're done reading chunks! */
+                  infof(data, "chunk reading DONE\n");
+                  keepon = FALSE;
+                }
+                else
+                  infof(data, "Read %d bytes of chunk, continue\n",
+                        tookcareof);
+              }
+            }
             else
               for(i = 0; i < gotbytes; ptr++, i++) {
                 perline++; /* amount of bytes in this line so far */
@@ -1366,7 +1387,8 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,
                   if(data->set.include_header)
                     writetype |= CLIENTWRITE_BODY;
 
-                  result = Curl_client_write(conn, writetype, line_start, perline);
+                  result = Curl_client_write(conn, writetype, line_start,
+                                             perline);
                   if(result)
                     return result;
 
@@ -1377,20 +1399,61 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,
                   if(('\r' == line_start[0]) ||
                      ('\n' == line_start[0])) {
                     /* end of response-headers from the proxy */
-                    if(cl && (407 == k->httpcode) &&
+                    if((407 == k->httpcode) && !data->state.authproblem) {
-                       !data->state.authproblem) {
                       /* If we get a 407 response code with content length
-                       * when we have no auth problem, we must ignore the
+                         when we have no auth problem, we must ignore the
-                       * whole response-body */
+                         whole response-body */
                       keepon = 2;
+
+                      if(cl) {
+
                         infof(data, "Ignore %" FORMAT_OFF_T
                               " bytes of response-body\n", cl);
-                      cl -= (gotbytes - i);/* remove the remaining chunk of
+                        /* remove the remaining chunk of what we already
-                                              what we already read */
+                           read */
+                        cl -= (gotbytes - i);
+
                         if(cl<=0)
-                        /* if the whole thing was already read, we are done! */
+                          /* if the whole thing was already read, we are done!
+                           */
                           keepon=FALSE;
                       }
+                      else if(chunked_encoding) {
+                        CHUNKcode r;
+                        /* We set ignorebody true here since the chunked
+                           decoder function will acknowledge that. Pay
+                           attention so that this is cleared again when this
+                           function returns! */
+                        k->ignorebody = TRUE;
+                        infof(data, "%d bytes of chunk left\n", gotbytes-i);
+
+                        if(line_start[1] == '\n') {
+                          /* this can only be a LF if the letter at index 0
+                             was a CR */
+                          line_start++;
+                          i++;
+                        }
+
+                        /* now parse the chunked piece of data so that we can
+                           properly tell when the stream ends */
+                        r = Curl_httpchunk_read(conn, line_start+1,
+                                                  gotbytes -i, &gotbytes);
+                        if(r == CHUNKE_STOP) {
+                          /* we're done reading chunks! */
+                          infof(data, "chunk reading DONE\n");
+                          keepon = FALSE;
+                        }
+                        else
+                          infof(data, "Read %d bytes of chunk, continue\n",
+                                gotbytes);
+                      }
+                      else {
+                        /* without content-length or chunked encoding, we
+                           can't keep the connection alive since the close is
+                           the end signal so we bail out at once instead */
+                        keepon=FALSE;
+                      }
+                    }
                     else
                       keepon = FALSE;
                     break; /* breaks out of for-loop, not switch() */
@@ -1415,6 +1478,13 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,
                   else if(Curl_compareheader(line_start,
                                              "Connection:", "close"))
                     closeConnection = TRUE;
+                  else if(Curl_compareheader(line_start,
+                                             "Transfer-Encoding:", "chunked")) {
+                    infof(data, "CONNECT responded chunked\n");
+                    chunked_encoding = TRUE;
+                    /* init our chunky engine */
+                    Curl_httpchunk_init(conn);
+                  }
                   else if(Curl_compareheader(line_start,
                                              "Proxy-Connection:", "close"))
                     closeConnection = TRUE;
@@ -1472,6 +1542,7 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,
   data->state.authproxy.done = TRUE;
 
   infof (data, "Proxy replied OK to CONNECT request\n");
+  k->ignorebody = FALSE; /* put it (back) to non-ignore state */
   return CURLE_OK;
 }
 

lib/http_chunks.c

@@ -84,7 +84,7 @@
 
 void Curl_httpchunk_init(struct connectdata *conn)
 {
-  struct Curl_chunker *chunk = &conn->data->reqdata.proto.http->chunk;
+  struct Curl_chunker *chunk = &conn->chunk;
   chunk->hexindex=0; /* start at 0 */
   chunk->dataleft=0; /* no data left yet! */
   chunk->state = CHUNK_HEX; /* we get hex first! */
@@ -108,7 +108,7 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn,
 {
   CURLcode result=CURLE_OK;
   struct SessionHandle *data = conn->data;
-  struct Curl_chunker *ch = &data->reqdata.proto.http->chunk;
+  struct Curl_chunker *ch = &conn->chunk;
   struct Curl_transfer_keeper *k = &data->reqdata.keep;
   size_t piece;
   size_t length = (size_t)datalen;
@@ -403,7 +403,6 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn,
         return CHUNKE_BAD_CHUNK;
       }
 
-
     default:
       return CHUNKE_STATE_ERROR;
     }

lib/transfer.c

@@ -1262,7 +1262,7 @@ CURLcode Curl_readwrite(struct connectdata *conn,
                  We DO care about this data if we are pipelining.
                  Push it back to be read on the next pass. */
 
-              dataleft = data->reqdata.proto.http->chunk.dataleft;
+              dataleft = conn->chunk.dataleft;
               if (dataleft != 0) {
                 infof(conn->data, "Leftovers after chunking. "
                       " Rewinding %d bytes\n",dataleft);
@@ -1617,7 +1617,7 @@ CURLcode Curl_readwrite(struct connectdata *conn,
     }
     else if(!(conn->bits.no_body) &&
             conn->bits.chunk &&
-            (data->reqdata.proto.http->chunk.state != CHUNK_STOP)) {
+            (conn->chunk.state != CHUNK_STOP)) {
       /*
        * In chunked mode, return an error if the connection is closed prior to
        * the empty (terminiating) chunk is read.

lib/urldata.h

@@ -296,7 +296,6 @@ struct HTTP {
 
   /* For FORM posting */
   struct Form form;
-  struct Curl_chunker chunk;
 
   struct back {
     curl_read_callback fread_func; /* backup storage for fread pointer */
@@ -818,6 +817,11 @@ struct connectdata {
      connection is used! */
   struct SessionHandle *data;
 
+  /* chunk is for HTTP chunked encoding, but is in the general connectdata
+     struct only because we can do just about any protocol through a HTTP proxy
+     and a HTTP proxy may in fact respond using chunked encoding */
+  struct Curl_chunker chunk;
+
   bool inuse; /* This is a marker for the connection cache logic. If this is
                  TRUE this handle is being used by an easy handle and cannot
                  be used by any other easy handle without careful

tests/data/Makefile.am

@@ -44,7 +44,8 @@ EXTRA_DIST = test1 test108 test117 test127 test20 test27 test34 test46	   \
  test409 test613 test614 test700 test701 test702 test704 test705 test703   \
  test706 test707 test350 test351 test352 test353 test289 test540 test354   \
  test231 test1000 test1001 test1002 test1003 test1004 test1005 test1006    \
- test615 test1007 test541 test1010 test1011 test1012 test542 test543 test536
+ test615 test1007 test541 test1010 test1011 test1012 test542 test543       \
+ test536 test1008
 
 filecheck:
 	@mkdir test-place; \

tests/data/test1008

@@ -0,0 +1,112 @@
+<testcase>
+# Server-side
+<reply>
+
+# this is returned first since we get no proxy-auth
+<data1001>
+HTTP/1.1 407 Authorization Required to proxy me my dear
+Proxy-Authenticate: NTLM TlRMTVNTUAACAAAAAgACADAAAAAGgoEAc51AYVDgyNcAAAAAAAAAAG4AbgAyAAAAQ0MCAAQAQwBDAAEAEgBFAEwASQBTAEEAQgBFAFQASAAEABgAYwBjAC4AaQBjAGUAZABlAHYALgBuAHUAAwAsAGUAbABpAHMAYQBiAGUAdABoAC4AYwBjAC4AaQBjAGUAZABlAHYALgBuAHUAAAAAAA==
+Transfer-Encoding: chunked
+
+20
+And you should ignore this data.
+FA0
+XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+0
+
+</data1001>
+
+# This is supposed to be returned when the server gets the second
+# Authorization: NTLM line passed-in from the client
+<data1002>
+HTTP/1.1 200 Things are fine in proxy land
+Server: Microsoft-IIS/5.0
+Content-Type: text/html; charset=iso-8859-1
+
+</data1002>
+
+# this is returned when we get a GET!
+<data2>
+HTTP/1.1 200 OK
+Date: Thu, 09 Nov 2010 14:49:00 GMT
+Content-Length: 7
+Connection: close
+Content-Type: text/html
+Funny-head: yesyes
+
+daniel
+</data2>
+
+# then this is returned when we get proxy-auth
+<data1000>
+HTTP/1.1 200 OK swsbounce
+Server: no
+
+Nice proxy auth sir!
+</data1000>
+
+<datacheck>
+HTTP/1.1 407 Authorization Required to proxy me my dear
+Proxy-Authenticate: NTLM TlRMTVNTUAACAAAAAgACADAAAAAGgoEAc51AYVDgyNcAAAAAAAAAAG4AbgAyAAAAQ0MCAAQAQwBDAAEAEgBFAEwASQBTAEEAQgBFAFQASAAEABgAYwBjAC4AaQBjAGUAZABlAHYALgBuAHUAAwAsAGUAbABpAHMAYQBiAGUAdABoAC4AYwBjAC4AaQBjAGUAZABlAHYALgBuAHUAAAAAAA==
+Transfer-Encoding: chunked
+
+HTTP/1.1 200 Things are fine in proxy land
+Server: Microsoft-IIS/5.0
+Content-Type: text/html; charset=iso-8859-1
+
+HTTP/1.1 200 OK
+Date: Thu, 09 Nov 2010 14:49:00 GMT
+Content-Length: 7
+Connection: close
+Content-Type: text/html
+Funny-head: yesyes
+
+daniel
+</datacheck>
+</reply>
+
+# Client-side
+<client>
+<server>
+http
+</server>
+<features>
+NTLM
+</features>
+ <name>
+HTTP proxy CONNECT auth NTLM with chunked-encoded 407 response
+ </name>
+ <command>
+http://test.remote.server.com:1008/path/10080002 --proxy http://%HOSTIP:%HTTPPORT --proxy-user silly:person --proxy-ntlm --proxytunnel
+</command>
+</client>
+
+# Verify data after the test has been "shot"
+<verify>
+<strip>
+^User-Agent: curl/.*
+</strip>
+# We strip off a large chunk of the type-2 NTLM message since it depends on
+# the local host name and thus differs on different machines!
+<strippart>
+s/^(Proxy-Authorization: NTLM TlRMTVNTUAADAAAAGAAYAEAAAAAYABgAWAAAAAAAAABwAAAABQAFAHAAAAA).*/$1/
+</strippart>
+<protocol>
+CONNECT test.remote.server.com:1008 HTTP/1.0
+Host: test.remote.server.com:1008
+Proxy-Authorization: NTLM TlRMTVNTUAABAAAABoIIAAAAAAAAAAAAAAAAAAAAAAA=
+Proxy-Connection: Keep-Alive
+
+CONNECT test.remote.server.com:1008 HTTP/1.0
+Host: test.remote.server.com:1008
+Proxy-Authorization: NTLM TlRMTVNTUAADAAAAGAAYAEAAAAAYABgAWAAAAAAAAABwAAAABQAFAHAAAAA
+Proxy-Connection: Keep-Alive
+
+GET /path/10080002 HTTP/1.1
+User-Agent: curl/7.12.3-CVS (i686-pc-linux-gnu) libcurl/7.12.3-CVS OpenSSL/0.9.6b zlib/1.1.4
+Host: test.remote.server.com:1008
+Accept: */*
+
+</protocol>
+</verify>
+</testcase>