From 28b932fb4ef14b8b9ebda6823c98fbedad6be4b2 Mon Sep 17 00:00:00 2001 From: Daniel Stenberg Date: Mon, 12 Feb 2007 21:13:47 +0000 Subject: [PATCH] - Shmulik Regev fixed so that the final CRLF of HTTP response headers are sent to the debug callback. - Shmulik Regev added CURLOPT_HTTP_CONTENT_DECODING and CURLOPT_HTTP_TRANSFER_DECODING that if set to zero will disable libcurl's internal decoding of content or transfer encoded content. This may be preferable in cases where you use libcurl for proxy purposes or similar. The command line tool got a --raw option to disable both at once. --- CHANGES | 9 +++++++++ RELEASE-NOTES | 8 +++++--- docs/curl.1 | 7 ++++++- docs/libcurl/curl_easy_setopt.3 | 12 +++++++++++- include/curl/curl.h | 5 +++++ lib/http_chunks.c | 20 ++++++++++++++++---- lib/transfer.c | 6 +++++- lib/url.c | 13 +++++++++++++ lib/urldata.h | 4 ++++ src/main.c | 14 ++++++++++++-- 10 files changed, 86 insertions(+), 12 deletions(-) diff --git a/CHANGES b/CHANGES index 1d395b7a7..2e19fd2c4 100644 --- a/CHANGES +++ b/CHANGES @@ -7,6 +7,15 @@ Changelog Daniel (12 February 2007) +- Shmulik Regev fixed so that the final CRLF of HTTP response headers are sent + to the debug callback. + +- Shmulik Regev added CURLOPT_HTTP_CONTENT_DECODING and + CURLOPT_HTTP_TRANSFER_DECODING that if set to zero will disable libcurl's + internal decoding of content or transfer encoded content. This may be + preferable in cases where you use libcurl for proxy purposes or similar. The + command line tool got a --raw option to disable both at once. + - Jeff Pohlmeyer fixed a flaw in curl_multi_add_handle() when adding a handle that has an easy handle present in the "closure" list pending closure. diff --git a/RELEASE-NOTES b/RELEASE-NOTES index df3983db4..28d6623e3 100644 --- a/RELEASE-NOTES +++ b/RELEASE-NOTES @@ -2,8 +2,8 @@ Curl and libcurl 7.16.2 Public curl release number: 98 Releases counted from the very beginning: 125 - Available command line options: 115 - Available curl_easy_setopt() options: 137 + Available command line options: 116 + Available curl_easy_setopt() options: 141 Number of public functions in libcurl: 54 Amount of public web site mirrors: 39 Number of known libcurl bindings: 35 @@ -12,6 +12,8 @@ Curl and libcurl 7.16.2 This release includes the following changes: o Added CURLOPT_TIMEOUT_MS and CURLOPT_CONNECTTIMEOUT_MS + o Added CURLOPT_HTTP_CONTENT_DECODING, CURLOPT_HTTP_TRANSFER_DECODING and + --raw This release includes the following bugfixes: @@ -34,6 +36,6 @@ New curl mirrors: This release would not have looked like this without help, code, reports and advice from friends like these: - Yang Tse, Manfred Schwarb, Michael Wallner, Jeff Pohlmeyer + Yang Tse, Manfred Schwarb, Michael Wallner, Jeff Pohlmeyer, Shmulik Regev Thanks! (and sorry if I forgot to mention someone) diff --git a/docs/curl.1 b/docs/curl.1 index 021f9505f..cd69f08ff 100644 --- a/docs/curl.1 +++ b/docs/curl.1 @@ -21,7 +21,7 @@ .\" * $Id$ .\" ************************************************************************** .\" -.TH curl 1 "3 Nov 2006" "Curl 7.16.1" "Curl Manual" +.TH curl 1 "12 Feb 2007" "Curl 7.16.2" "Curl Manual" .SH NAME curl \- transfer a URL .SH SYNOPSIS @@ -901,6 +901,11 @@ FTP range downloads only support the simple syntax 'start-stop' (optionally with one of the numbers omitted). It depends on the non-RFC command SIZE. If this option is used several times, the last one will be used. +.IP "--raw" +When used, it disables all internal HTTP decoding of content or transfer +encodings and instead makes them passed on unaltered, raw. (Added in 7.16.2) + +If this option is used several times, each occurrence toggles this on/off. .IP "-R/--remote-time" When used, this will make libcurl attempt to figure out the timestamp of the remote file, and if that is available make the local file get that same diff --git a/docs/libcurl/curl_easy_setopt.3 b/docs/libcurl/curl_easy_setopt.3 index 23d012de9..0c55e1400 100644 --- a/docs/libcurl/curl_easy_setopt.3 +++ b/docs/libcurl/curl_easy_setopt.3 @@ -21,7 +21,7 @@ .\" * $Id$ .\" ************************************************************************** .\" -.TH curl_easy_setopt 3 "5 Feb 2007" "libcurl 7.16.2" "libcurl Manual" +.TH curl_easy_setopt 3 "12 Feb 2007" "libcurl 7.16.2" "libcurl Manual" .SH NAME curl_easy_setopt \- set options for a curl easy handle .SH SYNOPSIS @@ -815,6 +815,16 @@ servers) which will report incorrect content length for files over 2 gigabytes. If this option is used, curl will not be able to accurately report progress, and will simply stop the download when the server ends the connection. (added in 7.14.1) +.IP CURLOPT_HTTP_CONTENT_DECODING +Pass a long to tell libcurl how to act on content decoding. If set to zero, +content decoding will be disabled. If set to 1 it is enabled. Note however +that libcurl has no default content decoding but requires you to use +\fICURLOPT_ENCODING\fP for that. (added in 7.16.2) +.IP CURLOPT_HTTP_TRANSFER_DECODING +Pass a long to tell libcurl how to act on transfer decoding. If set to zero, +transfer decoding will be disabled, if set to 1 it is enabled +(default). libcurl does chunked transfer decoding by default unless this +option is set to zero. (added in 7.16.2) .RE .SH FTP OPTIONS .IP CURLOPT_FTPPORT diff --git a/include/curl/curl.h b/include/curl/curl.h index 2c79f7dac..e911fca5a 100644 --- a/include/curl/curl.h +++ b/include/curl/curl.h @@ -1058,6 +1058,11 @@ typedef enum { CINIT(TIMEOUT_MS, LONG, 155), CINIT(CONNECTTIMEOUT_MS, LONG, 156), + /* set to zero to disable the libcurl's decoding and thus pass the raw body + data to the appliction even when it is encoded/compressed */ + CINIT(HTTP_TRANSFER_DECODING, LONG, 157), + CINIT(HTTP_CONTENT_DECODING, LONG, 158), + CURLOPT_LASTENTRY /* the last unused */ } CURLoption; diff --git a/lib/http_chunks.c b/lib/http_chunks.c index 1b03a5569..36bee789c 100644 --- a/lib/http_chunks.c +++ b/lib/http_chunks.c @@ -116,6 +116,12 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn, *wrote = 0; /* nothing's written yet */ + /* the original data is written to the client, but we go on with the + chunk read process, to properly calculate the content length*/ + if ( data->set.http_te_skip ) + Curl_client_write(conn, CLIENTWRITE_BODY, datap,datalen); + + while(length) { switch(ch->state) { case CHUNK_HEX: @@ -206,12 +212,17 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn, /* Write the data portion available */ #ifdef HAVE_LIBZ - switch (data->reqdata.keep.content_encoding) { + switch (conn->data->set.http_ce_skip? + IDENTITY : data->reqdata.keep.content_encoding) { case IDENTITY: #endif - if(!k->ignorebody) - result = Curl_client_write(conn, CLIENTWRITE_BODY, datap, - piece); + if(!k->ignorebody) { + if ( !data->set.http_te_skip ) + result = Curl_client_write(conn, CLIENTWRITE_BODY, datap, + piece); + else + result = CURLE_OK; + } #ifdef HAVE_LIBZ break; @@ -334,6 +345,7 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn, return(CHUNKE_BAD_CHUNK); } #endif /* CURL_DOES_CONVERSIONS */ + if ( !data->set.http_te_skip ) Curl_client_write(conn, CLIENTWRITE_HEADER, conn->trailer, conn->trlPos); } diff --git a/lib/transfer.c b/lib/transfer.c index 828108204..cf4264cc8 100644 --- a/lib/transfer.c +++ b/lib/transfer.c @@ -689,6 +689,9 @@ CURLcode Curl_readwrite(struct connectdata *conn, k->keepon &= ~KEEP_READ; } + if(data->set.verbose) + Curl_debug(data, CURLINFO_HEADER_IN, + k->str_start, headerlen, conn); break; /* exit header line loop */ } @@ -1286,7 +1289,8 @@ CURLcode Curl_readwrite(struct connectdata *conn, Make sure that ALL_CONTENT_ENCODINGS contains all the encodings handled here. */ #ifdef HAVE_LIBZ - switch (k->content_encoding) { + switch (conn->data->set.http_ce_skip ? + IDENTITY : k->content_encoding) { case IDENTITY: #endif /* This is the default when the server sends no diff --git a/lib/url.c b/lib/url.c index 076f50ebd..148d7b2bb 100644 --- a/lib/url.c +++ b/lib/url.c @@ -1725,6 +1725,19 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option, data->set.ssh_private_key = va_arg(param, char *); break; + case CURLOPT_HTTP_TRANSFER_DECODING: + /* + * disable libcurl transfer encoding is used + */ + data->set.http_te_skip = (bool)(0 == va_arg(param, long)); + break; + + case CURLOPT_HTTP_CONTENT_DECODING: + /* + * raw data passed to the application when content encoding is used + */ + data->set.http_ce_skip = (bool)(0 == va_arg(param, long)); + break; default: /* unknown tag and its companion, just ignore: */ result = CURLE_FAILED_INIT; /* correct this */ diff --git a/lib/urldata.h b/lib/urldata.h index acc9d1ba6..3ba7fcacd 100644 --- a/lib/urldata.h +++ b/lib/urldata.h @@ -1292,6 +1292,10 @@ struct UserDefined { authentication */ char *ssh_private_key; /* the path to the private key file for authentication */ + bool http_te_skip; /* pass the raw body data to the user, even when + transfer-encoded (chunked, compressed) */ + bool http_ce_skip; /* pass the raw body data to the user, even when + content-encoded (chunked, compressed) */ }; struct Names { diff --git a/src/main.c b/src/main.c index e35050025..d0b885151 100644 --- a/src/main.c +++ b/src/main.c @@ -20,7 +20,6 @@ * * $Id$ ***************************************************************************/ - #include "setup.h" #include @@ -470,7 +469,7 @@ struct Configurable { bool disable_sessionid; char *libcurl; /* output libcurl code to this file name */ - + bool raw; struct OutStruct *outs; }; @@ -683,6 +682,7 @@ static void help(void) " -Q/--quote Send command(s) to server before file transfer (F)", " -r/--range Retrieve a byte range from a HTTP/1.1 or FTP server", " --random-file File for reading random data from (SSL)", + " --raw Pass HTTP \"raw\", without any transfer decoding (H)", " -R/--remote-time Set the remote file's time on the local output", " --retry Retry request times if transient problems occur", " --retry-delay When retrying, wait this many seconds between each", @@ -1473,6 +1473,7 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */ {"$x", "ftp-ssl-control", FALSE}, {"$y", "ftp-ssl-ccc", FALSE}, {"$z", "libcurl", TRUE}, + {"$#", "raw", FALSE}, {"0", "http1.0", FALSE}, {"1", "tlsv1", FALSE}, @@ -1903,6 +1904,9 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */ case 'z': /* --libcurl */ GetStr(&config->libcurl, nextarg); break; + case '#': /* --raw */ + config->raw ^= TRUE; + break; } break; case '#': /* --progress-bar */ @@ -4253,6 +4257,12 @@ operate(struct Configurable *config, int argc, char *argv[]) my_setopt(curl, CURLOPT_SSL_SESSIONID_CACHE, !config->disable_sessionid); + /* curl 7.16.2 */ + if(config->raw) { + my_setopt(curl, CURLOPT_HTTP_CONTENT_DECODING, FALSE); + my_setopt(curl, CURLOPT_HTTP_TRANSFER_DECODING, FALSE); + } + retry_numretries = config->req_retry; retrystart = curlx_tvnow();