diff --git a/lib/Makefile.inc b/lib/Makefile.inc index fcb8c28ec..f97ef6d0e 100644 --- a/lib/Makefile.inc +++ b/lib/Makefile.inc @@ -23,8 +23,9 @@ CSOURCES = file.c timeval.c base64.c hostip.c progress.c formdata.c \ curl_rtmp.c openldap.c curl_gethostname.c gopher.c axtls.c \ idn_win32.c http_negotiate_sspi.c cyassl.c http_proxy.c non-ascii.c \ asyn-ares.c asyn-thread.c curl_gssapi.c curl_ntlm.c curl_ntlm_wb.c \ - curl_ntlm_core.c curl_ntlm_msgs.c curl_sasl.c curl_schannel.c \ - curl_multibyte.c curl_darwinssl.c hostcheck.c + curl_ntlm_core.c curl_ntlm_msgs.c curl_sasl.c curl_schannel.c \ + curl_multibyte.c curl_darwinssl.c hostcheck.c \ + bundles.c conncache.c HHEADERS = arpa_telnet.h netrc.h file.h timeval.h qssl.h hostip.h \ progress.h formdata.h cookie.h http.h sendf.h ftp.h url.h dict.h \ @@ -42,4 +43,4 @@ HHEADERS = arpa_telnet.h netrc.h file.h timeval.h qssl.h hostip.h \ gopher.h axtls.h cyassl.h http_proxy.h non-ascii.h asyn.h curl_ntlm.h \ curl_gssapi.h curl_ntlm_wb.h curl_ntlm_core.h curl_ntlm_msgs.h \ curl_sasl.h curl_schannel.h curl_multibyte.h curl_darwinssl.h \ - hostcheck.h + hostcheck.h bundles.h conncache.h diff --git a/lib/bundles.c b/lib/bundles.c new file mode 100644 index 000000000..046e3bb3b --- /dev/null +++ b/lib/bundles.c @@ -0,0 +1,101 @@ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 2012, Linus Nielsen Feltzing, + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +#include "setup.h" + +#include + +#include "urldata.h" +#include "url.h" +#include "progress.h" +#include "multiif.h" +#include "bundles.h" +#include "sendf.h" +#include "rawstr.h" + +#include "curl_memory.h" +/* The last #include file should be: */ +#include "memdebug.h" + +static void conn_llist_dtor(void *user, void *element) +{ + struct connectdata *data = element; + (void)user; + + data->bundle = NULL; +} + +CURLcode Curl_bundle_create(struct SessionHandle *data, + struct connectbundle **cb_ptr) +{ + (void)data; + *cb_ptr = malloc(sizeof(struct connectbundle)); + if(!*cb_ptr) + return CURLE_OUT_OF_MEMORY; + + (*cb_ptr)->num_connections = 0; + (*cb_ptr)->server_supports_pipelining = FALSE; + + (*cb_ptr)->conn_list = Curl_llist_alloc((curl_llist_dtor) conn_llist_dtor); + if(!(*cb_ptr)->conn_list) + return CURLE_OUT_OF_MEMORY; + return CURLE_OK; +} + +void Curl_bundle_destroy(struct connectbundle *cb_ptr) +{ + if(cb_ptr->conn_list) + Curl_llist_destroy(cb_ptr->conn_list, NULL); + Curl_safefree(cb_ptr); +} + +/* Add a connection to a bundle */ +CURLcode Curl_bundle_add_conn(struct connectbundle *cb_ptr, + struct connectdata *conn) +{ + if(!Curl_llist_insert_next(cb_ptr->conn_list, cb_ptr->conn_list->tail, conn)) + return CURLE_OUT_OF_MEMORY; + + conn->bundle = cb_ptr; + + cb_ptr->num_connections++; + return CURLE_OK; +} + +/* Remove a connection from a bundle */ +int Curl_bundle_remove_conn(struct connectbundle *cb_ptr, + struct connectdata *conn) +{ + struct curl_llist_element *curr; + + curr = cb_ptr->conn_list->head; + while(curr) { + if(curr->ptr == conn) { + Curl_llist_remove(cb_ptr->conn_list, curr, NULL); + cb_ptr->num_connections--; + conn->bundle = NULL; + return 1; /* we removed a handle */ + } + curr = curr->next; + } + return 0; +} diff --git a/lib/bundles.h b/lib/bundles.h new file mode 100644 index 000000000..e9e257fc9 --- /dev/null +++ b/lib/bundles.h @@ -0,0 +1,44 @@ +#ifndef __BUNDLES_H +#define __BUNDLES_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 2012, Linus Nielsen Feltzing, + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +struct connectbundle { + bool server_supports_pipelining; /* TRUE if server supports pipelining, + set after first response */ + size_t num_connections; /* Number of connections in the bundle */ + struct curl_llist *conn_list; /* The connectdata members of the bundle */ +}; + +CURLcode Curl_bundle_create(struct SessionHandle *data, + struct connectbundle **cb_ptr); + +void Curl_bundle_destroy(struct connectbundle *cb_ptr); + +CURLcode Curl_bundle_add_conn(struct connectbundle *cb_ptr, + struct connectdata *conn); + +int Curl_bundle_remove_conn(struct connectbundle *cb_ptr, + struct connectdata *conn); + + +#endif /* __BUNDLES_H */ diff --git a/lib/conncache.c b/lib/conncache.c new file mode 100644 index 000000000..b3186037e --- /dev/null +++ b/lib/conncache.c @@ -0,0 +1,270 @@ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 2012, Linus Nielsen Feltzing, + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +#include "setup.h" + +#include + +#include "urldata.h" +#include "url.h" +#include "progress.h" +#include "multiif.h" +#include "sendf.h" +#include "rawstr.h" +#include "bundles.h" +#include "conncache.h" + +#include "curl_memory.h" +/* The last #include file should be: */ +#include "memdebug.h" + +#define CONNECTION_HASH_SIZE 97 + +static void free_bundle_hash_entry(void *freethis) +{ + struct connectbundle *b = (struct connectbundle *) freethis; + + Curl_bundle_destroy(b); +} + +struct conncache *Curl_conncache_init(int type) +{ + struct conncache *connc; + + connc = calloc(1, sizeof(struct conncache)); + if(!connc) + return NULL; + + connc->hash = Curl_hash_alloc(CONNECTION_HASH_SIZE, Curl_hash_str, + Curl_str_key_compare, free_bundle_hash_entry); + + if(!connc->hash) { + free(connc); + return NULL; + } + + connc->type = type; + connc->num_connections = 0; + + return connc; +} + +void Curl_conncache_destroy(struct conncache *connc) +{ + Curl_hash_destroy(connc->hash); + free(connc); +} + +struct connectbundle *Curl_conncache_find_bundle(struct conncache *connc, + char *hostname) +{ + struct connectbundle *bundle = NULL; + + if(connc) + bundle = Curl_hash_pick(connc->hash, hostname, strlen(hostname)+1); + + return bundle; +} + +static bool conncache_add_bundle(struct conncache *connc, + char *hostname, + struct connectbundle *bundle) +{ + void *p; + + p = Curl_hash_add(connc->hash, hostname, strlen(hostname)+1, bundle); + + return p?TRUE:FALSE; +} + +static void conncache_remove_bundle(struct conncache *connc, + struct connectbundle *bundle) +{ + struct curl_hash_iterator iter; + struct curl_hash_element *he; + + if(!connc) + return; + + Curl_hash_start_iterate(connc->hash, &iter); + + he = Curl_hash_next_element(&iter); + while(he) { + if(he->ptr == bundle) { + /* The bundle is destroyed by the hash destructor function, + free_bundle_hash_entry() */ + Curl_hash_delete(connc->hash, he->key, he->key_len); + return; + } + + he = Curl_hash_next_element(&iter); + } +} + +CURLcode Curl_conncache_add_conn(struct conncache *connc, + struct connectdata *conn) +{ + CURLcode result; + struct connectbundle *bundle; + struct SessionHandle *data = conn->data; + + bundle = Curl_conncache_find_bundle(data->state.conn_cache, + conn->host.name); + if(!bundle) { + result = Curl_bundle_create(data, &bundle); + if(result != CURLE_OK) + return result; + + if(!conncache_add_bundle(data->state.conn_cache, + conn->host.name, bundle)) + return CURLE_OUT_OF_MEMORY; + } + + result = Curl_bundle_add_conn(bundle, conn); + if(result != CURLE_OK) + return result; + + connc->num_connections++; + + return CURLE_OK; +} + +void Curl_conncache_remove_conn(struct conncache *connc, + struct connectdata *conn) +{ + struct connectbundle *bundle = conn->bundle; + + /* The bundle pointer can be NULL, since this function can be called + due to a failed connection attempt, before being added to a bundle */ + if(bundle) { + Curl_bundle_remove_conn(bundle, conn); + if(bundle->num_connections == 0) { + conncache_remove_bundle(connc, bundle); + } + connc->num_connections--; + + DEBUGF(infof(conn->data, "The cache now contains %d members\n", + connc->num_connections)); + } +} + +/* This function iterates the entire connection cache and calls the + function func() with the connection pointer as the first argument + and the supplied 'param' argument as the other */ +void Curl_conncache_foreach(struct conncache *connc, + void *param, + void (*func)(void *conn, void *param)) +{ + struct curl_hash_iterator iter; + struct curl_llist_element *curr; + struct curl_hash_element *he; + + if(!connc) + return; + + Curl_hash_start_iterate(connc->hash, &iter); + + he = Curl_hash_next_element(&iter); + while(he) { + struct connectbundle *bundle; + struct connectdata *conn; + + bundle = he->ptr; + + curr = bundle->conn_list->head; + while(curr) { + /* Yes, we need to update curr before calling func(), because func() + might decide to remove the connection */ + conn = curr->ptr; + curr = curr->next; + + func(conn, param); + } + + he = Curl_hash_next_element(&iter); + } +} + +/* Return the first connection found in the cache. Used when closing all + connections */ +struct connectdata * +Curl_conncache_find_first_connection(struct conncache *connc) +{ + struct curl_hash_iterator iter; + struct curl_llist_element *curr; + struct curl_hash_element *he; + struct connectbundle *bundle; + + Curl_hash_start_iterate(connc->hash, &iter); + + he = Curl_hash_next_element(&iter); + while(he) { + bundle = he->ptr; + + curr = bundle->conn_list->head; + if(curr) { + return curr->ptr; + } + + he = Curl_hash_next_element(&iter); + } + + return NULL; +} + + +#if 0 +/* Useful for debugging the connection cache */ +void Curl_conncache_print(struct conncache *connc) +{ + struct curl_hash_iterator iter; + struct curl_llist_element *curr; + struct curl_hash_element *he; + + if(!connc) + return; + + fprintf(stderr, "=Bundle cache=\n"); + + Curl_hash_start_iterate(connc->hash, &iter); + + he = Curl_hash_next_element(&iter); + while(he) { + struct connectbundle *bundle; + struct connectdata *conn; + + bundle = he->ptr; + + fprintf(stderr, "%s -", he->key); + curr = bundle->conn_list->head; + while(curr) { + conn = curr->ptr; + + fprintf(stderr, " [%p %d]", (void *)conn, conn->inuse); + curr = curr->next; + } + fprintf(stderr, "\n"); + + he = Curl_hash_next_element(&iter); + } +} +#endif diff --git a/lib/conncache.h b/lib/conncache.h new file mode 100644 index 000000000..a6e1af70c --- /dev/null +++ b/lib/conncache.h @@ -0,0 +1,56 @@ +#ifndef __CONNCACHE_H +#define __CONNCACHE_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 2012, Linus Nielsen Feltzing, + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +struct conncache { + struct curl_hash *hash; + enum { + CONNCACHE_PRIVATE, /* used for an easy handle alone */ + CONNCACHE_MULTI /* shared within a multi handle */ + } type; + size_t num_connections; +}; + +struct conncache *Curl_conncache_init(int type); + +void Curl_conncache_destroy(struct conncache *connc); + +struct connectbundle *Curl_conncache_find_bundle(struct conncache *connc, + char *hostname); + +CURLcode Curl_conncache_add_conn(struct conncache *connc, + struct connectdata *conn); + +void Curl_conncache_remove_conn(struct conncache *connc, + struct connectdata *conn); + +void Curl_conncache_foreach(struct conncache *connc, + void *param, + void (*func)(void *, void *)); + +struct connectdata * +Curl_conncache_find_first_connection(struct conncache *connc); + +void Curl_conncache_print(struct conncache *connc); + +#endif /* __CONNCACHE_H */ diff --git a/lib/connect.c b/lib/connect.c index 1651e6e20..f615d81e9 100644 --- a/lib/connect.c +++ b/lib/connect.c @@ -1131,10 +1131,8 @@ curl_socket_t Curl_getconnectinfo(struct SessionHandle *data, DEBUGASSERT(data); - if((data->state.lastconnect != -1) && - (data->state.connc->connects[data->state.lastconnect] != NULL)) { - struct connectdata *c = - data->state.connc->connects[data->state.lastconnect]; + if(data->state.lastconnect) { + struct connectdata *c = data->state.lastconnect; if(connp) /* only store this if the caller cares for it */ *connp = c; diff --git a/lib/easy.c b/lib/easy.c index 6e8ff770a..100d003f3 100644 --- a/lib/easy.c +++ b/lib/easy.c @@ -70,6 +70,7 @@ #include "curl_rand.h" #include "non-ascii.h" #include "warnless.h" +#include "conncache.h" #define _MPRINTF_REPLACE /* use our functions only */ #include @@ -526,10 +527,10 @@ CURLcode curl_easy_perform(CURL *curl) } - if(!data->state.connc) { - /* oops, no connection cache, make one up */ - data->state.connc = Curl_mk_connc(CONNCACHE_PRIVATE, -1L); - if(!data->state.connc) + if(!data->state.conn_cache) { + /* Oops, no connection cache, create one */ + data->state.conn_cache = Curl_conncache_init(CONNCACHE_PRIVATE); + if(!data->state.conn_cache) return CURLE_OUT_OF_MEMORY; } @@ -616,9 +617,9 @@ CURL *curl_easy_duphandle(CURL *incurl) goto fail; /* the connection cache is setup on demand */ - outcurl->state.connc = NULL; + outcurl->state.conn_cache = NULL; - outcurl->state.lastconnect = -1; + outcurl->state.lastconnect = NULL; outcurl->progress.flags = data->progress.flags; outcurl->progress.callback = data->progress.callback; @@ -674,11 +675,6 @@ CURL *curl_easy_duphandle(CURL *incurl) fail: if(outcurl) { - if(outcurl->state.connc && - (outcurl->state.connc->type == CONNCACHE_PRIVATE)) { - Curl_rm_connc(outcurl->state.connc); - outcurl->state.connc = NULL; - } curl_slist_free_all(outcurl->change.cookielist); outcurl->change.cookielist = NULL; Curl_safefree(outcurl->state.headerbuff); diff --git a/lib/hash.c b/lib/hash.c index 4d85188fb..585285b03 100644 --- a/lib/hash.c +++ b/lib/hash.c @@ -322,34 +322,77 @@ size_t Curl_str_key_compare(void*k1, size_t key1_len, void*k2, size_t key2_len) return 0; } +void Curl_hash_start_iterate(struct curl_hash *hash, + struct curl_hash_iterator *iter) +{ + iter->hash = hash; + iter->slot_index = 0; + iter->current_element = NULL; +} + +struct curl_hash_element * +Curl_hash_next_element(struct curl_hash_iterator *iter) +{ + int i; + struct curl_hash *h = iter->hash; + + /* Get the next element in the current list, if any */ + if(iter->current_element) + iter->current_element = iter->current_element->next; + + /* If we have reached the end of the list, find the next one */ + if(!iter->current_element) { + for(i = iter->slot_index;i < h->slots;i++) { + if(h->table[i]->head) { + iter->current_element = h->table[i]->head; + iter->slot_index = i+1; + break; + } + } + } + + if(iter->current_element) { + struct curl_hash_element *he = iter->current_element->ptr; + return he; + } + else { + iter->current_element = NULL; + return NULL; + } +} + #if 0 /* useful function for debugging hashes and their contents */ void Curl_hash_print(struct curl_hash *h, void (*func)(void *)) { - int i; - struct curl_llist_element *le; - struct curl_llist *list; - struct curl_hash_element *he; + struct curl_hash_iterator iter; + struct curl_hash_element *he; + int last_index = -1; + if(!h) return; fprintf(stderr, "=Hash dump=\n"); - for(i = 0; i < h->slots; i++) { - list = h->table[i]; - le = list->head; /* get first list entry */ - if(le) { - fprintf(stderr, "index %d:", i); - while(le) { - he = le->ptr; - if(func) - func(he->ptr); - else - fprintf(stderr, " [%p]", he->ptr); - le = le->next; + Curl_hash_start_iterate(h, &iter); + + he = Curl_hash_next_element(&iter); + while(he) { + if(iter.slot_index != last_index) { + fprintf(stderr, "index %d:", iter.slot_index); + if(last_index >= 0) { + fprintf(stderr, "\n"); } - fprintf(stderr, "\n"); + last_index = iter.slot_index; } + + if(func) + func(he->ptr); + else + fprintf(stderr, " [%p]", he->ptr); + + he = Curl_hash_next_element(&iter); } + fprintf(stderr, "\n"); } #endif diff --git a/lib/hash.h b/lib/hash.h index 993aaedd2..5f7c2bf56 100644 --- a/lib/hash.h +++ b/lib/hash.h @@ -62,6 +62,11 @@ struct curl_hash_element { size_t key_len; }; +struct curl_hash_iterator { + struct curl_hash *hash; + int slot_index; + struct curl_llist_element *current_element; +}; int Curl_hash_init(struct curl_hash *h, int slots, @@ -89,4 +94,13 @@ size_t Curl_hash_str(void* key, size_t key_length, size_t slots_num); size_t Curl_str_key_compare(void*k1, size_t key1_len, void*k2, size_t key2_len); +void Curl_hash_start_iterate(struct curl_hash *hash, + struct curl_hash_iterator *iter); +struct curl_hash_element * +Curl_hash_next_element(struct curl_hash_iterator *iter); + +void Curl_hash_print(struct curl_hash *h, + void (*func)(void *)); + + #endif diff --git a/lib/http.c b/lib/http.c index 965b37503..0b6d7d4c5 100644 --- a/lib/http.c +++ b/lib/http.c @@ -3170,6 +3170,7 @@ CURLcode Curl_http_readwrite_headers(struct SessionHandle *data, } else if(conn->httpversion >= 11 && !conn->bits.close) { + /* If HTTP version is >= 1.1 and connection is persistent server supports pipelining. */ DEBUGF(infof(data, diff --git a/lib/multi.c b/lib/multi.c index d2d154fa7..e309c0d96 100644 --- a/lib/multi.c +++ b/lib/multi.c @@ -44,6 +44,8 @@ #include "select.h" #include "warnless.h" #include "speedcheck.h" +#include "conncache.h" +#include "bundles.h" #define _MPRINTF_REPLACE /* use our functions only */ #include @@ -97,11 +99,6 @@ typedef enum { #define GETSOCK_READABLE (0x00ff) #define GETSOCK_WRITABLE (0xff00) -struct closure { - struct closure *next; /* a simple one-way list of structs */ - struct SessionHandle *easy_handle; -}; - struct Curl_one_easy { /* first, two fields for the linked list of these */ struct Curl_one_easy *next; @@ -164,14 +161,16 @@ struct Curl_multi { /* Whether pipelining is enabled for this multi handle */ bool pipelining_enabled; - /* shared connection cache */ - struct conncache *connc; + /* Shared connection cache (bundles)*/ + struct conncache *conn_cache; + + /* This handle will be used for closing the cached connections in + curl_multi_cleanup() */ + struct SessionHandle *closure_handle; + long maxconnects; /* if >0, a fixed limit of the maximum number of entries we're allowed to grow the connection cache to */ - /* list of easy handles kept around for doing nice connection closures */ - struct closure *closure; - /* timer callback and user data pointer for the *socket() API */ curl_multi_timer_callback timer_cb; void *timer_userp; @@ -179,12 +178,8 @@ struct Curl_multi { previous callback */ }; -static void multi_connc_remove_handle(struct Curl_multi *multi, - struct SessionHandle *data); static void singlesocket(struct Curl_multi *multi, struct Curl_one_easy *easy); -static CURLMcode add_closure(struct Curl_multi *multi, - struct SessionHandle *data); static int update_timer(struct Curl_multi *multi); static CURLcode addHandleToSendOrPendPipeline(struct SessionHandle *handle, @@ -228,7 +223,7 @@ static void multi_freetimeout(void *a, void *b); static void multistate(struct Curl_one_easy *easy, CURLMstate state) { #ifdef DEBUGBUILD - long connectindex = -5000; + long connection_id = -5000; #endif CURLMstate oldstate = easy->state; @@ -242,12 +237,12 @@ static void multistate(struct Curl_one_easy *easy, CURLMstate state) if(easy->easy_conn) { if(easy->state > CURLM_STATE_CONNECT && easy->state < CURLM_STATE_COMPLETED) - connectindex = easy->easy_conn->connectindex; + connection_id = easy->easy_conn->connection_id; infof(easy->easy_handle, "STATE: %s => %s handle %p; (connection #%ld) \n", statename[oldstate], statename[easy->state], - (char *)easy, connectindex); + (char *)easy, connection_id); } #endif if(state == CURLM_STATE_COMPLETED) @@ -391,7 +386,6 @@ static void multi_freeamsg(void *a, void *b) (void)b; } - CURLM *curl_multi_init(void) { struct Curl_multi *multi = calloc(1, sizeof(struct Curl_multi)); @@ -409,8 +403,8 @@ CURLM *curl_multi_init(void) if(!multi->sockhash) goto error; - multi->connc = Curl_mk_connc(CONNCACHE_MULTI, -1L); - if(!multi->connc) + multi->conn_cache = Curl_conncache_init(CONNCACHE_MULTI); + if(!multi->conn_cache) goto error; multi->msglist = Curl_llist_alloc(multi_freeamsg); @@ -431,8 +425,8 @@ CURLM *curl_multi_init(void) multi->sockhash = NULL; Curl_hash_destroy(multi->hostcache); multi->hostcache = NULL; - Curl_rm_connc(multi->connc); - multi->connc = NULL; + Curl_conncache_destroy(multi->conn_cache); + multi->conn_cache = NULL; free(multi); return NULL; @@ -443,8 +437,6 @@ CURLMcode curl_multi_add_handle(CURLM *multi_handle, { struct curl_llist *timeoutlist; struct Curl_one_easy *easy; - struct closure *cl; - struct closure *prev = NULL; struct Curl_multi *multi = (struct Curl_multi *)multi_handle; struct SessionHandle *data = (struct SessionHandle *)easy_handle; @@ -462,22 +454,13 @@ CURLMcode curl_multi_add_handle(CURLM *multi_handle, /* possibly we should create a new unique error code for this condition */ return CURLM_BAD_EASY_HANDLE; - /* We want the connection cache to have plenty of room. Before we supported - the shared cache every single easy handle had 5 entries in their cache - by default. */ - if(((multi->num_easy + 1) * 4) > multi->connc->num) { - long newmax = (multi->num_easy + 1) * 4; - - if(multi->maxconnects && (newmax > multi->maxconnects)) - /* don't grow beyond the allowed size */ - newmax = multi->maxconnects; - - if(newmax > multi->connc->num) { - /* we only do this is we can in fact grow the cache */ - CURLcode res = Curl_ch_connc(data, multi->connc, newmax); - if(res) - return CURLM_OUT_OF_MEMORY; - } + /* This is a good time to allocate a fresh easy handle to use when closing + cached connections */ + if(!multi->closure_handle) { + multi->closure_handle = + (struct SessionHandle *)curl_easy_init(); + Curl_easy_addmulti(easy_handle, multi_handle); + multi->closure_handle->state.conn_cache = multi->conn_cache; } /* Allocate and initialize timeout list for easy handle */ @@ -504,27 +487,6 @@ CURLMcode curl_multi_add_handle(CURLM *multi_handle, data->state.timeoutlist = timeoutlist; timeoutlist = NULL; - /* Remove handle from the list of 'closure handles' in case it is there */ - cl = multi->closure; - while(cl) { - struct closure *next = cl->next; - if(cl->easy_handle == data) { - /* Remove node from list */ - free(cl); - if(prev) - prev->next = next; - else - multi->closure = next; - /* removed from closure list now, this might reuse an existing - existing connection but we don't know that at this point */ - data->state.shared_conn = NULL; - /* No need to continue, handle can only be present once in the list */ - break; - } - prev = cl; - cl = next; - } - /* set the easy handle */ easy->easy_handle = data; multistate(easy, CURLM_STATE_INIT); @@ -548,16 +510,15 @@ CURLMcode curl_multi_add_handle(CURLM *multi_handle, } /* On a multi stack the connection cache, owned by the multi handle, - is shared between all easy handles within the multi handle. */ - if(easy->easy_handle->state.connc && - (easy->easy_handle->state.connc->type == CONNCACHE_PRIVATE)) { - /* kill old private connection cache */ - Curl_rm_connc(easy->easy_handle->state.connc); - easy->easy_handle->state.connc = NULL; + is shared between all easy handles within the multi handle. + Therefore we free the private connection cache if there is one */ + if(easy->easy_handle->state.conn_cache && + easy->easy_handle->state.conn_cache->type == CONNCACHE_PRIVATE) { + Curl_conncache_destroy(easy->easy_handle->state.conn_cache); } + /* Point now to this multi's connection cache */ - easy->easy_handle->state.connc = multi->connc; - easy->easy_handle->state.connc->type = CONNCACHE_MULTI; + easy->easy_handle->state.conn_cache = multi->conn_cache; /* This adds the new entry at the 'end' of the doubly-linked circular list of Curl_one_easy structs to try and maintain a FIFO queue so @@ -701,42 +662,17 @@ CURLMcode curl_multi_remove_handle(CURLM *multi_handle, Note that this ignores the return code simply because there's nothing really useful to do with it anyway! */ (void)Curl_done(&easy->easy_conn, easy->result, premature); - - if(easy->easy_conn) - /* the connection is still alive, set back the association to enable - the check below to trigger TRUE */ - easy->easy_conn->data = easy->easy_handle; } else /* Clear connection pipelines, if Curl_done above was not called */ Curl_getoff_all_pipelines(easy->easy_handle, easy->easy_conn); } - /* figure out if the easy handle is used by one or more connections in the - cache */ - multi_connc_remove_handle(multi, easy->easy_handle); - - if(easy->easy_handle->state.connc->type == CONNCACHE_MULTI) { + if(easy->easy_handle->state.conn_cache->type == CONNCACHE_MULTI) { /* if this was using the shared connection cache we clear the pointer to that since we're not part of that handle anymore */ - easy->easy_handle->state.connc = NULL; - - /* Since we return the connection back to the communal connection pool - we mark the last connection as inaccessible */ - easy->easy_handle->state.lastconnect = -1; - - /* Modify the connectindex since this handle can't point to the - connection cache anymore. - - TODO: consider if this is really what we want. The connection cache - is within the multi handle and that owns the connections so we should - not need to touch connections like this when we just remove an easy - handle... - */ - if(easy->easy_conn && easy_owns_conn && - (easy->easy_conn->send_pipe->size + - easy->easy_conn->recv_pipe->size == 0)) - easy->easy_conn->connectindex = -1; + easy->easy_handle->state.conn_cache = NULL; + easy->easy_handle->state.lastconnect = NULL; } /* change state without using multistate(), only to make singlesocket() do @@ -745,6 +681,12 @@ CURLMcode curl_multi_remove_handle(CURLM *multi_handle, singlesocket(multi, easy); /* to let the application know what sockets that vanish with this handle */ + /* Remove the association between the connection and the handle */ + if(easy->easy_conn) { + easy->easy_conn->data = NULL; + easy->easy_conn = NULL; + } + Curl_easy_addmulti(easy->easy_handle, NULL); /* clear the association to this multi handle */ @@ -1324,8 +1266,8 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, case CURLM_STATE_WAITDO: /* Wait for our turn to DO when we're pipelining requests */ #ifdef DEBUGBUILD - infof(data, "Conn %ld send pipe %zu inuse %d athead %d\n", - easy->easy_conn->connectindex, + infof(data, "WAITDO: Conn %ld send pipe %zu inuse %d athead %d\n", + easy->easy_conn->connection_id, easy->easy_conn->send_pipe->size, easy->easy_conn->writechannel_inuse?1:0, isHandleAtHead(data, @@ -1514,8 +1456,8 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, } #ifdef DEBUGBUILD else { - infof(data, "Conn %ld recv pipe %zu inuse %d athead %d\n", - easy->easy_conn->connectindex, + infof(data, "WAITPERFORM: Conn %ld recv pipe %zu inuse %d athead %d\n", + easy->easy_conn->connection_id, easy->easy_conn->recv_pipe->size, easy->easy_conn->readchannel_inuse?1:0, isHandleAtHead(data, @@ -1891,45 +1833,41 @@ CURLMcode curl_multi_perform(CURLM *multi_handle, int *running_handles) return returncode; } +static void close_all_connections(struct Curl_multi *multi) +{ + struct connectdata *conn; + + conn = Curl_conncache_find_first_connection(multi->conn_cache); + while(conn) { + conn->data = multi->closure_handle; + + /* This will remove the connection from the cache */ + (void)Curl_disconnect(conn, FALSE); + + conn = Curl_conncache_find_first_connection(multi->conn_cache); + } +} + CURLMcode curl_multi_cleanup(CURLM *multi_handle) { struct Curl_multi *multi=(struct Curl_multi *)multi_handle; struct Curl_one_easy *easy; struct Curl_one_easy *nexteasy; - int i; - struct closure *cl; - struct closure *n; if(GOOD_MULTI_HANDLE(multi)) { multi->type = 0; /* not good anymore */ - /* go over all connections that have close actions */ - for(i=0; i< multi->connc->num; i++) { - if(multi->connc->connects[i] && - multi->connc->connects[i]->handler->flags & PROTOPT_CLOSEACTION) { - Curl_disconnect(multi->connc->connects[i], FALSE); - multi->connc->connects[i] = NULL; - } - } - /* now walk through the list of handles we kept around only to be - able to close connections "properly" */ - cl = multi->closure; - while(cl) { - cl->easy_handle->state.shared_conn = NULL; /* allow cleanup */ - if(cl->easy_handle->state.closed) - /* close handle only if curl_easy_cleanup() already has been called - for this easy handle */ - Curl_close(cl->easy_handle); - n = cl->next; - free(cl); - cl= n; - } + /* Close all the connections in the connection cache */ + close_all_connections(multi); + + Curl_close(multi->closure_handle); + multi->closure_handle = NULL; Curl_hash_destroy(multi->sockhash); multi->sockhash = NULL; - Curl_rm_connc(multi->connc); - multi->connc = NULL; + Curl_conncache_destroy(multi->conn_cache); + multi->conn_cache = NULL; /* remove the pending list of messages */ Curl_llist_destroy(multi->msglist, NULL); @@ -1947,7 +1885,7 @@ CURLMcode curl_multi_cleanup(CURLM *multi_handle) } /* Clear the pointer to the connection cache */ - easy->easy_handle->state.connc = NULL; + easy->easy_handle->state.conn_cache = NULL; Curl_easy_addmulti(easy->easy_handle, NULL); /* clear the association */ @@ -2803,125 +2741,6 @@ CURLMcode curl_multi_assign(CURLM *multi_handle, return CURLM_OK; } -static void multi_connc_remove_handle(struct Curl_multi *multi, - struct SessionHandle *data) -{ - /* a connection in the connection cache pointing to the given 'data' ? */ - int i; - - for(i=0; i< multi->connc->num; i++) { - struct connectdata * conn = multi->connc->connects[i]; - - if(conn && conn->data == data) { - /* If this easy_handle was the last one in charge for one or more - connections in the shared connection cache, we might need to keep - this handle around until either A) the connection is closed and - killed properly, or B) another easy_handle uses the connection. - - The reason why we need to have a easy_handle associated with a live - connection is simply that some connections will need a handle to get - closed down properly. Currently, the only connections that need to - keep a easy_handle handle around are using FTP(S). Such connections - have the PROT_CLOSEACTION bit set. - - Thus, we need to check for all connections in the shared cache that - points to this handle and are using PROT_CLOSEACTION. If there's any, - we need to add this handle to the list of "easy handles kept around - for nice connection closures". - */ - - if(conn->handler->flags & PROTOPT_CLOSEACTION) { - /* this handle is still being used by a shared connection and - thus we leave it around for now */ - if(add_closure(multi, data) == CURLM_OK) - data->state.shared_conn = multi; - else { - /* out of memory - so much for graceful shutdown */ - Curl_disconnect(conn, /* dead_connection */ FALSE); - multi->connc->connects[i] = NULL; - data->state.shared_conn = NULL; - } - } - else { - /* disconect the easy handle from the connection since the connection - will now remain but this easy handle is going */ - data->state.shared_conn = NULL; - conn->data = NULL; - } - } - } -} - -/* Add the given data pointer to the list of 'closure handles' that are kept - around only to be able to close some connections nicely - just make sure - that this handle isn't already added, like for the cases when an easy - handle is removed, added and removed again... */ -static CURLMcode add_closure(struct Curl_multi *multi, - struct SessionHandle *data) -{ - struct closure *cl = multi->closure; - struct closure *p = NULL; - bool add = TRUE; - - /* Before adding, scan through all the other currently kept handles and see - if there are any connections still referring to them and kill them if - not. */ - while(cl) { - struct closure *n; - bool inuse = FALSE; - int i; - - for(i=0; i< multi->connc->num; i++) { - if(multi->connc->connects[i] && - (multi->connc->connects[i]->data == cl->easy_handle)) { - inuse = TRUE; - break; - } - } - - n = cl->next; - - if(!inuse) { - /* cl->easy_handle is now killable */ - - /* unmark it as not having a connection around that uses it anymore */ - cl->easy_handle->state.shared_conn= NULL; - - if(cl->easy_handle->state.closed) { - infof(data, "Delayed kill of easy handle %p\n", cl->easy_handle); - /* close handle only if curl_easy_cleanup() already has been called - for this easy handle */ - Curl_close(cl->easy_handle); - } - if(p) - p->next = n; - else - multi->closure = n; - free(cl); - } - else { - if(cl->easy_handle == data) - add = FALSE; - - p = cl; - } - - cl = n; - } - - if(add) { - cl = calloc(1, sizeof(struct closure)); - if(!cl) - return CURLM_OUT_OF_MEMORY; - - cl->easy_handle = data; - cl->next = multi->closure; - multi->closure = cl; - } - - return CURLM_OK; -} - #ifdef DEBUGBUILD void Curl_multi_dump(const struct Curl_multi *multi_handle) { diff --git a/lib/url.c b/lib/url.c index a781798e9..574751716 100644 --- a/lib/url.c +++ b/lib/url.c @@ -126,6 +126,8 @@ int curl_win32_idn_to_ascii(const char *in, char **out); #include "curl_rtmp.h" #include "gopher.h" #include "http_proxy.h" +#include "bundles.h" +#include "conncache.h" #define _MPRINTF_REPLACE /* use our functions only */ #include @@ -135,7 +137,7 @@ int curl_win32_idn_to_ascii(const char *in, char **out); #include "memdebug.h" /* Local static prototypes */ -static long ConnectionKillOne(struct SessionHandle *data); +static bool ConnectionKillOne(struct SessionHandle *data); static void conn_free(struct connectdata *conn); static void signalPipeClose(struct curl_llist *pipeline, bool pipe_broke); static CURLcode do_init(struct connectdata *conn); @@ -262,10 +264,10 @@ static const struct Curl_handler Curl_handler_dummy = { static void close_connections(struct SessionHandle *data) { /* Loop through all open connections and kill them one by one */ - long i; + bool killed; do { - i = ConnectionKillOne(data); - } while(i != -1L); + killed = ConnectionKillOne(data); + } while(killed); } void Curl_freeset(struct SessionHandle * data) @@ -378,66 +380,6 @@ CURLcode Curl_close(struct SessionHandle *data) { struct Curl_multi *m = data->multi; -#ifdef DEBUGBUILD - /* only for debugging, scan through all connections and see if there's a - pipe reference still identifying this handle */ - - if(data->state.connc && data->state.connc->type == CONNCACHE_MULTI) { - struct conncache *c = data->state.connc; - long i; - struct curl_llist *pipeline; - struct curl_llist_element *curr; - struct connectdata *connptr; - - for(i=0; i< c->num; i++) { - connptr = c->connects[i]; - if(!connptr) - continue; - - pipeline = connptr->send_pipe; - if(pipeline) { - for(curr = pipeline->head; curr; curr=curr->next) { - if(data == (struct SessionHandle *) curr->ptr) { - fprintf(stderr, - "problem we %p are still in send pipe for %p done %d\n", - data, connptr, (int)connptr->bits.done); - } - } - } - pipeline = connptr->recv_pipe; - if(pipeline) { - for(curr = pipeline->head; curr; curr=curr->next) { - if(data == (struct SessionHandle *) curr->ptr) { - fprintf(stderr, - "problem we %p are still in recv pipe for %p done %d\n", - data, connptr, (int)connptr->bits.done); - } - } - } - pipeline = connptr->done_pipe; - if(pipeline) { - for(curr = pipeline->head; curr; curr=curr->next) { - if(data == (struct SessionHandle *) curr->ptr) { - fprintf(stderr, - "problem we %p are still in done pipe for %p done %d\n", - data, connptr, (int)connptr->bits.done); - } - } - } - pipeline = connptr->pend_pipe; - if(pipeline) { - for(curr = pipeline->head; curr; curr=curr->next) { - if(data == (struct SessionHandle *) curr->ptr) { - fprintf(stderr, - "problem we %p are still in pend pipe for %p done %d\n", - data, connptr, (int)connptr->bits.done); - } - } - } - } - } -#endif - Curl_expire(data, 0); /* shut off timers */ if(m) @@ -457,26 +399,16 @@ CURLcode Curl_close(struct SessionHandle *data) the multi handle, since that function uses the magic field! */ - if(data->state.connc) { - - if(data->state.connc->type == CONNCACHE_PRIVATE) { + if(data->state.conn_cache) { + if(data->state.conn_cache->type == CONNCACHE_PRIVATE) { /* close all connections still alive that are in the private connection cache, as we no longer have the pointer left to the shared one. */ close_connections(data); - - /* free the connection cache if allocated privately */ - Curl_rm_connc(data->state.connc); - data->state.connc = NULL; + Curl_conncache_destroy(data->state.conn_cache); + data->state.conn_cache = NULL; } } - if(data->state.shared_conn) { - /* marked to be used by a pending connection so we can't kill this handle - just yet */ - data->state.closed = TRUE; - return CURLE_OK; - } - if(data->dns.hostcachetype == HCACHE_PRIVATE) Curl_hostcache_destroy(data); @@ -533,124 +465,6 @@ CURLcode Curl_close(struct SessionHandle *data) return CURLE_OK; } -/* create a connection cache of a private or multi type */ -struct conncache *Curl_mk_connc(int type, - long amount) /* set -1 to use default */ -{ - /* It is subject for debate how many default connections to have for a multi - connection cache... */ - - struct conncache *c; - long default_amount; - long max_amount = (long)(((size_t)INT_MAX) / sizeof(struct connectdata *)); - - if(type == CONNCACHE_PRIVATE) { - default_amount = (amount < 1L) ? 5L : amount; - } - else { - default_amount = (amount < 1L) ? 10L : amount; - } - - if(default_amount > max_amount) - default_amount = max_amount; - - c = calloc(1, sizeof(struct conncache)); - if(!c) - return NULL; - - c->connects = calloc((size_t)default_amount, sizeof(struct connectdata *)); - if(!c->connects) { - free(c); - return NULL; - } - - c->num = default_amount; - - return c; -} - -/* Change number of entries of a connection cache */ -CURLcode Curl_ch_connc(struct SessionHandle *data, - struct conncache *c, - long newamount) -{ - long i; - struct connectdata **newptr; - long max_amount = (long)(((size_t)INT_MAX) / sizeof(struct connectdata *)); - - if(newamount < 1) - newamount = 1; /* we better have at least one entry */ - - if(!c) { - /* we get a NULL pointer passed in as connection cache, which means that - there is no cache created for this SessionHandle just yet, we create a - brand new with the requested size. - */ - data->state.connc = Curl_mk_connc(CONNCACHE_PRIVATE, newamount); - if(!data->state.connc) - return CURLE_OUT_OF_MEMORY; - return CURLE_OK; - } - - if(newamount < c->num) { - /* Since this number is *decreased* from the existing number, we must - close the possibly open connections that live on the indexes that - are being removed! - - NOTE: for conncache_multi cases we must make sure that we only - close handles not in use. - */ - for(i=newamount; i< c->num; i++) { - Curl_disconnect(c->connects[i], /* dead_connection */ FALSE); - c->connects[i] = NULL; - } - - /* If the most recent connection is no longer valid, mark it - invalid. */ - if(data->state.lastconnect <= newamount) - data->state.lastconnect = -1; - } - if(newamount > 0) { - if(newamount > max_amount) - newamount = max_amount; - newptr = realloc(c->connects, sizeof(struct connectdata *) * newamount); - if(!newptr) - /* we closed a few connections in vain, but so what? */ - return CURLE_OUT_OF_MEMORY; - - /* nullify the newly added pointers */ - for(i=c->num; iconnects = newptr; - c->num = newamount; - } - /* we no longer support less than 1 as size for the connection cache, and - I'm not sure it ever worked to set it to zero */ - return CURLE_OK; -} - -/* Free a connection cache. This is called from Curl_close() and - curl_multi_cleanup(). */ -void Curl_rm_connc(struct conncache *c) -{ - if(!c) - return; - - if(c->connects) { - long i; - for(i = 0; i < c->num; ++i) { - conn_free(c->connects[i]); - c->connects[i] = NULL; - } - free(c->connects); - c->connects = NULL; - } - c->num = 0; - - free(c); -} - /* * Initialize the UserDefined fields within a SessionHandle. * This may be safely called on a new or existing SessionHandle. @@ -807,7 +621,7 @@ CURLcode Curl_open(struct SessionHandle **curl) Curl_convert_init(data); /* most recent connection is not yet defined */ - data->state.lastconnect = -1; + data->state.lastconnect = NULL; data->progress.flags |= PGRS_HIDE; data->state.current_speed = -1; /* init to negative == impossible */ @@ -880,7 +694,7 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option, * Set the absolute number of maximum simultaneous alive connection that * libcurl is allowed to have. */ - result = Curl_ch_connc(data, data->state.connc, va_arg(param, long)); + data->set.maxconnects = va_arg(param, long); break; case CURLOPT_FORBID_REUSE: /* @@ -2744,14 +2558,9 @@ CURLcode Curl_disconnect(struct connectdata *conn, bool dead_connection) /* This is set if protocol-specific cleanups should be made */ conn->handler->disconnect(conn, dead_connection); - if(-1 != conn->connectindex) { /* unlink ourselves! */ - infof(data, "Closing connection #%ld\n", conn->connectindex); - if(data->state.connc) - /* only clear the table entry if we still know in which cache we - used to be in */ - data->state.connc->connects[conn->connectindex] = NULL; - } + infof(data, "Closing connection %d\n", conn->connection_id); + Curl_conncache_remove_conn(data->state.conn_cache, conn); #if defined(USE_LIBIDN) if(conn->host.encalloc) @@ -2938,225 +2747,229 @@ ConnectionExists(struct SessionHandle *data, struct connectdata *needle, struct connectdata **usethis) { - long i; struct connectdata *check; struct connectdata *chosen = 0; bool canPipeline = IsPipeliningPossible(data, needle); bool wantNTLM = (data->state.authhost.want==CURLAUTH_NTLM) || (data->state.authhost.want==CURLAUTH_NTLM_WB) ? TRUE : FALSE; + struct connectbundle *bundle; - for(i=0; i< data->state.connc->num; i++) { - bool match = FALSE; - bool credentialsMatch = FALSE; - size_t pipeLen = 0; - /* - * Note that if we use a HTTP proxy, we check connections to that - * proxy and not to the actual remote server. - */ - check = data->state.connc->connects[i]; - if(!check) - /* NULL pointer means not filled-in entry */ - continue; + /* Look up the bundle with all the connections to this + particular host */ + bundle = Curl_conncache_find_bundle(data->state.conn_cache, + needle->host.name); + if(bundle) { + struct curl_llist_element *curr; - pipeLen = check->send_pipe->size + check->recv_pipe->size; + infof(data, "Found bundle for host %s: %p\n", needle->host.name, bundle); - if(check->connectindex == -1) { - check->connectindex = i; /* Set this appropriately since it might have - been set to -1 when the easy was removed - from the multi */ - } + curr = bundle->conn_list->head; + while(curr) { + bool match = FALSE; + bool credentialsMatch = FALSE; + size_t pipeLen; - if(!pipeLen && !check->inuse) { - /* The check for a dead socket makes sense only if there are no - handles in pipeline and the connection isn't already marked in - use */ - bool dead; - if(check->handler->protocol & CURLPROTO_RTSP) - /* RTSP is a special case due to RTP interleaving */ - dead = Curl_rtsp_connisdead(check); - else - dead = SocketIsDead(check->sock[FIRSTSOCKET]); + /* + * Note that if we use a HTTP proxy, we check connections to that + * proxy and not to the actual remote server. + */ + check = curr->ptr; + curr = curr->next; - if(dead) { - check->data = data; - infof(data, "Connection #%ld seems to be dead!\n", i); + pipeLen = check->send_pipe->size + check->recv_pipe->size; - /* disconnect resources */ - Curl_disconnect(check, /* dead_connection */ TRUE); - data->state.connc->connects[i]=NULL; /* nothing here */ + if(!pipeLen && !check->inuse) { + /* The check for a dead socket makes sense only if there are no + handles in pipeline and the connection isn't already marked in + use */ + bool dead; + if(check->handler->protocol & CURLPROTO_RTSP) + /* RTSP is a special case due to RTP interleaving */ + dead = Curl_rtsp_connisdead(check); + else + dead = SocketIsDead(check->sock[FIRSTSOCKET]); - continue; - } - } + if(dead) { + check->data = data; + infof(data, "Connection %d seems to be dead!\n", + check->connection_id); - if(canPipeline) { - /* Make sure the pipe has only GET requests */ - struct SessionHandle* sh = gethandleathead(check->send_pipe); - struct SessionHandle* rh = gethandleathead(check->recv_pipe); - if(sh) { - if(!IsPipeliningPossible(sh, check)) - continue; - } - else if(rh) { - if(!IsPipeliningPossible(rh, check)) + /* disconnect resources */ + Curl_disconnect(check, /* dead_connection */ TRUE); continue; + } } + if(canPipeline) { + /* Make sure the pipe has only GET requests */ + struct SessionHandle* sh = gethandleathead(check->send_pipe); + struct SessionHandle* rh = gethandleathead(check->recv_pipe); + if(sh) { + if(!IsPipeliningPossible(sh, check)) + continue; + } + else if(rh) { + if(!IsPipeliningPossible(rh, check)) + continue; + } #ifdef DEBUGBUILD if(pipeLen > MAX_PIPELINE_LENGTH) { infof(data, "BAD! Connection #%ld has too big pipeline!\n", - check->connectindex); + check->connection_id); } #endif - } - else { - if(pipeLen > 0) { - /* can only happen within multi handles, and means that another easy - handle is using this connection */ - continue; } + else { + if(pipeLen > 0) { + /* can only happen within multi handles, and means that another easy + handle is using this connection */ + continue; + } - if(Curl_resolver_asynch()) { - /* ip_addr_str[0] is NUL only if the resolving of the name hasn't - completed yet and until then we don't re-use this connection */ - if(!check->ip_addr_str[0]) { - infof(data, - "Connection #%ld hasn't finished name resolve, can't reuse\n", - check->connectindex); + if(Curl_resolver_asynch()) { + /* ip_addr_str[0] is NUL only if the resolving of the name hasn't + completed yet and until then we don't re-use this connection */ + if(!check->ip_addr_str[0]) { + infof(data, + "Connection #%ld is still name resolving, can't reuse\n", + check->connection_id); + continue; + } + } + + if((check->sock[FIRSTSOCKET] == CURL_SOCKET_BAD) || + check->bits.close) { + /* Don't pick a connection that hasn't connected yet or that is going + to get closed. */ + infof(data, "Connection #%ld isn't open enough, can't reuse\n", + check->connection_id); +#ifdef DEBUGBUILD + if(check->recv_pipe->size > 0) { + infof(data, + "BAD! Unconnected #%ld has a non-empty recv pipeline!\n", + check->connection_id); + } +#endif continue; } } - if((check->sock[FIRSTSOCKET] == CURL_SOCKET_BAD) || check->bits.close) { - /* Don't pick a connection that hasn't connected yet or that is going - to get closed. */ - infof(data, "Connection #%ld isn't open enough, can't reuse\n", - check->connectindex); -#ifdef DEBUGBUILD - if(check->recv_pipe->size > 0) { - infof(data, "BAD! Unconnected #%ld has a non-empty recv pipeline!\n", - check->connectindex); - } -#endif - continue; + if((needle->handler->flags&PROTOPT_SSL) != + (check->handler->flags&PROTOPT_SSL)) + /* don't do mixed SSL and non-SSL connections */ + if(!(needle->handler->protocol & check->handler->protocol)) + /* except protocols that have been upgraded via TLS */ + continue; + + if(needle->handler->flags&PROTOPT_SSL) { + if((data->set.ssl.verifypeer != check->verifypeer) || + (data->set.ssl.verifyhost != check->verifyhost)) + continue; } - } - if((needle->handler->flags&PROTOPT_SSL) != - (check->handler->flags&PROTOPT_SSL)) - /* don't do mixed SSL and non-SSL connections */ - if(!(needle->handler->protocol & check->handler->protocol)) - /* except protocols that have been upgraded via TLS */ + if(needle->bits.proxy != check->bits.proxy) + /* don't do mixed proxy and non-proxy connections */ continue; - if(needle->handler->flags&PROTOPT_SSL) { - if((data->set.ssl.verifypeer != check->verifypeer) || - (data->set.ssl.verifyhost != check->verifyhost)) + if(!canPipeline && check->inuse) + /* this request can't be pipelined but the checked connection is + already in use so we skip it */ continue; - } - if(needle->bits.proxy != check->bits.proxy) - /* don't do mixed proxy and non-proxy connections */ - continue; + if(needle->localdev || needle->localport) { + /* If we are bound to a specific local end (IP+port), we must not + re-use a random other one, although if we didn't ask for a + particular one we can reuse one that was bound. - if(!canPipeline && check->inuse) - /* this request can't be pipelined but the checked connection is already - in use so we skip it */ - continue; - - if(needle->localdev || needle->localport) { - /* If we are bound to a specific local end (IP+port), we must not re-use - a random other one, although if we didn't ask for a particular one we - can reuse one that was bound. - - This comparison is a bit rough and too strict. Since the input - parameters can be specified in numerous ways and still end up the - same it would take a lot of processing to make it really accurate. - Instead, this matching will assume that re-uses of bound connections - will most likely also re-use the exact same binding parameters and - missing out a few edge cases shouldn't hurt anyone very much. - */ - if((check->localport != needle->localport) || - (check->localportrange != needle->localportrange) || - !check->localdev || - !needle->localdev || - strcmp(check->localdev, needle->localdev)) - continue; - } - - if(!needle->bits.httpproxy || needle->handler->flags&PROTOPT_SSL || - (needle->bits.httpproxy && check->bits.httpproxy && - needle->bits.tunnel_proxy && check->bits.tunnel_proxy && - Curl_raw_equal(needle->proxy.name, check->proxy.name) && - (needle->port == check->port))) { - /* The requested connection does not use a HTTP proxy or it uses SSL or - it is a non-SSL protocol tunneled over the same http proxy name and - port number or it is a non-SSL protocol which is allowed to be - upgraded via TLS */ - - if((Curl_raw_equal(needle->handler->scheme, check->handler->scheme) || - needle->handler->protocol & check->handler->protocol) && - Curl_raw_equal(needle->host.name, check->host.name) && - needle->remote_port == check->remote_port) { - if(needle->handler->flags & PROTOPT_SSL) { - /* This is a SSL connection so verify that we're using the same - SSL options as well */ - if(!Curl_ssl_config_matches(&needle->ssl_config, - &check->ssl_config)) { - DEBUGF(infof(data, - "Connection #%ld has different SSL parameters, " - "can't reuse\n", - check->connectindex)); - continue; - } - else if(check->ssl[FIRSTSOCKET].state != ssl_connection_complete) { - DEBUGF(infof(data, - "Connection #%ld has not started SSL connect, " - "can't reuse\n", - check->connectindex)); - continue; - } - } - if((needle->handler->protocol & CURLPROTO_FTP) || - ((needle->handler->protocol & CURLPROTO_HTTP) && wantNTLM)) { - /* This is FTP or HTTP+NTLM, verify that we're using the same name - and password as well */ - if(!strequal(needle->user, check->user) || - !strequal(needle->passwd, check->passwd)) { - /* one of them was different */ - continue; - } - credentialsMatch = TRUE; - } - match = TRUE; + This comparison is a bit rough and too strict. Since the input + parameters can be specified in numerous ways and still end up the + same it would take a lot of processing to make it really accurate. + Instead, this matching will assume that re-uses of bound connections + will most likely also re-use the exact same binding parameters and + missing out a few edge cases shouldn't hurt anyone very much. + */ + if((check->localport != needle->localport) || + (check->localportrange != needle->localportrange) || + !check->localdev || + !needle->localdev || + strcmp(check->localdev, needle->localdev)) + continue; } - } - else { /* The requested needle connection is using a proxy, - is the checked one using the same host, port and type? */ - if(check->bits.proxy && - (needle->proxytype == check->proxytype) && - (needle->bits.tunnel_proxy == check->bits.tunnel_proxy) && - Curl_raw_equal(needle->proxy.name, check->proxy.name) && - needle->port == check->port) { - /* This is the same proxy connection, use it! */ - match = TRUE; + + if(!needle->bits.httpproxy || needle->handler->flags&PROTOPT_SSL || + (needle->bits.httpproxy && check->bits.httpproxy && + needle->bits.tunnel_proxy && check->bits.tunnel_proxy && + Curl_raw_equal(needle->proxy.name, check->proxy.name) && + (needle->port == check->port))) { + /* The requested connection does not use a HTTP proxy or it uses SSL or + it is a non-SSL protocol tunneled over the same http proxy name and + port number or it is a non-SSL protocol which is allowed to be + upgraded via TLS */ + + if((Curl_raw_equal(needle->handler->scheme, check->handler->scheme) || + needle->handler->protocol & check->handler->protocol) && + Curl_raw_equal(needle->host.name, check->host.name) && + needle->remote_port == check->remote_port) { + if(needle->handler->flags & PROTOPT_SSL) { + /* This is a SSL connection so verify that we're using the same + SSL options as well */ + if(!Curl_ssl_config_matches(&needle->ssl_config, + &check->ssl_config)) { + DEBUGF(infof(data, + "Connection #%ld has different SSL parameters, " + "can't reuse\n", + check->connection_id)); + continue; + } + else if(check->ssl[FIRSTSOCKET].state != ssl_connection_complete) { + DEBUGF(infof(data, + "Connection #%ld has not started SSL connect, " + "can't reuse\n", + check->connection_id)); + continue; + } + } + if((needle->handler->protocol & CURLPROTO_FTP) || + ((needle->handler->protocol & CURLPROTO_HTTP) && wantNTLM)) { + /* This is FTP or HTTP+NTLM, verify that we're using the same name + and password as well */ + if(!strequal(needle->user, check->user) || + !strequal(needle->passwd, check->passwd)) { + /* one of them was different */ + continue; + } + credentialsMatch = TRUE; + } + match = TRUE; + } + } + else { /* The requested needle connection is using a proxy, + is the checked one using the same host, port and type? */ + if(check->bits.proxy && + (needle->proxytype == check->proxytype) && + (needle->bits.tunnel_proxy == check->bits.tunnel_proxy) && + Curl_raw_equal(needle->proxy.name, check->proxy.name) && + needle->port == check->port) { + /* This is the same proxy connection, use it! */ + match = TRUE; + } } - } - if(match) { - chosen = check; + if(match) { + chosen = check; - /* If we are not looking for an NTLM connection, we can choose this one - immediately. */ - if(!wantNTLM) - break; + /* If we are not looking for an NTLM connection, we can choose this one + immediately. */ + if(!wantNTLM) + break; - /* Otherwise, check if this is already authenticating with the right - credentials. If not, keep looking so that we can reuse NTLM - connections if possible. (Especially we must reuse the same - connection if partway through a handshake!) */ - if(credentialsMatch && chosen->ntlm.state != NTLMSTATE_NONE) - break; + /* Otherwise, check if this is already authenticating with the right + credentials. If not, keep looking so that we can reuse NTLM + connections if possible. (Especially we must reuse the same + connection if partway through a handshake!) */ + if(credentialsMatch && chosen->ntlm.state != NTLMSTATE_NONE) + break; + } } } @@ -3170,53 +2983,67 @@ ConnectionExists(struct SessionHandle *data, return FALSE; /* no matching connecting exists */ } - - /* * This function kills and removes an existing connection in the connection * cache. The connection that has been unused for the longest time. * - * Returns -1 if it can't find any unused connection to kill. + * Returns FALSE if it can't find any unused connection to kill. */ -static long +static bool ConnectionKillOne(struct SessionHandle *data) { - long i; - struct connectdata *conn; + struct conncache *bc = data->state.conn_cache; + struct curl_hash_iterator iter; + struct curl_llist_element *curr; + struct curl_hash_element *he; long highscore=-1; - long connindex=-1; long score; struct timeval now; + struct connectdata *conn_candidate = NULL; + struct connectbundle *bundle; now = Curl_tvnow(); - for(i=0; data->state.connc && (i< data->state.connc->num); i++) { - conn = data->state.connc->connects[i]; + Curl_hash_start_iterate(bc->hash, &iter); - if(!conn || conn->inuse) - continue; + he = Curl_hash_next_element(&iter); + while(he) { + struct connectdata *conn; - /* Set higher score for the age passed since the connection was used */ - score = Curl_tvdiff(now, conn->now); + bundle = he->ptr; - if(score > highscore) { - highscore = score; - connindex = i; + curr = bundle->conn_list->head; + while(curr) { + conn = curr->ptr; + + if(!conn->inuse) { + /* Set higher score for the age passed since the connection was used */ + score = Curl_tvdiff(now, conn->now); + + if(score > highscore) { + highscore = score; + conn_candidate = conn; + } + } + curr = curr->next; } + + he = Curl_hash_next_element(&iter); } - if(connindex >= 0) { + + if(conn_candidate) { /* Set the connection's owner correctly */ - conn = data->state.connc->connects[connindex]; - conn->data = data; + conn_candidate->data = data; + + bundle = conn_candidate->bundle; /* the winner gets the honour of being disconnected */ - (void)Curl_disconnect(conn, /* dead_connection */ FALSE); + (void)Curl_disconnect(conn_candidate, /* dead_connection */ FALSE); - /* clean the array entry */ - data->state.connc->connects[connindex] = NULL; + return TRUE; } - return connindex; /* return the available index or -1 */ + return FALSE; } /* this connection can now be marked 'idle' */ @@ -3234,41 +3061,16 @@ ConnectionDone(struct connectdata *conn) * The given connection should be unique. That must've been checked prior to * this call. */ -static void ConnectionStore(struct SessionHandle *data, - struct connectdata *conn) +static CURLcode ConnectionStore(struct SessionHandle *data, + struct connectdata *conn) { - long i; - for(i=0; i< data->state.connc->num; i++) { - if(!data->state.connc->connects[i]) - break; - } - if(i == data->state.connc->num) { - /* there was no room available, kill one */ - i = ConnectionKillOne(data); - if(-1 != i) - infof(data, "Connection (#%ld) was killed to make room (holds %ld)\n", - i, data->state.connc->num); - else - infof(data, "This connection did not fit in the connection cache\n"); - } + static int connection_id_counter = 0; - conn->connectindex = i; /* Make the child know where the pointer to this - particular data is stored. But note that this -1 - if this is not within the cache and this is - probably not checked for everywhere (yet). */ - conn->inuse = TRUE; - if(-1 != i) { - /* Only do this if a true index was returned, if -1 was returned there - is no room in the cache for an unknown reason and we cannot store - this there. + /* Assign a number to the connection for easier tracking in the log + output */ + conn->connection_id = connection_id_counter++; - TODO: make sure we really can work with more handles than positions in - the cache, or possibly we should (allow to automatically) resize the - connection cache when we add more easy handles to a multi handle! - */ - data->state.connc->connects[i] = conn; /* fill in this */ - conn->data = data; - } + return Curl_conncache_add_conn(data->state.conn_cache, conn); } /* after a TCP connection to the proxy has been verified, this function does @@ -3318,7 +3120,7 @@ static CURLcode ConnectPlease(struct SessionHandle *data, infof(data, "About to connect() to %s%s port %ld (#%ld)\n", conn->bits.proxy?"proxy ":"", - hostname, conn->port, conn->connectindex); + hostname, conn->port, conn->connection_id); #else (void)data; #endif @@ -3359,7 +3161,7 @@ void Curl_verboseconnect(struct connectdata *conn) if(conn->data->set.verbose) infof(conn->data, "Connected to %s (%s) port %ld (#%ld)\n", conn->bits.proxy ? conn->proxy.dispname : conn->host.dispname, - conn->ip_addr_str, conn->port, conn->connectindex); + conn->ip_addr_str, conn->port, conn->connection_id); } #endif @@ -3618,7 +3420,7 @@ static struct connectdata *allocate_conn(struct SessionHandle *data) conn->sock[FIRSTSOCKET] = CURL_SOCKET_BAD; /* no file descriptor */ conn->sock[SECONDARYSOCKET] = CURL_SOCKET_BAD; /* no file descriptor */ - conn->connectindex = -1; /* no index */ + conn->connection_id = -1; /* no ID */ conn->port = -1; /* unknown at this point */ /* Default protocol-independent behavior doesn't support persistent @@ -4898,7 +4700,7 @@ static CURLcode create_conn(struct SessionHandle *data, urllen=LEAST_PATH_ALLOC; /* - * We malloc() the buffers below urllen+2 to make room for to possibilities: + * We malloc() the buffers below urllen+2 to make room for 2 possibilities: * 1 - an extra terminating zero * 2 - an extra slash (in case a syntax like "www.host.com?moo" is used) */ @@ -4960,8 +4762,8 @@ static CURLcode create_conn(struct SessionHandle *data, /* according to rfc3986, allow the query (?foo=bar) also on protocols that can't handle it. - cut the string-part after '?' - */ + cut the string-part after '?' + */ /* terminate the string */ path_q_sep[0] = 0; @@ -4975,7 +4777,7 @@ static CURLcode create_conn(struct SessionHandle *data, if(conn->bits.proxy_user_passwd) { result = parse_proxy_auth(data, conn); if(result != CURLE_OK) - return result; + return result; } /************************************************************* @@ -5178,7 +4980,7 @@ static CURLcode create_conn(struct SessionHandle *data, fix_hostname(data, conn, &conn->host); infof(data, "Re-using existing connection! (#%ld) with host %s\n", - conn->connectindex, + conn->connection_id, conn->proxy.name?conn->proxy.dispname:conn->host.dispname); } else { @@ -5443,12 +5245,8 @@ CURLcode Curl_done(struct connectdata **connp, state it is for re-using, so we're forced to close it. In a perfect world we can add code that keep track of if we really must close it here or not, but currently we have no such detail knowledge. - - connectindex == -1 here means that the connection has no spot in the - connection cache and thus we must disconnect it here. */ - if(data->set.reuse_forbid || conn->bits.close || premature || - (-1 == conn->connectindex)) { + if(data->set.reuse_forbid || conn->bits.close || premature) { CURLcode res2 = Curl_disconnect(conn, premature); /* close connection */ /* If we had an error already, make sure we return that one. But @@ -5460,10 +5258,10 @@ CURLcode Curl_done(struct connectdata **connp, ConnectionDone(conn); /* the connection is no longer in use */ /* remember the most recently used connection */ - data->state.lastconnect = conn->connectindex; + data->state.lastconnect = conn; infof(data, "Connection #%ld to host %s left intact\n", - conn->connectindex, + conn->connection_id, conn->bits.httpproxy?conn->proxy.dispname:conn->host.dispname); } diff --git a/lib/url.h b/lib/url.h index c858706a1..ab6d3d048 100644 --- a/lib/url.h +++ b/lib/url.h @@ -46,15 +46,6 @@ CURLcode Curl_protocol_doing(struct connectdata *conn, bool *done); CURLcode Curl_setup_conn(struct connectdata *conn, bool *protocol_done); -/* create a connection cache */ -struct conncache *Curl_mk_connc(int type, long amount); -/* free a connection cache */ -void Curl_rm_connc(struct conncache *c); -/* Change number of entries of a connection cache */ -CURLcode Curl_ch_connc(struct SessionHandle *data, - struct conncache *c, - long newamount); - int Curl_protocol_getsock(struct connectdata *conn, curl_socket_t *socks, int numsocks); diff --git a/lib/urldata.h b/lib/urldata.h index 4116c341f..cd50f623f 100644 --- a/lib/urldata.h +++ b/lib/urldata.h @@ -795,8 +795,8 @@ struct connectdata { consideration (== only for pipelining). */ /**** Fields set when inited and not modified again */ - long connectindex; /* what index in the connection cache connects index this - particular struct has */ + long connection_id; /* Contains a unique number to make it easier to + track the connections in the log output */ /* 'dns_entry' is the particular host we use. This points to an entry in the DNS cache and it will not get pruned while locked. It gets unlocked in @@ -924,7 +924,6 @@ struct connectdata { handle */ bool server_supports_pipelining; /* TRUE if server supports pipelining, set after first response */ - struct curl_llist *send_pipe; /* List of handles waiting to send on this pipeline */ struct curl_llist *recv_pipe; /* List of handles waiting to read @@ -934,7 +933,6 @@ struct connectdata { struct curl_llist *done_pipe; /* Handles that are finished, but still reference this connectdata */ #define MAX_PIPELINE_LENGTH 5 - char* master_buffer; /* The master buffer allocated on-demand; used for pipelining. */ size_t read_pos; /* Current read position in the master buffer */ @@ -1011,6 +1009,8 @@ struct connectdata { TUNNEL_CONNECT, /* CONNECT has been sent off */ TUNNEL_COMPLETE /* CONNECT response received completely */ } tunnel_state[2]; /* two separate ones to allow FTP */ + + struct connectbundle *bundle; /* The bundle we are member of */ }; /* The end of connectdata. */ @@ -1146,18 +1146,6 @@ struct auth { be RFC compliant */ }; -struct conncache { - /* 'connects' will be an allocated array with pointers. If the pointer is - set, it holds an allocated connection. */ - struct connectdata **connects; - long num; /* number of entries of the 'connects' array */ - enum { - CONNCACHE_PRIVATE, /* used for an easy handle alone */ - CONNCACHE_MULTI /* shared within a multi handle */ - } type; -}; - - struct UrlState { enum { Curl_if_none, @@ -1165,13 +1153,20 @@ struct UrlState { Curl_if_multi } used_interface; - struct conncache *connc; /* points to the connection cache this handle - uses */ + /* Points to the connection cache */ + struct conncache *conn_cache; /* buffers to store authentication data in, as parsed from input options */ struct timeval keeps_speed; /* for the progress meter really */ - long lastconnect; /* index of most recent connect or -1 if undefined */ + struct connectdata *pending_conn; /* This points to the connection we want + to open when we are waiting in the + CONNECT_PEND state in the multi + interface. This to avoid recreating it + when we enter the CONNECT state again. + */ + + struct connectdata *lastconnect; /* The last connection, NULL if undefined */ char *headerbuff; /* allocated buffer to store headers in */ size_t headersize; /* size of the allocation */ @@ -1250,14 +1245,6 @@ struct UrlState { /* for FTP downloads: how many CRLFs did we converted to LFs? */ curl_off_t crlf_conversions; #endif - /* If set to non-NULL, there's a connection in a shared connection cache - that uses this handle so we can't kill this SessionHandle just yet but - must keep it around and add it to the list of handles to kill once all - its connections are gone */ - void *shared_conn; - bool closed; /* set to TRUE when curl_easy_cleanup() has been called on this - handle, but it is kept around as mentioned for - shared_conn */ char *pathbuffer;/* allocated buffer to store the URL's path part in */ char *path; /* path to use, points to somewhere within the pathbuffer area */ @@ -1593,6 +1580,8 @@ struct UserDefined { bool tcp_keepalive; /* use TCP keepalives */ long tcp_keepidle; /* seconds in idle before sending keepalive probe */ long tcp_keepintvl; /* seconds between TCP keepalive probes */ + + size_t maxconnects; /* Max idle connections in the connection cache */ }; struct Names {