dynbuf: introduce internal generic dynamic buffer functions

A common set of functions instead of many separate implementations for
creating buffers that can grow when appending data to them. Existing
functionality has been ported over.

In my early basic testing, the total number of allocations seem at
roughly the same amount as before, possibly a few less.

See docs/DYNBUF.md for a description of the API.

Closes #5300

docs/DYNBUF.md

@@ -0,0 +1,80 @@
+# dynbuf
+
+This is the internal module for creating and handling "dynamic buffers". This
+means buffers that can be appended to, dynamically and grow in size to adapt.
+
+There will always be a terminating zero put at the end of the dynamic buffer.
+
+The `struct dynbuf` is used to hold data for each instance of a dynamic
+buffer. The members of that struct **MUST NOT** be accessed or modified
+without using the dedicated dynbuf API.
+
+## init
+
+    void Curl_dyn_init(struct dynbuf *s, size_t toobig);
+
+This inits a struct to use for dynbuf and it can't fail. The `toobig` value
+**must** be set to the maximum size we allow this buffer instance to grow to.
+The functions below will return `CURLE_OUT_OF_MEMORY` when hitting this limit.
+
+## free
+
+    void Curl_dyn_free(struct dynbuf *s);
+
+Free the associated memory and clean up. After a free, the `dynbuf` struct can
+be re-used to start appending new data to.
+
+## addn
+
+    CURLcode Curl_dyn_addn(struct dynbuf *s, const void *mem, size_t len);
+
+Append arbitrary data of a given length to the end of the buffer.
+
+## add
+
+    CURLcode Curl_dyn_add(struct dynbuf *s, const char *str);
+
+Append a C string to the end of the buffer.
+
+## addf
+
+    CURLcode Curl_dyn_addf(struct dynbuf *s, const char *fmt, ...);
+
+Append a `printf()`-style string to the end of the buffer.
+
+## reset
+
+    void Curl_dyn_reset(struct dynbuf *s);
+
+Reset the buffer length, but leave the allocation.
+
+## tail
+
+    CURLcode Curl_dyn_trail(struct dynbuf *s, size_t length)
+
+Keep `length` bytes of the buffer tail (the last `length` bytes of the
+buffer). The rest of the buffer is dropped. The specified `length` must not be
+larger than the buffer length. (**This function is currently not provided**.)
+
+## ptr
+
+    char *Curl_dyn_ptr(const struct dynbuf *s);
+
+Returns a `char *` to the buffer. Since the buffer may be reallocated, this
+pointer should not be trusted or used anymore after the next buffer
+manipulation call.
+
+## uptr
+
+    unsigned char *Curl_dyn_uptr(const struct dynbuf *s);
+
+Returns an `unsigned char *` to the buffer. Since the buffer may be
+reallocated, this pointer should not be trusted or used anymore after the next
+buffer manipulation call.
+
+## len
+
+    size_t Curl_dyn_len(const struct dynbuf *s);
+
+Returns the length of the buffer in bytes. Does not include the terminating
+zero byte.

docs/Makefile.am

@@ -54,6 +54,7 @@ EXTRA_DIST =                                    \
  CONTRIBUTE.md                                  \
  CURL-DISABLE.md                                \
  DEPRECATE.md                                   \
+ DYNBUF.md                                      \
  ESNI.md                                        \
  EXPERIMENTAL.md                                \
  FAQ                                            \

lib/Makefile.inc

@@ -60,7 +60,7 @@ LIB_CFILES = altsvc.c amigaos.c asyn-ares.c asyn-thread.c base64.c            \
   sendf.c setopt.c sha256.c share.c slist.c smb.c smtp.c socketpair.c socks.c \
   socks_gssapi.c socks_sspi.c speedcheck.c splay.c strcase.c strdup.c         \
   strerror.c strtok.c strtoofft.c system_win32.c telnet.c tftp.c timeval.c    \
-  transfer.c urlapi.c version.c warnless.c wildcard.c x509asn1.c
+  transfer.c urlapi.c version.c warnless.c wildcard.c x509asn1.c dynbuf.c
 
 LIB_HFILES = altsvc.h amigaos.h arpa_telnet.h asyn.h conncache.h connect.h    \
   content_encoding.h cookie.h curl_addrinfo.h curl_base64.h curl_ctype.h      \
@@ -79,7 +79,7 @@ LIB_HFILES = altsvc.h amigaos.h arpa_telnet.h asyn.h conncache.h connect.h    \
   smb.h smtp.h sockaddr.h socketpair.h socks.h speedcheck.h splay.h strcase.h \
   strdup.h strerror.h strtok.h strtoofft.h system_win32.h telnet.h tftp.h     \
   timeval.h transfer.h urlapi-int.h urldata.h warnless.h wildcard.h           \
-  x509asn1.h
+  x509asn1.h dynbuf.h
 
 LIB_RCFILES = libcurl.rc
 

lib/curl_ntlm_wb.c

@@ -261,15 +261,11 @@ done:
 static CURLcode ntlm_wb_response(struct Curl_easy *data, struct ntlmdata *ntlm,
                                  const char *input, curlntlm state)
 {
-  char *buf = malloc(NTLM_BUFSIZE);
   size_t len_in = strlen(input), len_out = 0;
-
+  struct dynbuf b;
-#if defined(CURL_DISABLE_VERBOSE_STRINGS)
+  char *ptr = NULL;
-  (void) data;
+  unsigned char *buf = (unsigned char *)data->state.buffer;
-#endif
+  Curl_dyn_init(&b, MAX_NTLM_WB_RESPONSE);
-
-  if(!buf)
-    return CURLE_OUT_OF_MEMORY;
 
   while(len_in > 0) {
     ssize_t written = swrite(ntlm->ntlm_auth_hlpr_socket, input, len_in);
@@ -285,10 +281,8 @@ static CURLcode ntlm_wb_response(struct Curl_easy *data, struct ntlmdata *ntlm,
   }
   /* Read one line */
   while(1) {
-    ssize_t size;
+    ssize_t size =
-    char *newbuf;
+      sread(ntlm->ntlm_auth_hlpr_socket, buf, data->set.buffer_size);
-
-    size = sread(ntlm->ntlm_auth_hlpr_socket, buf + len_out, NTLM_BUFSIZE);
     if(size == -1) {
       if(errno == EINTR)
         continue;
@@ -297,48 +291,41 @@ static CURLcode ntlm_wb_response(struct Curl_easy *data, struct ntlmdata *ntlm,
     else if(size == 0)
       goto done;
 
-    len_out += size;
+    if(Curl_dyn_addn(&b, buf, size))
-    if(buf[len_out - 1] == '\n') {
+      goto done;
-      buf[len_out - 1] = '\0';
+
-      break;
+    len_out = Curl_dyn_len(&b);
+    ptr = Curl_dyn_ptr(&b);
+    if(len_out && ptr[len_out - 1] == '\n') {
+      ptr[len_out - 1] = '\0';
+      break; /* done! */
     }
-
+    /* loop */
-    if(len_out > MAX_NTLM_WB_RESPONSE) {
-      failf(data, "too large ntlm_wb response!");
-      free(buf);
-      return CURLE_OUT_OF_MEMORY;
-    }
-
-    newbuf = Curl_saferealloc(buf, len_out + NTLM_BUFSIZE);
-    if(!newbuf)
-      return CURLE_OUT_OF_MEMORY;
-
-    buf = newbuf;
   }
 
   /* Samba/winbind installed but not configured */
   if(state == NTLMSTATE_TYPE1 &&
      len_out == 3 &&
-     buf[0] == 'P' && buf[1] == 'W')
+     ptr[0] == 'P' && ptr[1] == 'W')
     goto done;
   /* invalid response */
   if(len_out < 4)
     goto done;
   if(state == NTLMSTATE_TYPE1 &&
-     (buf[0]!='Y' || buf[1]!='R' || buf[2]!=' '))
+     (ptr[0]!='Y' || ptr[1]!='R' || ptr[2]!=' '))
     goto done;
   if(state == NTLMSTATE_TYPE2 &&
-     (buf[0]!='K' || buf[1]!='K' || buf[2]!=' ') &&
+     (ptr[0]!='K' || ptr[1]!='K' || ptr[2]!=' ') &&
-     (buf[0]!='A' || buf[1]!='F' || buf[2]!=' '))
+     (ptr[0]!='A' || ptr[1]!='F' || ptr[2]!=' '))
     goto done;
 
-  ntlm->response = aprintf("%.*s", len_out - 4, buf + 3);
+  ntlm->response = strdup(ptr + 3);
-  free(buf);
+  Curl_dyn_free(&b);
   if(!ntlm->response)
     return CURLE_OUT_OF_MEMORY;
   return CURLE_OK;
 done:
-  free(buf);
+  Curl_dyn_free(&b);
   return CURLE_REMOTE_ACCESS_DENIED;
 }
 

lib/doh.c

@@ -35,13 +35,13 @@
 #include "curl_base64.h"
 #include "connect.h"
 #include "strdup.h"
+#include "dynbuf.h"
 /* The last 3 #include files should be in this order */
 #include "curl_printf.h"
 #include "curl_memory.h"
 #include "memdebug.h"
 
 #define DNS_CLASS_IN 0x01
-#define DOH_MAX_RESPONSE_SIZE 3000 /* bytes */
 
 #ifndef CURL_DISABLE_VERBOSE_STRINGS
 static const char * const errors[]={
@@ -177,20 +177,11 @@ static size_t
 doh_write_cb(const void *contents, size_t size, size_t nmemb, void *userp)
 {
   size_t realsize = size * nmemb;
-  struct dohresponse *mem = (struct dohresponse *)userp;
+  struct dynbuf *mem = (struct dynbuf *)userp;
 
-  if((mem->size + realsize) > DOH_MAX_RESPONSE_SIZE)
+  if(Curl_dyn_addn(mem, contents, realsize))
-    /* suspiciously much for us */
     return 0;
 
-  mem->memory = Curl_saferealloc(mem->memory, mem->size + realsize);
-  if(!mem->memory)
-    /* out of memory! */
-    return 0;
-
-  memcpy(&(mem->memory[mem->size]), contents, realsize);
-  mem->size += realsize;
-
   return realsize;
 }
 
@@ -238,10 +229,7 @@ static CURLcode dohprobe(struct Curl_easy *data,
   }
 
   p->dnstype = dnstype;
-  p->serverdoh.memory = NULL;
+  Curl_dyn_init(&p->serverdoh, DYN_DOH_RESPONSE);
-  /* the memory will be grown as needed by realloc in the doh_write_cb
-     function */
-  p->serverdoh.size = 0;
 
   /* Note: this is code for sending the DoH request with GET but there's still
      no logic that actually enables this. We should either add that ability or
@@ -272,7 +260,7 @@ static CURLcode dohprobe(struct Curl_easy *data,
   if(!result) {
     /* pass in the struct pointer via a local variable to please coverity and
        the gcc typecheck helpers */
-    struct dohresponse *resp = &p->serverdoh;
+    struct dynbuf *resp = &p->serverdoh;
     ERROR_CHECK_SETOPT(CURLOPT_URL, url);
     ERROR_CHECK_SETOPT(CURLOPT_WRITEFUNCTION, doh_write_cb);
     ERROR_CHECK_SETOPT(CURLOPT_WRITEDATA, resp);
@@ -506,38 +494,12 @@ static DOHcode store_aaaa(const unsigned char *doh,
   return DOH_OK;
 }
 
-static DOHcode cnameappend(struct cnamestore *c,
-                           const unsigned char *src,
-                           size_t len)
-{
-  if(!c->alloc) {
-    c->allocsize = len + 1;
-    c->alloc = malloc(c->allocsize);
-    if(!c->alloc)
-      return DOH_OUT_OF_MEM;
-  }
-  else if(c->allocsize < (c->allocsize + len + 1)) {
-    char *ptr;
-    c->allocsize += len + 1;
-    ptr = realloc(c->alloc, c->allocsize);
-    if(!ptr) {
-      free(c->alloc);
-      return DOH_OUT_OF_MEM;
-    }
-    c->alloc = ptr;
-  }
-  memcpy(&c->alloc[c->len], src, len);
-  c->len += len;
-  c->alloc[c->len] = 0; /* keep it zero terminated */
-  return DOH_OK;
-}
-
 static DOHcode store_cname(const unsigned char *doh,
                            size_t dohlen,
                            unsigned int index,
                            struct dohentry *d)
 {
-  struct cnamestore *c;
+  struct dynbuf *c;
   unsigned int loop = 128; /* a valid DNS name can never loop this much */
   unsigned char length;
 
@@ -566,18 +528,15 @@ static DOHcode store_cname(const unsigned char *doh,
       index++;
 
     if(length) {
-      DOHcode rc;
+      if(Curl_dyn_len(c)) {
-      if(c->len) {
+        if(Curl_dyn_add(c, "."))
-        rc = cnameappend(c, (unsigned char *)".", 1);
+          return DOH_OUT_OF_MEM;
-        if(rc)
-          return rc;
       }
       if((index + length) > dohlen)
         return DOH_DNS_BAD_LABEL;
 
-      rc = cnameappend(c, &doh[index], length);
+      if(Curl_dyn_addn(c, &doh[index], length))
-      if(rc)
+        return DOH_OUT_OF_MEM;
-        return rc;
       index += length;
     }
   } while(length && --loop);
@@ -630,10 +589,13 @@ static DOHcode rdata(const unsigned char *doh,
   return DOH_OK;
 }
 
-static void init_dohentry(struct dohentry *de)
+UNITTEST void de_init(struct dohentry *de)
 {
+  int i;
   memset(de, 0, sizeof(*de));
   de->ttl = INT_MAX;
+  for(i = 0; i < DOH_MAX_CNAME; i++)
+    Curl_dyn_init(&de->cname[i], DYN_DOH_CNAME);
 }
 
 
@@ -808,7 +770,7 @@ static void showdoh(struct Curl_easy *data,
     }
   }
   for(i = 0; i < d->numcname; i++) {
-    infof(data, "CNAME: %s\n", d->cname[i].alloc);
+    infof(data, "CNAME: %s\n", Curl_dyn_ptr(&d->cname[i]));
   }
 }
 #else
@@ -941,7 +903,7 @@ UNITTEST void de_cleanup(struct dohentry *d)
 {
   int i = 0;
   for(i = 0; i < d->numcname; i++) {
-    free(d->cname[i].alloc);
+    Curl_dyn_free(&d->cname[i]);
   }
 }
 
@@ -959,7 +921,9 @@ CURLcode Curl_doh_is_resolved(struct connectdata *conn,
       CURLE_COULDNT_RESOLVE_HOST;
   }
   else if(!data->req.doh.pending) {
-    DOHcode rc[DOH_PROBE_SLOTS];
+    DOHcode rc[DOH_PROBE_SLOTS] = {
+      DOH_OK, DOH_OK
+    };
     struct dohentry de;
     int slot;
     /* remove DOH handles from multi handle and close them */
@@ -968,17 +932,19 @@ CURLcode Curl_doh_is_resolved(struct connectdata *conn,
       Curl_close(&data->req.doh.probe[slot].easy);
     }
     /* parse the responses, create the struct and return it! */
-    init_dohentry(&de);
+    de_init(&de);
     for(slot = 0; slot < DOH_PROBE_SLOTS; slot++) {
-      rc[slot] = doh_decode(data->req.doh.probe[slot].serverdoh.memory,
+      struct dnsprobe *p = &data->req.doh.probe[slot];
-                            data->req.doh.probe[slot].serverdoh.size,
+      if(!p->dnstype)
-                            data->req.doh.probe[slot].dnstype,
+        continue;
+      rc[slot] = doh_decode(Curl_dyn_uptr(&p->serverdoh),
+                            Curl_dyn_len(&p->serverdoh),
+                            p->dnstype,
                             &de);
-      Curl_safefree(data->req.doh.probe[slot].serverdoh.memory);
+      Curl_dyn_free(&p->serverdoh);
       if(rc[slot]) {
         infof(data, "DOH: %s type %s for %s\n", doh_strerror(rc[slot]),
-              type2name(data->req.doh.probe[slot].dnstype),
+              type2name(p->dnstype), data->req.doh.host);
-              data->req.doh.host);
       }
     } /* next slot */
 

lib/doh.h

@@ -70,12 +70,6 @@ typedef enum {
 #define DOH_MAX_ADDR 24
 #define DOH_MAX_CNAME 4
 
-struct cnamestore {
-  size_t len;       /* length of cname */
-  char *alloc;      /* allocated pointer */
-  size_t allocsize; /* allocated size */
-};
-
 struct dohaddr {
   int type;
   union {
@@ -85,11 +79,11 @@ struct dohaddr {
 };
 
 struct dohentry {
-  unsigned int ttl;
+  struct dynbuf cname[DOH_MAX_CNAME];
-  int numaddr;
   struct dohaddr addr[DOH_MAX_ADDR];
+  int numaddr;
+  unsigned int ttl;
   int numcname;
-  struct cnamestore cname[DOH_MAX_CNAME];
 };
 
 
@@ -103,6 +97,7 @@ DOHcode doh_decode(const unsigned char *doh,
                    size_t dohlen,
                    DNStype dnstype,
                    struct dohentry *d);
+void de_init(struct dohentry *d);
 void de_cleanup(struct dohentry *d);
 #endif
 

lib/dynbuf.c

@@ -0,0 +1,228 @@
+/***************************************************************************
+ *                                  _   _ ____  _
+ *  Project                     ___| | | |  _ \| |
+ *                             / __| | | | |_) | |
+ *                            | (__| |_| |  _ <| |___
+ *                             \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * This software is licensed as described in the file COPYING, which
+ * you should have received as part of this distribution. The terms
+ * are also available at https://curl.haxx.se/docs/copyright.html.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the COPYING file.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ ***************************************************************************/
+
+#include "curl_setup.h"
+#include "strdup.h"
+#include "dynbuf.h"
+
+/* The last 3 #include files should be in this order */
+#include "curl_printf.h"
+#include "curl_memory.h"
+#include "memdebug.h"
+
+#define MIN_FIRST_ALLOC 32
+
+#define DYNINIT 0xbee51da /* random pattern */
+
+/*
+ * Init a dynbuf struct.
+ */
+void Curl_dyn_init(struct dynbuf *s, size_t toobig)
+{
+  DEBUGASSERT(s);
+  DEBUGASSERT(toobig);
+  s->bufr = NULL;
+  s->leng = 0;
+  s->allc = 0;
+  s->toobig = toobig;
+#ifdef DEBUGBUILD
+  s->init = DYNINIT;
+#endif
+}
+
+/*
+ * free the buffer and re-init the necessary fields. It doesn't touch the
+ * 'init' field and thus this buffer can be reused to add data to again.
+ */
+void Curl_dyn_free(struct dynbuf *s)
+{
+  DEBUGASSERT(s);
+  Curl_safefree(s->bufr);
+  s->leng = s->allc = 0;
+}
+
+/*
+ * Store/append an chunk of memory to the dynbuf.
+ */
+static CURLcode dyn_nappend(struct dynbuf *s,
+                            const unsigned char *mem, size_t len)
+{
+  size_t indx = s->leng;
+  size_t a = s->allc;
+  size_t fit = len + indx + 1; /* new string + old string + zero byte */
+
+  /* try to detect if there's rubbish in the struct */
+  DEBUGASSERT(s->init == DYNINIT);
+  DEBUGASSERT(s->toobig);
+  DEBUGASSERT(indx < s->toobig);
+  DEBUGASSERT(!s->leng || s->bufr);
+
+  if(fit > s->toobig) {
+    Curl_dyn_free(s);
+    return CURLE_OUT_OF_MEMORY;
+  }
+  else if(!a) {
+    DEBUGASSERT(!indx);
+    /* first invoke */
+    if(fit < MIN_FIRST_ALLOC)
+      a = MIN_FIRST_ALLOC;
+    else
+      a = fit;
+  }
+  else {
+    while(a < fit)
+      a *= 2;
+  }
+
+  if(a != s->allc) {
+    s->bufr = Curl_saferealloc(s->bufr, a);
+    if(!s->bufr) {
+      s->leng = s->allc = 0;
+      return CURLE_OUT_OF_MEMORY;
+    }
+    s->allc = a;
+  }
+
+  if(len)
+    memcpy(&s->bufr[indx], mem, len);
+  s->leng = indx + len;
+  s->bufr[s->leng] = 0;
+  return CURLE_OK;
+}
+
+/*
+ * Clears the string, keeps the allocation. This can also be called on a
+ * buffer that already was freed.
+ */
+void Curl_dyn_reset(struct dynbuf *s)
+{
+  DEBUGASSERT(s);
+  DEBUGASSERT(s->init == DYNINIT);
+  DEBUGASSERT(!s->leng || s->bufr);
+  if(s->leng)
+    s->bufr[0] = 0;
+  s->leng = 0;
+}
+
+#if 0
+/*
+ * Specify the size of the tail to keep (number of bytes from the end of the
+ * buffer). The rest will be dropped.
+ */
+CURLcode Curl_dyn_tail(struct dynbuf *s, size_t trail)
+{
+  DEBUGASSERT(s);
+  DEBUGASSERT(s->init == DYNINIT);
+  DEBUGASSERT(!s->leng || s->bufr);
+  if(trail > s->leng)
+    return CURLE_BAD_FUNCTION_ARGUMENT;
+  else if(trail == s->leng)
+    return CURLE_OK;
+  else if(!trail) {
+    Curl_dyn_reset(s);
+  }
+  else {
+    memmove(&s->bufr[0], &s->bufr[s->leng - trail], trail);
+    s->leng = trail;
+  }
+  return CURLE_OK;
+
+}
+
+#endif
+
+/*
+ * Appends a buffer with length.
+ */
+CURLcode Curl_dyn_addn(struct dynbuf *s, const void *mem, size_t len)
+{
+  DEBUGASSERT(s);
+  DEBUGASSERT(s->init == DYNINIT);
+  DEBUGASSERT(!s->leng || s->bufr);
+  return dyn_nappend(s, mem, len);
+}
+
+/*
+ * Append a zero terminated string at the end.
+ */
+CURLcode Curl_dyn_add(struct dynbuf *s, const char *str)
+{
+  size_t n = strlen(str);
+  DEBUGASSERT(s);
+  DEBUGASSERT(s->init == DYNINIT);
+  DEBUGASSERT(!s->leng || s->bufr);
+  return dyn_nappend(s, (unsigned char *)str, n);
+}
+
+/*
+ * Append a string printf()-style
+ */
+CURLcode Curl_dyn_addf(struct dynbuf *s, const char *fmt, ...)
+{
+  char *str;
+  va_list ap;
+  va_start(ap, fmt);
+  str = vaprintf(fmt, ap); /* this allocs a new string to append */
+  va_end(ap);
+
+  if(str) {
+    CURLcode result = dyn_nappend(s, (unsigned char *)str, strlen(str));
+    free(str);
+    return result;
+  }
+  /* If we failed, we cleanup the whole buffer and return error */
+  Curl_dyn_free(s);
+  return CURLE_OUT_OF_MEMORY;
+}
+
+/*
+ * Returns a pointer to the buffer.
+ */
+char *Curl_dyn_ptr(const struct dynbuf *s)
+{
+  DEBUGASSERT(s);
+  DEBUGASSERT(s->init == DYNINIT);
+  DEBUGASSERT(!s->leng || s->bufr);
+  return s->leng ? s->bufr : (char *)"";
+}
+
+/*
+ * Returns an unsigned pointer to the buffer.
+ */
+unsigned char *Curl_dyn_uptr(const struct dynbuf *s)
+{
+  DEBUGASSERT(s);
+  DEBUGASSERT(s->init == DYNINIT);
+  DEBUGASSERT(!s->leng || s->bufr);
+  return s->leng ? (unsigned char *)s->bufr : (unsigned char *)"";
+}
+
+/*
+ * Returns the length of the buffer.
+ */
+size_t Curl_dyn_len(const struct dynbuf *s)
+{
+  DEBUGASSERT(s);
+  DEBUGASSERT(s->init == DYNINIT);
+  DEBUGASSERT(!s->leng || s->bufr);
+  return s->leng;
+}

lib/dynbuf.h

@@ -0,0 +1,61 @@
+#ifndef HEADER_CURL_DYNBUF_H
+#define HEADER_CURL_DYNBUF_H
+/***************************************************************************
+ *                                  _   _ ____  _
+ *  Project                     ___| | | |  _ \| |
+ *                             / __| | | | |_) | |
+ *                            | (__| |_| |  _ <| |___
+ *                             \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * This software is licensed as described in the file COPYING, which
+ * you should have received as part of this distribution. The terms
+ * are also available at https://curl.haxx.se/docs/copyright.html.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the COPYING file.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ ***************************************************************************/
+
+struct dynbuf {
+  char *bufr;    /* point to a zero terminated allocated buffer */
+  size_t leng;   /* number of bytes *EXCLUDING* the zero terminator */
+  size_t allc;   /* size of the current allocation */
+  size_t toobig; /* size limit for the buffer */
+#ifdef DEBUGBUILD
+  int init;     /* detect API usage mistakes */
+#endif
+};
+
+void Curl_dyn_init(struct dynbuf *s, size_t toobig);
+void Curl_dyn_free(struct dynbuf *s);
+CURLcode Curl_dyn_addn(struct dynbuf *s, const void *mem, size_t len)
+  WARN_UNUSED_RESULT;
+CURLcode Curl_dyn_add(struct dynbuf *s, const char *str)
+  WARN_UNUSED_RESULT;
+CURLcode Curl_dyn_addf(struct dynbuf *s, const char *fmt, ...)
+  WARN_UNUSED_RESULT;
+void Curl_dyn_reset(struct dynbuf *s);
+CURLcode Curl_dyn_tail(struct dynbuf *s, size_t trail);
+char *Curl_dyn_ptr(const struct dynbuf *s);
+unsigned char *Curl_dyn_uptr(const struct dynbuf *s);
+size_t Curl_dyn_len(const struct dynbuf *s);
+
+/* Dynamic buffer max sizes */
+#define DYN_DOH_RESPONSE    3000
+#define DYN_DOH_CNAME       256
+#define DYN_PAUSE_BUFFER    (64 * 1024 * 1024)
+#define DYN_HAXPROXY        2048
+#define DYN_HTTP_REQUEST    (128*1024)
+#define DYN_H2_HEADERS      (128*1024)
+#define DYN_H2_TRAILERS     (128*1024)
+#define DYN_APRINTF         8000000
+#define DYN_RTSP_REQ_HEADER (64*1024)
+#define DYN_TRAILERS        (64*1024)
+
+#endif

lib/easy.c

@@ -77,6 +77,7 @@
 #include "http_digest.h"
 #include "system_win32.h"
 #include "http2.h"
+#include "dynbuf.h"
 
 /* The last 3 #include files should be in this order */
 #include "curl_printf.h"
@@ -820,15 +821,12 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
   if(!outcurl->state.buffer)
     goto fail;
 
-  outcurl->state.headerbuff = malloc(HEADERSIZE);
-  if(!outcurl->state.headerbuff)
-    goto fail;
-  outcurl->state.headersize = HEADERSIZE;
-
   /* copy all userdefined values */
   if(dupset(outcurl, data))
     goto fail;
 
+  Curl_dyn_init(&outcurl->state.headerb, CURL_MAX_HTTP_HEADER);
+
   /* the connection cache is setup on demand */
   outcurl->state.conn_cache = NULL;
 
@@ -921,7 +919,7 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data)
     curl_slist_free_all(outcurl->change.cookielist);
     outcurl->change.cookielist = NULL;
     Curl_safefree(outcurl->state.buffer);
-    Curl_safefree(outcurl->state.headerbuff);
+    Curl_dyn_free(&outcurl->state.headerb);
     Curl_safefree(outcurl->change.url);
     Curl_safefree(outcurl->change.referer);
     Curl_freeset(outcurl);
@@ -1040,7 +1038,7 @@ CURLcode curl_easy_pause(struct Curl_easy *data, int action)
       /* copy the structs to allow for immediate re-pausing */
       for(i = 0; i < data->state.tempcount; i++) {
         writebuf[i] = data->state.tempwrite[i];
-        data->state.tempwrite[i].buf = NULL;
+        Curl_dyn_init(&data->state.tempwrite[i].b, DYN_PAUSE_BUFFER);
       }
       data->state.tempcount = 0;
 
@@ -1054,9 +1052,10 @@ CURLcode curl_easy_pause(struct Curl_easy *data, int action)
         /* even if one function returns error, this loops through and frees
            all buffers */
         if(!result)
-          result = Curl_client_write(conn, writebuf[i].type, writebuf[i].buf,
+          result = Curl_client_write(conn, writebuf[i].type,
-                                     writebuf[i].len);
+                                     Curl_dyn_ptr(&writebuf[i].b),
-        free(writebuf[i].buf);
+                                     Curl_dyn_len(&writebuf[i].b));
+        Curl_dyn_free(&writebuf[i].b);
       }
 
       /* recover previous owner of the connection */

lib/easyif.h

@@ -7,7 +7,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms

lib/escape.c

@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -79,57 +79,42 @@ char *curl_unescape(const char *string, int length)
 char *curl_easy_escape(struct Curl_easy *data, const char *string,
                        int inlength)
 {
-  size_t alloc;
-  char *ns;
-  char *testing_ptr = NULL;
-  size_t newlen;
-  size_t strindex = 0;
   size_t length;
   CURLcode result;
+  struct dynbuf d;
 
   if(inlength < 0)
     return NULL;
 
-  alloc = (inlength?(size_t)inlength:strlen(string)) + 1;
+  Curl_dyn_init(&d, CURL_MAX_INPUT_LENGTH);
-  newlen = alloc;
 
-  ns = malloc(alloc);
+  length = (inlength?(size_t)inlength:strlen(string));
-  if(!ns)
-    return NULL;
-
-  length = alloc-1;
   while(length--) {
     unsigned char in = *string; /* we need to treat the characters unsigned */
 
-    if(Curl_isunreserved(in))
+    if(Curl_isunreserved(in)) {
-      /* just copy this */
+      /* append this */
-      ns[strindex++] = in;
+      if(Curl_dyn_addn(&d, &in, 1))
+        return NULL;
+    }
     else {
       /* encode it */
-      newlen += 2; /* the size grows with two, since this'll become a %XX */
+      char encoded[4];
-      if(newlen > alloc) {
-        alloc *= 2;
-        testing_ptr = Curl_saferealloc(ns, alloc);
-        if(!testing_ptr)
-          return NULL;
-        ns = testing_ptr;
-      }
-
       result = Curl_convert_to_network(data, (char *)&in, 1);
       if(result) {
         /* Curl_convert_to_network calls failf if unsuccessful */
-        free(ns);
+        Curl_dyn_free(&d);
         return NULL;
       }
 
-      msnprintf(&ns[strindex], 4, "%%%02X", in);
+      msnprintf(encoded, sizeof(encoded), "%%%02X", in);
-
+      if(Curl_dyn_add(&d, encoded))
-      strindex += 3;
+        return NULL;
     }
     string++;
   }
-  ns[strindex] = 0; /* terminate it */
+
-  return ns;
+  return Curl_dyn_ptr(&d);
 }
 
 /*

lib/http.h

@@ -44,38 +44,19 @@ char *Curl_copy_header_value(const char *header);
 
 char *Curl_checkProxyheaders(const struct connectdata *conn,
                              const char *thisheader);
-/* ------------------------------------------------------------------------- */
+CURLcode Curl_buffer_send(struct dynbuf *in,
-/*
+                          struct connectdata *conn,
- * The add_buffer series of functions are used to build one large memory chunk
+                          curl_off_t *bytes_written,
- * from repeated function invokes. Used so that the entire HTTP request can
+                          size_t included_body_bytes,
- * be sent in one go.
+                          int socketindex);
- */
-struct Curl_send_buffer {
-  char *buffer;
-  size_t size_max;
-  size_t size_used;
-};
-typedef struct Curl_send_buffer Curl_send_buffer;
-
-Curl_send_buffer *Curl_add_buffer_init(void);
-void Curl_add_buffer_free(Curl_send_buffer **inp);
-CURLcode Curl_add_bufferf(Curl_send_buffer **inp, const char *fmt, ...)
-  WARN_UNUSED_RESULT;
-CURLcode Curl_add_buffer(Curl_send_buffer **inp, const void *inptr,
-                         size_t size) WARN_UNUSED_RESULT;
-CURLcode Curl_add_buffer_send(Curl_send_buffer **inp,
-                              struct connectdata *conn,
-                              curl_off_t *bytes_written,
-                              size_t included_body_bytes,
-                              int socketindex);
 
 CURLcode Curl_add_timecondition(const struct connectdata *conn,
-                                Curl_send_buffer *buf);
+                                struct dynbuf *buf);
 CURLcode Curl_add_custom_headers(struct connectdata *conn,
                                  bool is_connect,
-                                 Curl_send_buffer *req_buffer);
+                                 struct dynbuf *req_buffer);
 CURLcode Curl_http_compile_trailers(struct curl_slist *trailers,
-                                    Curl_send_buffer **buffer,
+                                    struct dynbuf *buf,
                                     struct Curl_easy *handle);
 
 /* protocol-specific functions set up to be called by the main engine */
@@ -154,9 +135,9 @@ struct HTTP {
   } sending;
 
 #ifndef CURL_DISABLE_HTTP
-  Curl_send_buffer *send_buffer; /* used if the request couldn't be sent in
+  struct dynbuf send_buffer; /* used if the request couldn't be sent in one
-                                    one chunk, points to an allocated
+                                chunk, points to an allocated send_buffer
-                                    send_buffer struct */
+                                struct */
 #endif
 #ifdef USE_NGHTTP2
   /*********** for HTTP/2 we store stream-local data here *************/
@@ -164,10 +145,10 @@ struct HTTP {
 
   bool bodystarted;
   /* We store non-final and final response headers here, per-stream */
-  Curl_send_buffer *header_recvbuf;
+  struct dynbuf header_recvbuf;
   size_t nread_header_recvbuf; /* number of bytes in header_recvbuf fed into
                                   upper layer */
-  Curl_send_buffer *trailer_recvbuf;
+  struct dynbuf trailer_recvbuf;
   int status_code; /* HTTP status code */
   const uint8_t *pausedata; /* pointer to data received in on_data_chunk */
   size_t pauselen; /* the number of bytes left in data */

lib/http2.c

@@ -36,6 +36,7 @@
 #include "connect.h"
 #include "strtoofft.h"
 #include "strdup.h"
+#include "dynbuf.h"
 /* The last 3 #include files should be in this order */
 #include "curl_printf.h"
 #include "curl_memory.h"
@@ -124,8 +125,8 @@ static int http2_getsock(struct connectdata *conn,
 static void http2_stream_free(struct HTTP *http)
 {
   if(http) {
-    Curl_add_buffer_free(&http->header_recvbuf);
+    Curl_dyn_free(&http->header_recvbuf);
-    Curl_add_buffer_free(&http->trailer_recvbuf);
+    Curl_dyn_free(&http->trailer_recvbuf);
     for(; http->push_headers_used > 0; --http->push_headers_used) {
       free(http->push_headers[http->push_headers_used - 1]);
     }
@@ -259,7 +260,6 @@ void Curl_http2_setup_req(struct Curl_easy *data)
 {
   struct HTTP *http = data->req.protop;
 
-  http->nread_header_recvbuf = 0;
   http->bodystarted = FALSE;
   http->status_code = -1;
   http->pausedata = NULL;
@@ -463,15 +463,9 @@ static struct Curl_easy *duphandle(struct Curl_easy *data)
     }
     else {
       second->req.protop = http;
-      http->header_recvbuf = Curl_add_buffer_init();
+      Curl_dyn_init(&http->header_recvbuf, DYN_H2_HEADERS);
-      if(!http->header_recvbuf) {
+      Curl_http2_setup_req(second);
-        free(http);
+      second->state.stream_weight = data->state.stream_weight;
-        (void)Curl_close(&second);
-      }
-      else {
-        Curl_http2_setup_req(second);
-        second->state.stream_weight = data->state.stream_weight;
-      }
     }
   }
   return second;
@@ -668,15 +662,17 @@ static int on_frame_recv(nghttp2_session *session, const nghttp2_frame *frame,
       stream->status_code = -1;
     }
 
-    result = Curl_add_buffer(&stream->header_recvbuf, "\r\n", 2);
+    result = Curl_dyn_add(&stream->header_recvbuf, "\r\n");
     if(result)
       return NGHTTP2_ERR_CALLBACK_FAILURE;
 
-    left = stream->header_recvbuf->size_used - stream->nread_header_recvbuf;
+    left = Curl_dyn_len(&stream->header_recvbuf) -
+      stream->nread_header_recvbuf;
     ncopy = CURLMIN(stream->len, left);
 
     memcpy(&stream->mem[stream->memlen],
-           stream->header_recvbuf->buffer + stream->nread_header_recvbuf,
+           Curl_dyn_ptr(&stream->header_recvbuf) +
+           stream->nread_header_recvbuf,
            ncopy);
     stream->nread_header_recvbuf += ncopy;
 
@@ -852,12 +848,6 @@ static int on_begin_headers(nghttp2_session *session,
     return 0;
   }
 
-  if(!stream->trailer_recvbuf) {
-    stream->trailer_recvbuf = Curl_add_buffer_init();
-    if(!stream->trailer_recvbuf) {
-      return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
-    }
-  }
   return 0;
 }
 
@@ -980,19 +970,19 @@ static int on_header(nghttp2_session *session, const nghttp2_frame *frame,
     H2BUGF(infof(data_s, "h2 trailer: %.*s: %.*s\n", namelen, name, valuelen,
                  value));
 
-    result = Curl_add_buffer(&stream->trailer_recvbuf, &n, sizeof(n));
+    result = Curl_dyn_addn(&stream->trailer_recvbuf, &n, sizeof(n));
     if(result)
       return NGHTTP2_ERR_CALLBACK_FAILURE;
-    result = Curl_add_buffer(&stream->trailer_recvbuf, name, namelen);
+    result = Curl_dyn_addn(&stream->trailer_recvbuf, name, namelen);
     if(result)
       return NGHTTP2_ERR_CALLBACK_FAILURE;
-    result = Curl_add_buffer(&stream->trailer_recvbuf, ": ", 2);
+    result = Curl_dyn_add(&stream->trailer_recvbuf, ": ");
     if(result)
       return NGHTTP2_ERR_CALLBACK_FAILURE;
-    result = Curl_add_buffer(&stream->trailer_recvbuf, value, valuelen);
+    result = Curl_dyn_addn(&stream->trailer_recvbuf, value, valuelen);
     if(result)
       return NGHTTP2_ERR_CALLBACK_FAILURE;
-    result = Curl_add_buffer(&stream->trailer_recvbuf, "\r\n\0", 3);
+    result = Curl_dyn_add(&stream->trailer_recvbuf, "\r\n\0");
     if(result)
       return NGHTTP2_ERR_CALLBACK_FAILURE;
 
@@ -1007,14 +997,14 @@ static int on_header(nghttp2_session *session, const nghttp2_frame *frame,
     stream->status_code = decode_status_code(value, valuelen);
     DEBUGASSERT(stream->status_code != -1);
 
-    result = Curl_add_buffer(&stream->header_recvbuf, "HTTP/2 ", 7);
+    result = Curl_dyn_add(&stream->header_recvbuf, "HTTP/2 ");
     if(result)
       return NGHTTP2_ERR_CALLBACK_FAILURE;
-    result = Curl_add_buffer(&stream->header_recvbuf, value, valuelen);
+    result = Curl_dyn_addn(&stream->header_recvbuf, value, valuelen);
     if(result)
       return NGHTTP2_ERR_CALLBACK_FAILURE;
     /* the space character after the status code is mandatory */
-    result = Curl_add_buffer(&stream->header_recvbuf, " \r\n", 3);
+    result = Curl_dyn_add(&stream->header_recvbuf, " \r\n");
     if(result)
       return NGHTTP2_ERR_CALLBACK_FAILURE;
     /* if we receive data for another handle, wake that up */
@@ -1029,16 +1019,16 @@ static int on_header(nghttp2_session *session, const nghttp2_frame *frame,
   /* nghttp2 guarantees that namelen > 0, and :status was already
      received, and this is not pseudo-header field . */
   /* convert to a HTTP1-style header */
-  result = Curl_add_buffer(&stream->header_recvbuf, name, namelen);
+  result = Curl_dyn_addn(&stream->header_recvbuf, name, namelen);
   if(result)
     return NGHTTP2_ERR_CALLBACK_FAILURE;
-  result = Curl_add_buffer(&stream->header_recvbuf, ": ", 2);
+  result = Curl_dyn_add(&stream->header_recvbuf, ": ");
   if(result)
     return NGHTTP2_ERR_CALLBACK_FAILURE;
-  result = Curl_add_buffer(&stream->header_recvbuf, value, valuelen);
+  result = Curl_dyn_addn(&stream->header_recvbuf, value, valuelen);
   if(result)
     return NGHTTP2_ERR_CALLBACK_FAILURE;
-  result = Curl_add_buffer(&stream->header_recvbuf, "\r\n", 2);
+  result = Curl_dyn_add(&stream->header_recvbuf, "\r\n");
   if(result)
     return NGHTTP2_ERR_CALLBACK_FAILURE;
   /* if we receive data for another handle, wake that up */
@@ -1139,17 +1129,15 @@ void Curl_http2_done(struct Curl_easy *data, bool premature)
 
   /* there might be allocated resources done before this got the 'h2' pointer
      setup */
-  if(http->header_recvbuf) {
+  Curl_dyn_free(&http->header_recvbuf);
-    Curl_add_buffer_free(&http->header_recvbuf);
+  Curl_dyn_free(&http->trailer_recvbuf);
-    Curl_add_buffer_free(&http->trailer_recvbuf);
+  if(http->push_headers) {
-    if(http->push_headers) {
+    /* if they weren't used and then freed before */
-      /* if they weren't used and then freed before */
+    for(; http->push_headers_used > 0; --http->push_headers_used) {
-      for(; http->push_headers_used > 0; --http->push_headers_used) {
+      free(http->push_headers[http->push_headers_used - 1]);
-        free(http->push_headers[http->push_headers_used - 1]);
-      }
-      free(http->push_headers);
-      http->push_headers = NULL;
     }
+    free(http->push_headers);
+    http->push_headers = NULL;
   }
 
   if(!httpc->h2) /* not HTTP/2 ? */
@@ -1238,7 +1226,7 @@ static CURLcode http2_init(struct connectdata *conn)
 /*
  * Append headers to ask for a HTTP1.1 to HTTP2 upgrade.
  */
-CURLcode Curl_http2_request_upgrade(Curl_send_buffer *req,
+CURLcode Curl_http2_request_upgrade(struct dynbuf *req,
                                     struct connectdata *conn)
 {
   CURLcode result;
@@ -1257,7 +1245,7 @@ CURLcode Curl_http2_request_upgrade(Curl_send_buffer *req,
                                          httpc->local_settings_num);
   if(!binlen) {
     failf(conn->data, "nghttp2 unexpectedly failed on pack_settings_payload");
-    Curl_add_buffer_free(&req);
+    Curl_dyn_free(req);
     return CURLE_FAILED_INIT;
   }
   conn->proto.httpc.binlen = binlen;
@@ -1265,15 +1253,15 @@ CURLcode Curl_http2_request_upgrade(Curl_send_buffer *req,
   result = Curl_base64url_encode(conn->data, (const char *)binsettings, binlen,
                                  &base64, &blen);
   if(result) {
-    Curl_add_buffer_free(&req);
+    Curl_dyn_free(req);
     return result;
   }
 
-  result = Curl_add_bufferf(&req,
+  result = Curl_dyn_addf(req,
-                            "Connection: Upgrade, HTTP2-Settings\r\n"
+                         "Connection: Upgrade, HTTP2-Settings\r\n"
-                            "Upgrade: %s\r\n"
+                         "Upgrade: %s\r\n"
-                            "HTTP2-Settings: %s\r\n",
+                         "HTTP2-Settings: %s\r\n",
-                            NGHTTP2_CLEARTEXT_PROTO_VERSION_ID, base64);
+                         NGHTTP2_CLEARTEXT_PROTO_VERSION_ID, base64);
   free(base64);
 
   k->upgr101 = UPGR101_REQUESTED;
@@ -1432,9 +1420,9 @@ static ssize_t http2_handle_stream_close(struct connectdata *conn,
     return -1;
   }
 
-  if(stream->trailer_recvbuf && stream->trailer_recvbuf->buffer) {
+  if(Curl_dyn_len(&stream->trailer_recvbuf)) {
-    trailer_pos = stream->trailer_recvbuf->buffer;
+    trailer_pos = Curl_dyn_ptr(&stream->trailer_recvbuf);
-    trailer_end = trailer_pos + stream->trailer_recvbuf->size_used;
+    trailer_end = trailer_pos + Curl_dyn_len(&stream->trailer_recvbuf);
 
     for(; trailer_pos < trailer_end;) {
       uint32_t n;
@@ -1541,13 +1529,13 @@ static ssize_t http2_recv(struct connectdata *conn, int sockindex,
    */
 
   if(stream->bodystarted &&
-     stream->nread_header_recvbuf < stream->header_recvbuf->size_used) {
+     stream->nread_header_recvbuf < Curl_dyn_len(&stream->header_recvbuf)) {
-    /* If there is body data pending for this stream to return, do that */
+    /* If there is header data pending for this stream to return, do that */
     size_t left =
-      stream->header_recvbuf->size_used - stream->nread_header_recvbuf;
+      Curl_dyn_len(&stream->header_recvbuf) - stream->nread_header_recvbuf;
     size_t ncopy = CURLMIN(len, left);
-    memcpy(mem, stream->header_recvbuf->buffer + stream->nread_header_recvbuf,
+    memcpy(mem, Curl_dyn_ptr(&stream->header_recvbuf) +
-           ncopy);
+           stream->nread_header_recvbuf, ncopy);
     stream->nread_header_recvbuf += ncopy;
 
     H2BUGF(infof(data, "http2_recv: Got %d bytes from header_recvbuf\n",
@@ -2129,11 +2117,8 @@ CURLcode Curl_http2_setup(struct connectdata *conn)
 
   stream->stream_id = -1;
 
-  if(!stream->header_recvbuf) {
+  Curl_dyn_init(&stream->header_recvbuf, DYN_H2_HEADERS);
-    stream->header_recvbuf = Curl_add_buffer_init();
+  Curl_dyn_init(&stream->trailer_recvbuf, DYN_H2_TRAILERS);
-    if(!stream->header_recvbuf)
-      return CURLE_OUT_OF_MEMORY;
-  }
 
   if((conn->handler == &Curl_handler_http2_ssl) ||
      (conn->handler == &Curl_handler_http2))
@@ -2146,7 +2131,7 @@ CURLcode Curl_http2_setup(struct connectdata *conn)
 
   result = http2_init(conn);
   if(result) {
-    Curl_add_buffer_free(&stream->header_recvbuf);
+    Curl_dyn_free(&stream->header_recvbuf);
     return result;
   }
 

lib/http2.h

@@ -42,7 +42,7 @@ const char *Curl_http2_strerror(uint32_t err);
 CURLcode Curl_http2_init(struct connectdata *conn);
 void Curl_http2_init_state(struct UrlState *state);
 void Curl_http2_init_userset(struct UserDefined *set);
-CURLcode Curl_http2_request_upgrade(Curl_send_buffer *req,
+CURLcode Curl_http2_request_upgrade(struct dynbuf *req,
                                     struct connectdata *conn);
 CURLcode Curl_http2_setup(struct connectdata *conn);
 CURLcode Curl_http2_switched(struct connectdata *conn,

lib/http_proxy.c

@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -204,7 +204,7 @@ static CURLcode CONNECT(struct connectdata *conn,
     if(TUNNEL_INIT == s->tunnel_state) {
       /* BEGIN CONNECT PHASE */
       char *host_port;
-      Curl_send_buffer *req_buffer;
+      struct dynbuf req_buffer;
 
       infof(data, "Establish HTTP proxy tunnel to %s:%d\n",
             hostname, remote_port);
@@ -215,17 +215,12 @@ static CURLcode CONNECT(struct connectdata *conn,
       free(data->req.newurl);
       data->req.newurl = NULL;
 
-      /* initialize a dynamic send-buffer */
-      req_buffer = Curl_add_buffer_init();
-
-      if(!req_buffer)
-        return CURLE_OUT_OF_MEMORY;
-
       host_port = aprintf("%s:%d", hostname, remote_port);
-      if(!host_port) {
+      if(!host_port)
-        Curl_add_buffer_free(&req_buffer);
         return CURLE_OUT_OF_MEMORY;
-      }
+
+      /* initialize a dynamic send-buffer */
+      Curl_dyn_init(&req_buffer, DYN_HTTP_REQUEST);
 
       /* Setup the proxy-authorization header, if any */
       result = Curl_http_output_auth(conn, "CONNECT", host_port, TRUE);
@@ -248,7 +243,7 @@ static CURLcode CONNECT(struct connectdata *conn,
           aprintf("%s%s%s:%d", ipv6_ip?"[":"", hostname, ipv6_ip?"]":"",
                   remote_port);
         if(!hostheader) {
-          Curl_add_buffer_free(&req_buffer);
+          Curl_dyn_free(&req_buffer);
           return CURLE_OUT_OF_MEMORY;
         }
 
@@ -256,7 +251,7 @@ static CURLcode CONNECT(struct connectdata *conn,
           host = aprintf("Host: %s\r\n", hostheader);
           if(!host) {
             free(hostheader);
-            Curl_add_buffer_free(&req_buffer);
+            Curl_dyn_free(&req_buffer);
             return CURLE_OUT_OF_MEMORY;
           }
         }
@@ -268,44 +263,43 @@ static CURLcode CONNECT(struct connectdata *conn,
           useragent = conn->allocptr.uagent;
 
         result =
-          Curl_add_bufferf(&req_buffer,
+          Curl_dyn_addf(&req_buffer,
-                           "CONNECT %s HTTP/%s\r\n"
+                        "CONNECT %s HTTP/%s\r\n"
-                           "%s"  /* Host: */
+                        "%s"  /* Host: */
-                           "%s"  /* Proxy-Authorization */
+                        "%s"  /* Proxy-Authorization */
-                           "%s"  /* User-Agent */
+                        "%s"  /* User-Agent */
-                           "%s", /* Proxy-Connection */
+                        "%s", /* Proxy-Connection */
-                           hostheader,
+                        hostheader,
-                           http,
+                        http,
-                           host?host:"",
+                        host?host:"",
-                           conn->allocptr.proxyuserpwd?
+                        conn->allocptr.proxyuserpwd?
-                           conn->allocptr.proxyuserpwd:"",
+                        conn->allocptr.proxyuserpwd:"",
-                           useragent,
+                        useragent,
-                           proxyconn);
+                        proxyconn);
 
         if(host)
           free(host);
         free(hostheader);
 
         if(!result)
-          result = Curl_add_custom_headers(conn, TRUE, req_buffer);
+          result = Curl_add_custom_headers(conn, TRUE, &req_buffer);
 
         if(!result)
           /* CRLF terminate the request */
-          result = Curl_add_bufferf(&req_buffer, "\r\n");
+          result = Curl_dyn_addf(&req_buffer, "\r\n");
 
         if(!result) {
           /* Send the connect request to the proxy */
           /* BLOCKING */
           result =
-            Curl_add_buffer_send(&req_buffer, conn,
+            Curl_buffer_send(&req_buffer, conn,
-                                 &data->info.request_size, 0, sockindex);
+                             &data->info.request_size, 0, sockindex);
         }
-        req_buffer = NULL;
         if(result)
           failf(data, "Failed sending CONNECT to proxy");
       }
 
-      Curl_add_buffer_free(&req_buffer);
+      Curl_dyn_free(&req_buffer);
       if(result)
         return result;
 

lib/mprintf.c

@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1999 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1999 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -36,6 +36,7 @@
  */
 
 #include "curl_setup.h"
+#include "dynbuf.h"
 #include <curl/mprintf.h>
 
 #include "curl_memory.h"
@@ -168,11 +169,9 @@ struct nsprintf {
 };
 
 struct asprintf {
-  char *buffer; /* allocated buffer */
+  struct dynbuf b;
-  size_t len;   /* length of string */
+  bool fail; /* if an alloc has failed and thus the output is not the complete
-  size_t alloc; /* length of alloc */
+                data */
-  int fail;     /* (!= 0) if an alloc has failed and thus
-                   the output is not the complete data */
 };
 
 static long dprintf_DollarString(char *input, char **end)
@@ -1031,35 +1030,10 @@ static int alloc_addbyter(int output, FILE *data)
   struct asprintf *infop = (struct asprintf *)data;
   unsigned char outc = (unsigned char)output;
 
-  if(!infop->buffer) {
+  if(Curl_dyn_addn(&infop->b, &outc, 1)) {
-    infop->buffer = malloc(32);
+    infop->fail = 1;
-    if(!infop->buffer) {
+    return -1; /* fail */
-      infop->fail = 1;
-      return -1; /* fail */
-    }
-    infop->alloc = 32;
-    infop->len = 0;
   }
-  else if(infop->len + 1 >= infop->alloc) {
-    char *newptr = NULL;
-    size_t newsize = infop->alloc*2;
-
-    /* detect wrap-around or other overflow problems */
-    if(newsize > infop->alloc)
-      newptr = realloc(infop->buffer, newsize);
-
-    if(!newptr) {
-      infop->fail = 1;
-      return -1; /* fail */
-    }
-    infop->buffer = newptr;
-    infop->alloc = newsize;
-  }
-
-  infop->buffer[ infop->len ] = outc;
-
-  infop->len++;
-
   return outc; /* fputc() returns like this on success */
 }
 
@@ -1068,24 +1042,18 @@ char *curl_maprintf(const char *format, ...)
   va_list ap_save; /* argument pointer */
   int retcode;
   struct asprintf info;
-
+  Curl_dyn_init(&info.b, DYN_APRINTF);
-  info.buffer = NULL;
-  info.len = 0;
-  info.alloc = 0;
   info.fail = 0;
 
   va_start(ap_save, format);
   retcode = dprintf_formatf(&info, alloc_addbyter, format, ap_save);
   va_end(ap_save);
   if((-1 == retcode) || info.fail) {
-    if(info.alloc)
+    Curl_dyn_free(&info.b);
-      free(info.buffer);
     return NULL;
   }
-  if(info.alloc) {
+  if(Curl_dyn_len(&info.b))
-    info.buffer[info.len] = 0; /* we terminate this with a zero byte */
+    return Curl_dyn_ptr(&info.b);
-    return info.buffer;
-  }
   return strdup("");
 }
 
@@ -1093,23 +1061,16 @@ char *curl_mvaprintf(const char *format, va_list ap_save)
 {
   int retcode;
   struct asprintf info;
-
+  Curl_dyn_init(&info.b, DYN_APRINTF);
-  info.buffer = NULL;
-  info.len = 0;
-  info.alloc = 0;
   info.fail = 0;
 
   retcode = dprintf_formatf(&info, alloc_addbyter, format, ap_save);
   if((-1 == retcode) || info.fail) {
-    if(info.alloc)
+    Curl_dyn_free(&info.b);
-      free(info.buffer);
     return NULL;
   }
-
+  if(Curl_dyn_len(&info.b))
-  if(info.alloc) {
+    return Curl_dyn_ptr(&info.b);
-    info.buffer[info.len] = 0; /* we terminate this with a zero byte */
-    return info.buffer;
-  }
   return strdup("");
 }
 

lib/multi.c

@@ -616,7 +616,7 @@ static CURLcode multi_done(struct Curl_easy *data,
   /* if the transfer was completed in a paused state there can be buffered
      data left to free */
   for(i = 0; i < data->state.tempcount; i++) {
-    free(data->state.tempwrite[i].buf);
+    Curl_dyn_free(&data->state.tempwrite[i].b);
   }
   data->state.tempcount = 0;
 

lib/rtsp.c

@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -233,7 +233,7 @@ static CURLcode rtsp_do(struct connectdata *conn, bool *done)
   CURLcode result = CURLE_OK;
   Curl_RtspReq rtspreq = data->set.rtspreq;
   struct RTSP *rtsp = data->req.protop;
-  Curl_send_buffer *req_buffer;
+  struct dynbuf req_buffer;
   curl_off_t postsize = 0; /* for ANNOUNCE and SET_PARAMETER */
   curl_off_t putsize = 0; /* for ANNOUNCE and SET_PARAMETER */
 
@@ -430,16 +430,13 @@ static CURLcode rtsp_do(struct connectdata *conn, bool *done)
   }
 
   /* Initialize a dynamic send buffer */
-  req_buffer = Curl_add_buffer_init();
+  Curl_dyn_init(&req_buffer, DYN_RTSP_REQ_HEADER);
-
-  if(!req_buffer)
-    return CURLE_OUT_OF_MEMORY;
 
   result =
-    Curl_add_bufferf(&req_buffer,
+    Curl_dyn_addf(&req_buffer,
-                     "%s %s RTSP/1.0\r\n" /* Request Stream-URI RTSP/1.0 */
+                  "%s %s RTSP/1.0\r\n" /* Request Stream-URI RTSP/1.0 */
-                     "CSeq: %ld\r\n", /* CSeq */
+                  "CSeq: %ld\r\n", /* CSeq */
-                     p_request, p_stream_uri, rtsp->CSeq_sent);
+                  p_request, p_stream_uri, rtsp->CSeq_sent);
   if(result)
     return result;
 
@@ -448,7 +445,7 @@ static CURLcode rtsp_do(struct connectdata *conn, bool *done)
    * to make comparison easier
    */
   if(p_session_id) {
-    result = Curl_add_bufferf(&req_buffer, "Session: %s\r\n", p_session_id);
+    result = Curl_dyn_addf(&req_buffer, "Session: %s\r\n", p_session_id);
     if(result)
       return result;
   }
@@ -456,24 +453,24 @@ static CURLcode rtsp_do(struct connectdata *conn, bool *done)
   /*
    * Shared HTTP-like options
    */
-  result = Curl_add_bufferf(&req_buffer,
+  result = Curl_dyn_addf(&req_buffer,
-                            "%s" /* transport */
+                         "%s" /* transport */
-                            "%s" /* accept */
+                         "%s" /* accept */
-                            "%s" /* accept-encoding */
+                         "%s" /* accept-encoding */
-                            "%s" /* range */
+                         "%s" /* range */
-                            "%s" /* referrer */
+                         "%s" /* referrer */
-                            "%s" /* user-agent */
+                         "%s" /* user-agent */
-                            "%s" /* proxyuserpwd */
+                         "%s" /* proxyuserpwd */
-                            "%s" /* userpwd */
+                         "%s" /* userpwd */
-                            ,
+                         ,
-                            p_transport ? p_transport : "",
+                         p_transport ? p_transport : "",
-                            p_accept ? p_accept : "",
+                         p_accept ? p_accept : "",
-                            p_accept_encoding ? p_accept_encoding : "",
+                         p_accept_encoding ? p_accept_encoding : "",
-                            p_range ? p_range : "",
+                         p_range ? p_range : "",
-                            p_referrer ? p_referrer : "",
+                         p_referrer ? p_referrer : "",
-                            p_uagent ? p_uagent : "",
+                         p_uagent ? p_uagent : "",
-                            p_proxyuserpwd ? p_proxyuserpwd : "",
+                         p_proxyuserpwd ? p_proxyuserpwd : "",
-                            p_userpwd ? p_userpwd : "");
+                         p_userpwd ? p_userpwd : "");
 
   /*
    * Free userpwd now --- cannot reuse this for Negotiate and possibly NTLM
@@ -486,12 +483,12 @@ static CURLcode rtsp_do(struct connectdata *conn, bool *done)
     return result;
 
   if((rtspreq == RTSPREQ_SETUP) || (rtspreq == RTSPREQ_DESCRIBE)) {
-    result = Curl_add_timecondition(conn, req_buffer);
+    result = Curl_add_timecondition(conn, &req_buffer);
     if(result)
       return result;
   }
 
-  result = Curl_add_custom_headers(conn, FALSE, req_buffer);
+  result = Curl_add_custom_headers(conn, FALSE, &req_buffer);
   if(result)
     return result;
 
@@ -516,9 +513,9 @@ static CURLcode rtsp_do(struct connectdata *conn, bool *done)
        * actually set a custom Content-Length in the headers */
       if(!Curl_checkheaders(conn, "Content-Length")) {
         result =
-          Curl_add_bufferf(&req_buffer,
+          Curl_dyn_addf(&req_buffer,
-                           "Content-Length: %" CURL_FORMAT_CURL_OFF_T"\r\n",
+                        "Content-Length: %" CURL_FORMAT_CURL_OFF_T"\r\n",
-                           (data->set.upload ? putsize : postsize));
+                        (data->set.upload ? putsize : postsize));
         if(result)
           return result;
       }
@@ -526,8 +523,8 @@ static CURLcode rtsp_do(struct connectdata *conn, bool *done)
       if(rtspreq == RTSPREQ_SET_PARAMETER ||
          rtspreq == RTSPREQ_GET_PARAMETER) {
         if(!Curl_checkheaders(conn, "Content-Type")) {
-          result = Curl_add_bufferf(&req_buffer,
+          result = Curl_dyn_addf(&req_buffer,
-                                    "Content-Type: text/parameters\r\n");
+                                 "Content-Type: text/parameters\r\n");
           if(result)
             return result;
         }
@@ -535,8 +532,8 @@ static CURLcode rtsp_do(struct connectdata *conn, bool *done)
 
       if(rtspreq == RTSPREQ_ANNOUNCE) {
         if(!Curl_checkheaders(conn, "Content-Type")) {
-          result = Curl_add_bufferf(&req_buffer,
+          result = Curl_dyn_addf(&req_buffer,
-                                    "Content-Type: application/sdp\r\n");
+                                 "Content-Type: application/sdp\r\n");
           if(result)
             return result;
         }
@@ -554,20 +551,20 @@ static CURLcode rtsp_do(struct connectdata *conn, bool *done)
   /* RTSP never allows chunked transfer */
   data->req.forbidchunk = TRUE;
   /* Finish the request buffer */
-  result = Curl_add_buffer(&req_buffer, "\r\n", 2);
+  result = Curl_dyn_add(&req_buffer, "\r\n");
   if(result)
     return result;
 
   if(postsize > 0) {
-    result = Curl_add_buffer(&req_buffer, data->set.postfields,
+    result = Curl_dyn_addn(&req_buffer, data->set.postfields,
-                             (size_t)postsize);
+                           (size_t)postsize);
     if(result)
       return result;
   }
 
   /* issue the request */
-  result = Curl_add_buffer_send(&req_buffer, conn,
+  result = Curl_buffer_send(&req_buffer, conn,
-                                &data->info.request_size, 0, FIRSTSOCKET);
+                            &data->info.request_size, 0, FIRSTSOCKET);
   if(result) {
     failf(data, "Failed sending RTSP request");
     return result;

lib/sendf.c

@@ -498,7 +498,6 @@ static CURLcode pausewrite(struct Curl_easy *data,
      is again enabled */
   struct SingleRequest *k = &data->req;
   struct UrlState *s = &data->state;
-  char *dupl;
   unsigned int i;
   bool newtype = TRUE;
 
@@ -518,44 +517,21 @@ static CURLcode pausewrite(struct Curl_easy *data,
   else
     i = 0;
 
-  if(!newtype) {
+  if(newtype) {
-    /* append new data to old data */
-
-    /* figure out the new size of the data to save */
-    size_t newlen = len + s->tempwrite[i].len;
-    /* allocate the new memory area */
-    char *newptr = realloc(s->tempwrite[i].buf, newlen);
-    if(!newptr)
-      return CURLE_OUT_OF_MEMORY;
-    /* copy the new data to the end of the new area */
-    memcpy(newptr + s->tempwrite[i].len, ptr, len);
-
-    /* update the pointer and the size */
-    s->tempwrite[i].buf = newptr;
-    s->tempwrite[i].len = newlen;
-
-    len = newlen; /* for the debug output below */
-  }
-  else {
-    dupl = Curl_memdup(ptr, len);
-    if(!dupl)
-      return CURLE_OUT_OF_MEMORY;
-
     /* store this information in the state struct for later use */
-    s->tempwrite[i].buf = dupl;
+    Curl_dyn_init(&s->tempwrite[i].b, DYN_PAUSE_BUFFER);
-    s->tempwrite[i].len = len;
     s->tempwrite[i].type = type;
 
     if(newtype)
       s->tempcount++;
   }
 
+  if(Curl_dyn_addn(&s->tempwrite[i].b, (unsigned char *)ptr, len))
+    return CURLE_OUT_OF_MEMORY;
+
   /* mark the connection as RECV paused */
   k->keepon |= KEEP_RECV_PAUSE;
 
-  DEBUGF(infof(data, "Paused %zu bytes in buffer for type %02x\n",
-               len, type));
-
   return CURLE_OK;
 }
 

lib/transfer.c

@@ -128,12 +128,13 @@ static size_t Curl_trailers_read(char *buffer, size_t size, size_t nitems,
                                  void *raw)
 {
   struct Curl_easy *data = (struct Curl_easy *)raw;
-  Curl_send_buffer *trailers_buf = data->state.trailers_buf;
+  struct dynbuf *trailers_buf = &data->state.trailers_buf;
-  size_t bytes_left = trailers_buf->size_used-data->state.trailers_bytes_sent;
+  size_t bytes_left = Curl_dyn_len(trailers_buf) -
+    data->state.trailers_bytes_sent;
   size_t to_copy = (size*nitems < bytes_left) ? size*nitems : bytes_left;
   if(to_copy) {
     memcpy(buffer,
-           &trailers_buf->buffer[data->state.trailers_bytes_sent],
+           Curl_dyn_ptr(trailers_buf) + data->state.trailers_bytes_sent,
            to_copy);
     data->state.trailers_bytes_sent += to_copy;
   }
@@ -143,8 +144,8 @@ static size_t Curl_trailers_read(char *buffer, size_t size, size_t nitems,
 static size_t Curl_trailers_left(void *raw)
 {
   struct Curl_easy *data = (struct Curl_easy *)raw;
-  Curl_send_buffer *trailers_buf = data->state.trailers_buf;
+  struct dynbuf *trailers_buf = &data->state.trailers_buf;
-  return trailers_buf->size_used - data->state.trailers_bytes_sent;
+  return Curl_dyn_len(trailers_buf) - data->state.trailers_bytes_sent;
 }
 #endif
 
@@ -186,11 +187,8 @@ CURLcode Curl_fillreadbuffer(struct connectdata *conn, size_t bytes,
     infof(data,
           "Moving trailers state machine from initialized to sending.\n");
     data->state.trailers_state = TRAILERS_SENDING;
-    data->state.trailers_buf = Curl_add_buffer_init();
+    Curl_dyn_init(&data->state.trailers_buf, DYN_TRAILERS);
-    if(!data->state.trailers_buf) {
+
-      failf(data, "Unable to allocate trailing headers buffer !");
-      return CURLE_OUT_OF_MEMORY;
-    }
     data->state.trailers_bytes_sent = 0;
     Curl_set_in_callback(data, true);
     trailers_ret_code = data->set.trailer_callback(&trailers,
@@ -206,7 +204,7 @@ CURLcode Curl_fillreadbuffer(struct connectdata *conn, size_t bytes,
       result = CURLE_ABORTED_BY_CALLBACK;
     }
     if(result) {
-      Curl_add_buffer_free(&data->state.trailers_buf);
+      Curl_dyn_free(&data->state.trailers_buf);
       curl_slist_free_all(trailers);
       return result;
     }
@@ -369,7 +367,7 @@ CURLcode Curl_fillreadbuffer(struct connectdata *conn, size_t bytes,
 #ifndef CURL_DISABLE_HTTP
     if(data->state.trailers_state == TRAILERS_SENDING &&
        !Curl_trailers_left(data)) {
-      Curl_add_buffer_free(&data->state.trailers_buf);
+      Curl_dyn_free(&data->state.trailers_buf);
       data->state.trailers_state = TRAILERS_DONE;
       data->set.trailer_data = NULL;
       data->set.trailer_callback = NULL;
@@ -770,8 +768,9 @@ static CURLcode readwrite_data(struct Curl_easy *data,
       /* pass data to the debug function before it gets "dechunked" */
       if(data->set.verbose) {
         if(k->badheader) {
-          Curl_debug(data, CURLINFO_DATA_IN, data->state.headerbuff,
+          Curl_debug(data, CURLINFO_DATA_IN,
-                     (size_t)k->hbuflen);
+                     Curl_dyn_ptr(&data->state.headerb),
+                     Curl_dyn_len(&data->state.headerb));
           if(k->badheader == HEADER_PARTHEADER)
             Curl_debug(data, CURLINFO_DATA_IN,
                        k->str, (size_t)nread);
@@ -822,9 +821,9 @@ static CURLcode readwrite_data(struct Curl_easy *data,
 
       /* Account for body content stored in the header buffer */
       if((k->badheader == HEADER_PARTHEADER) && !k->ignorebody) {
-        DEBUGF(infof(data, "Increasing bytecount by %zu from hbuflen\n",
+        size_t headlen = Curl_dyn_len(&data->state.headerb);
-                     k->hbuflen));
+        DEBUGF(infof(data, "Increasing bytecount by %zu\n", headlen));
-        k->bytecount += k->hbuflen;
+        k->bytecount += headlen;
       }
 
       if((-1 != k->maxdownload) &&
@@ -858,15 +857,16 @@ static CURLcode readwrite_data(struct Curl_easy *data,
         if(k->badheader && !k->ignorebody) {
           /* we parsed a piece of data wrongly assuming it was a header
              and now we output it as body instead */
+          size_t headlen = Curl_dyn_len(&data->state.headerb);
 
           /* Don't let excess data pollute body writes */
-          if(k->maxdownload == -1 || (curl_off_t)k->hbuflen <= k->maxdownload)
+          if(k->maxdownload == -1 || (curl_off_t)headlen <= k->maxdownload)
             result = Curl_client_write(conn, CLIENTWRITE_BODY,
-                                       data->state.headerbuff,
+                                       Curl_dyn_ptr(&data->state.headerb),
-                                       k->hbuflen);
+                                       headlen);
           else
             result = Curl_client_write(conn, CLIENTWRITE_BODY,
-                                       data->state.headerbuff,
+                                       Curl_dyn_ptr(&data->state.headerb),
                                        (size_t)k->maxdownload);
 
           if(result)

lib/url.c

@@ -122,6 +122,7 @@ bool curl_win32_idn_to_ascii(const char *in, char **out);
 #include "strdup.h"
 #include "setopt.h"
 #include "altsvc.h"
+#include "dynbuf.h"
 
 /* The last 3 #include files should be in this order */
 #include "curl_printf.h"
@@ -380,7 +381,7 @@ CURLcode Curl_close(struct Curl_easy **datap)
 
   up_free(data);
   Curl_safefree(data->state.buffer);
-  Curl_safefree(data->state.headerbuff);
+  Curl_dyn_free(&data->state.headerb);
   Curl_safefree(data->state.ulbuf);
   Curl_flush_cookies(data, TRUE);
 #ifdef USE_ALTSVC
@@ -408,8 +409,8 @@ CURLcode Curl_close(struct Curl_easy **datap)
   }
 
 #ifndef CURL_DISABLE_DOH
-  free(data->req.doh.probe[0].serverdoh.memory);
+  Curl_dyn_free(&data->req.doh.probe[0].serverdoh);
-  free(data->req.doh.probe[1].serverdoh.memory);
+  Curl_dyn_free(&data->req.doh.probe[1].serverdoh);
   curl_slist_free_all(data->req.doh.headers);
 #endif
 
@@ -609,15 +610,9 @@ CURLcode Curl_open(struct Curl_easy **curl)
     result = CURLE_OUT_OF_MEMORY;
   }
   else {
-    data->state.headerbuff = malloc(HEADERSIZE);
+    result = Curl_init_userdefined(data);
-    if(!data->state.headerbuff) {
+    if(!result) {
-      DEBUGF(fprintf(stderr, "Error: malloc of headerbuff failed\n"));
+      Curl_dyn_init(&data->state.headerb, CURL_MAX_HTTP_HEADER);
-      result = CURLE_OUT_OF_MEMORY;
-    }
-    else {
-      result = Curl_init_userdefined(data);
-
-      data->state.headersize = HEADERSIZE;
       Curl_convert_init(data);
       Curl_initinfo(data);
 
@@ -632,7 +627,7 @@ CURLcode Curl_open(struct Curl_easy **curl)
   if(result) {
     Curl_resolver_cleanup(data->state.resolver);
     free(data->state.buffer);
-    free(data->state.headerbuff);
+    Curl_dyn_free(&data->state.headerb);
     Curl_freeset(data);
     free(data);
     data = NULL;
@@ -3969,7 +3964,6 @@ CURLcode Curl_init_do(struct Curl_easy *data, struct connectdata *conn)
   k->bytecount = 0;
 
   k->buf = data->state.buffer;
-  k->hbufp = data->state.headerbuff;
   k->ignorebody = FALSE;
 
   Curl_speedinit(data);

lib/urldata.h

@@ -104,6 +104,7 @@
 #include "hostip.h"
 #include "hash.h"
 #include "splay.h"
+#include "dynbuf.h"
 
 /* return the count of bytes sent, or -1 on error */
 typedef ssize_t (Curl_send)(struct connectdata *conn, /* connection data */
@@ -556,18 +557,13 @@ enum doh_slots {
   DOH_PROBE_SLOTS
 };
 
-struct dohresponse {
-  unsigned char *memory;
-  size_t size;
-};
-
 /* one of these for each DoH request */
 struct dnsprobe {
   CURL *easy;
   int dnstype;
   unsigned char dohbuffer[512];
   size_t dohlen;
-  struct dohresponse serverdoh;
+  struct dynbuf serverdoh;
 };
 
 struct dohdata {
@@ -611,12 +607,7 @@ struct SingleRequest {
                                    written as body */
   int headerline;               /* counts header lines to better track the
                                    first one */
-  char *hbufp;                  /* points at *end* of header line */
-  size_t hbuflen;
   char *str;                    /* within buf */
-  char *str_start;              /* within buf */
-  char *end_ptr;                /* within buf */
-  char *p;                      /* within headerbuff */
   curl_off_t offset;            /* possible resume offset read from the
                                    Content-Range: header */
   int httpcode;                 /* error code from the 'HTTP/1.? XXX' or
@@ -1278,9 +1269,7 @@ struct Curl_http2_dep {
  * BODY).
  */
 struct tempbuf {
-  char *buf;  /* allocated buffer to keep data in when a write callback
+  struct dynbuf b;
-                 returns to make the connection paused */
-  size_t len; /* size of the 'tempwrite' allocated buffer */
   int type;   /* type of the 'tempwrite' buffer as a bitmask that is used with
                  Curl_client_write() */
 };
@@ -1340,9 +1329,7 @@ struct UrlState {
   struct curltime keeps_speed; /* for the progress meter really */
 
   struct connectdata *lastconnect; /* The last connection, NULL if undefined */
-
+  struct dynbuf headerb; /* buffer to store headers in */
-  char *headerbuff; /* allocated buffer to store headers in */
-  size_t headersize;   /* size of the allocation */
 
   char *buffer; /* download buffer */
   char *ulbuf; /* allocated upload buffer or NULL */
@@ -1422,8 +1409,8 @@ struct UrlState {
   struct urlpieces up;
 #ifndef CURL_DISABLE_HTTP
   size_t trailers_bytes_sent;
-  Curl_send_buffer *trailers_buf; /* a buffer containing the compiled trailing
+  struct dynbuf trailers_buf; /* a buffer containing the compiled trailing
-                                  headers */
+                                 headers */
 #endif
   trailers_state trailers_state; /* whether we are sending trailers
                                        and what stage are we at */

tests/data/test558

@@ -38,7 +38,6 @@ nothing
 <file name="log/memdump">
 MEM lib558.c: malloc()
 MEM lib558.c: free()
-MEM escape.c: malloc()
 MEM strdup.c: realloc()
 MEM strdup.c: realloc()
 MEM escape.c: free()

tests/libtest/Makefile.inc

@@ -59,7 +59,7 @@ noinst_PROGRAMS = chkhostname libauthretry libntlmconnect                \
  lib2033
 
 chkdecimalpoint_SOURCES = chkdecimalpoint.c ../../lib/mprintf.c \
- ../../lib/curl_ctype.c
+ ../../lib/curl_ctype.c  ../../lib/dynbuf.c ../../lib/strdup.c
 chkdecimalpoint_LDADD =
 chkdecimalpoint_CPPFLAGS = $(AM_CPPFLAGS) -DCURL_STATICLIB \
  -DCURLX_NO_MEMORY_CALLBACKS

tests/server/Makefile.inc

@@ -28,14 +28,18 @@ CURLX_SRCS = \
  ../../lib/nonblock.c \
  ../../lib/strtoofft.c \
  ../../lib/warnless.c \
- ../../lib/curl_ctype.c
+ ../../lib/curl_ctype.c \
+ ../../lib/dynbuf.c \
+ ../../lib/strdup.c
 
 CURLX_HDRS = \
  ../../lib/curlx.h \
  ../../lib/nonblock.h \
  ../../lib/strtoofft.h \
  ../../lib/warnless.h \
- ../../lib/curl_ctype.h
+ ../../lib/curl_ctype.h \
+ ../../lib/dynbuf.h \
+ ../../lib/strdup.h
 
 USEFUL = \
  getpart.c \

tests/unit/unit1650.c

@@ -22,6 +22,7 @@
 #include "curlcheck.h"
 
 #include "doh.h"
+#include "dynbuf.h"
 
 static CURLcode unit_setup(void)
 {
@@ -184,7 +185,7 @@ UNITTEST_START
     char *ptr;
     size_t len;
     int u;
-    memset(&d, 0, sizeof(d));
+    de_init(&d);
     rc = doh_decode((const unsigned char *)resp[i].packet, resp[i].size,
                     resp[i].type, &d);
     if(rc != resp[i].rc) {
@@ -222,7 +223,7 @@ UNITTEST_START
     }
     for(u = 0; u < d.numcname; u++) {
       size_t o;
-      msnprintf(ptr, len, "%s ", d.cname[u].alloc);
+      msnprintf(ptr, len, "%s ", Curl_dyn_ptr(&d.cname[u]));
       o = strlen(ptr);
       len -= o;
       ptr += o;