curl: two new command line options for etags

--etag-compare and --etag-save

Suggested-by: Paul Hoffman
Fixes #4277
Closes #4543

docs/cmdline-opts/Makefile.inc

@@ -38,6 +38,8 @@ DPAGES =					\
   dump-header.d					\
   egd-file.d					\
   engine.d					\
+  etag-save.d                   \
+  etag-compare.d                \
   expect100-timeout.d				\
   fail-early.d					\
   fail.d					\

docs/cmdline-opts/etag-compare.d

@@ -0,0 +1,17 @@
+Long: etag-compare
+Arg: <file>
+Help: Pass an ETag from a file as a custom header
+Protocols: HTTP
+---
+This option makes a conditional HTTP request for the specific
+ETag read from the given file by sending a custom If-None-Match
+header using the extracted ETag.
+
+For correct results, make sure that specified file contains only a single
+line with a desired ETag. An empty file is parsed as an empty ETag.
+
+Use the option --etag-save to first save the ETag from a response, and
+then use this option to compare using the saved ETag in a subsequent request.
+
+\fCOMPARISON\fP: There are 2 types of comparison or ETags, Weak and Strong.
+This option expects, and uses a strong comparison.

docs/cmdline-opts/etag-save.d

@@ -0,0 +1,15 @@
+Long: etag-save
+Arg: <file>
+Help: Parse ETag from a request and save it to a file
+Protocols: HTTP
+---
+This option saves an HTTP ETag to the specified file. Etag is
+usually part of headers returned by a request. When server sends an
+ETag, it must be enveloped by a double quote. This option extracts the
+ETag without the double quotes and saves it into the <file>.
+
+A server can send a week ETag which is prefixed by "W/". This identifier
+is not considered, and only relevant ETag between quotation marks is parsed.
+
+It an ETag wasn't send by the server or it cannot be parsed, and empty
+file is created.

src/tool_cb_hdr.c

@@ -59,6 +59,7 @@ size_t tool_header_cb(char *ptr, size_t size, size_t nmemb, void *userdata)
   struct HdrCbData *hdrcbdata = &per->hdrcbdata;
   struct OutStruct *outs = &per->outs;
   struct OutStruct *heads = &per->heads;
+  struct OutStruct *etag_save = &per->etag_save;
   const char *str = ptr;
   const size_t cb = size * nmemb;
   const char *end = (char *)ptr + cb;
@@ -95,6 +96,59 @@ size_t tool_header_cb(char *ptr, size_t size, size_t nmemb, void *userdata)
     (void)fflush(heads->stream);
   }
 
+  /*
+   * Write etag to file when --etag-save option is given.
+   * etag string that we want is enveloped in double quotes
+   */
+  if(etag_save->config->etag_save_file && etag_save->stream) {
+    /* match only header that start with etag (case insensitive) */
+    if(curl_strnequal(str, "etag:", 5)) {
+      char *etag_h = NULL;
+      char *first = NULL;
+      char *last = NULL;
+      size_t etag_length = 0;
+
+      etag_h = ptr;
+      /* point to first occurence of double quote */
+      first = memchr(etag_h, '\"', cb);
+
+      /*
+       * if server side messed with the etag header and doesn't include
+       * double quotes around the etag, kindly exit with a warning
+       */
+
+      if(!first) {
+        warnf(
+          etag_save->config->global,
+          "\nReceived header etag is missing double quote/s\n");
+        return 1;
+      }
+      else {
+        /* discard first double quote */
+        first++;
+      }
+
+      /* point to last occurence of double quote */
+      last = memchr(first, '\"', cb);
+
+      if(!last) {
+        warnf(
+          etag_save->config->global,
+          "\nReceived header etag is missing double quote/s\n");
+        return 1;
+      }
+
+      /* get length of desired etag */
+      etag_length = (size_t)last - (size_t)first;
+
+      fwrite(first, size, etag_length, etag_save->stream);
+      /* terminate with new line */
+      fputc('\n', etag_save->stream);
+    }
+
+    (void)fflush(etag_save->stream);
+  }
+
   /*
    * This callback sets the filename where output shall be written when
    * curl options --remote-name (-O) and --remote-header-name (-J) have

src/tool_cb_hdr.h

@@ -7,7 +7,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -43,6 +43,7 @@ struct HdrCbData {
   struct OperationConfig *config;
   struct OutStruct *outs;
   struct OutStruct *heads;
+  struct OutStruct *etag_save;
   bool honor_cd_filename;
 };
 

src/tool_cfgable.c

@@ -128,6 +128,8 @@ static void free_config_fields(struct OperationConfig *config)
   Curl_safefree(config->pubkey);
   Curl_safefree(config->hostpubmd5);
   Curl_safefree(config->engine);
+  Curl_safefree(config->etag_save_file);
+  Curl_safefree(config->etag_compare_file);
   Curl_safefree(config->request_target);
   Curl_safefree(config->customrequest);
   Curl_safefree(config->krblevel);

src/tool_cfgable.h

@@ -156,6 +156,8 @@ struct OperationConfig {
   char *pubkey;
   char *hostpubmd5;
   char *engine;
+  char *etag_save_file;
+  char *etag_compare_file;
   bool crlf;
   char *customrequest;
   char *krblevel;

src/tool_getparam.c

@@ -268,6 +268,8 @@ static const struct LongShort aliases[]= {
   {"E9", "proxy-tlsv1",              ARG_NONE},
   {"EA", "socks5-basic",             ARG_BOOL},
   {"EB", "socks5-gssapi",            ARG_BOOL},
+  {"EC", "etag-save",                ARG_FILENAME},
+  {"ED", "etag-compare",             ARG_FILENAME},
   {"f",  "fail",                     ARG_BOOL},
   {"fa", "fail-early",               ARG_BOOL},
   {"fb", "styled-output",            ARG_BOOL},
@@ -1697,6 +1699,14 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
           config->socks5_auth &= ~CURLAUTH_GSSAPI;
         break;
 
+      case 'C':
+        GetStr(&config->etag_save_file, nextarg);
+        break;
+
+      case 'D':
+        GetStr(&config->etag_compare_file, nextarg);
+        break;
+
       default: /* unknown flag */
         return PARAM_OPTION_UNKNOWN;
       }

src/tool_help.c

@@ -131,6 +131,10 @@ static const struct helptxt helptext[] = {
    "EGD socket path for random data"},
   {"    --engine <name>",
    "Crypto engine to use"},
+  {"    --etag-save <file>",
+   "Get an ETag from response header and save it to a FILE"},
+  {"    --etag-compare <file>",
+   "Get an ETag from a file and send a conditional request"},
   {"    --expect100-timeout <seconds>",
    "How long to wait for 100-continue"},
   {"-f, --fail",

src/tool_operate.c

@@ -644,6 +644,12 @@ static CURLcode post_per_transfer(struct GlobalConfig *global,
   if(per->heads.alloc_filename)
     Curl_safefree(per->heads.filename);
 
+  if(per->etag_save.fopened && per->etag_save.stream)
+    fclose(per->etag_save.stream);
+
+  if(per->etag_save.alloc_filename)
+    Curl_safefree(per->etag_save.filename);
+
   curl_easy_cleanup(per->curl);
   if(outs->alloc_filename)
     free(outs->filename);
@@ -834,6 +840,7 @@ static CURLcode single_transfer(struct GlobalConfig *global,
         struct OutStruct *outs;
         struct InStruct *input;
         struct OutStruct *heads;
+        struct OutStruct *etag_save;
         struct HdrCbData *hdrcbdata = NULL;
         CURL *curl = curl_easy_init();
         result = add_per_transfer(&per);
@@ -882,6 +889,99 @@ static CURLcode single_transfer(struct GlobalConfig *global,
           }
         }
 
+        /* disallowing simultaneous use of --etag-save and --etag-compare */
+        if(config->etag_save_file && config->etag_compare_file) {
+          warnf(
+            config->global,
+            "Cannot use --etag-save and --etag-compare at the same time\n");
+
+          result = CURLE_UNKNOWN_OPTION;
+          break;
+        }
+
+        /* --etag-save */
+        etag_save = &per->etag_save;
+        etag_save->stream = stdout;
+        etag_save->config = config;
+        if(config->etag_save_file) {
+          /* open file for output: */
+          if(strcmp(config->etag_save_file, "-")) {
+            FILE *newfile = fopen(config->etag_save_file, "wb");
+            if(!newfile) {
+              warnf(
+                config->global,
+                "Failed to open %s\n", config->etag_save_file);
+
+              result = CURLE_WRITE_ERROR;
+              break;
+            }
+            else {
+              etag_save->filename = config->etag_save_file;
+              etag_save->s_isreg = TRUE;
+              etag_save->fopened = TRUE;
+              etag_save->stream = newfile;
+            }
+          }
+          else {
+            /* always use binary mode for protocol header output */
+            set_binmode(etag_save->stream);
+          }
+        }
+
+        /* --etag-compare */
+        if(config->etag_compare_file) {
+          char *etag_from_file = NULL;
+          char *header = NULL;
+          size_t file_size = 0;
+
+          /* open file for reading: */
+          FILE *file = fopen(config->etag_compare_file, FOPEN_READTEXT);
+          if(!file) {
+            warnf(
+              config->global,
+              "Failed to open %s\n", config->etag_compare_file);
+
+            result = CURLE_READ_ERROR;
+            break;
+          }
+
+          /* get file size */
+          fseek(file, 0, SEEK_END);
+          file_size = ftell(file);
+
+          /*
+           * check if file is empty, if it's not load etag
+           * else continue with empty etag
+           */
+          if(file_size != 0) {
+            fseek(file, 0, SEEK_SET);
+            file2string(&etag_from_file, file);
+
+            header = aprintf("If-None-Match: \"%s\"", etag_from_file);
+          }
+          else {
+            header = aprintf("If-None-Match: \"\"");
+          }
+
+          if(!header) {
+            warnf(
+              config->global,
+              "Failed to allocate memory for custom etag header\n");
+
+            result = CURLE_OUT_OF_MEMORY;
+            break;
+          }
+
+          /* add Etag from file to list of custom headers */
+          add2list(&config->headers, header);
+
+          Curl_safefree(header);
+          Curl_safefree(etag_from_file);
+
+          if(file) {
+            fclose(file);
+          }
+        }
 
         hdrcbdata = &per->hdrcbdata;
 
@@ -1769,6 +1869,7 @@ static CURLcode single_transfer(struct GlobalConfig *global,
 
         hdrcbdata->outs = outs;
         hdrcbdata->heads = heads;
+        hdrcbdata->etag_save = etag_save;
         hdrcbdata->global = global;
         hdrcbdata->config = config;
 

src/tool_operate.h

@@ -48,6 +48,7 @@ struct per_transfer {
   struct ProgressData progressbar;
   struct OutStruct outs;
   struct OutStruct heads;
+  struct OutStruct etag_save;
   struct InStruct input;
   struct HdrCbData hdrcbdata;
   char errorbuffer[CURL_ERROR_SIZE];

tests/data/Makefile.inc

@@ -57,8 +57,7 @@ test298 test299 test300 test301 test302 test303 test304 test305 test306 \
 test307 test308 test309 test310 test311 test312 test313 test314 test315 \
 test316 test317 test318 test319 test320 test321 test322 test323 test324 \
 test325 test326 test327 test328 test329 test330 test331 test332 test333 \
-test334 test335 test336 test337 test338 \
-test340 \
+test334 test335 test336 test337 test338 test339 test340 test341 test342 \
 \
 test350 test351 test352 test353 test354 test355 test356 \
 test393 test394 test395 \

tests/data/test339

@@ -0,0 +1,63 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+</keywords>
+</info>
+#
+# Server-side
+<reply>
+<data nocheck="yes">
+HTTP/1.1 200 funky chunky!
+Server: fakeit/0.9 fakeitbad/1.0
+Transfer-Encoding: chunked
+Trailer: chunky-trailer
+Connection: mooo
+ETag: "asdf"
+
+40
+aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+30
+bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
+21;heresatest=moooo
+cccccccccccccccccccccccccccccccc
+
+0
+chunky-trailer: header data
+
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+Check if --etag-save saved correct etag to a file
+</name>
+<command>
+http://%HOSTIP:%HTTPPORT/339 --etag-save log/etag339
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<strip>
+^User-Agent:.*
+</strip>
+<protocol>
+GET /339 HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+Accept: */*
+
+</protocol>
+<file name="log/etag339">
+asdf
+</file>
+</verify>
+
+</testcase>

tests/data/test341

@@ -0,0 +1,57 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+</keywords>
+</info>
+#
+# Server-side
+<reply>
+<data nocheck="yes">
+HTTP/1.1 200 funky chunky!
+Server: fakeit/0.9 fakeitbad/1.0
+Transfer-Encoding: chunked
+Trailer: chunky-trailer
+Connection: mooo
+ETag: "asdf"
+
+40
+aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+30
+bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
+21;heresatest=moooo
+cccccccccccccccccccccccccccccccc
+
+0
+chunky-trailer: header data
+
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+Try to open a non existing file with --etag-compare should return an error
+</name>
+<command>
+http://%HOSTIP:%HTTPPORT/341 --etag-compare log/etag341
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<strip>
+^User-Agent:.*
+</strip>
+<errorcode>
+26
+</errorcode>
+</verify>
+
+</testcase>

tests/data/test342

@@ -0,0 +1,59 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data>
+HTTP/1.1 200 OK
+Date: Thu, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 6
+Connection: close
+Content-Type: text/html
+Funny-head: yesyes
+
+-foo-
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+Check if --etag-compare set correct etag in header
+</name>
+<file name="log/etag342">
+21025-dc7-39462498
+</file>
+<command>
+http://%HOSTIP:%HTTPPORT/342 --etag-compare log/etag342
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<strip>
+^User-Agent:.*
+</strip>
+<protocol>
+GET /342 HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+Accept: */*
+If-None-Match: "21025-dc7-39462498"
+
+</protocol>
+</verify>
+</testcase>