setopt: return CURLE_BAD_FUNCTION_ARGUMENT on bad argument

Fixed two return code mixups. CURLE_UNKNOWN_OPTION is saved for when the
option is, yeah, not known. Clarified this in the setopt man page too.

Closes #5993

docs/libcurl/curl_easy_setopt.3

@@ -645,6 +645,9 @@ Strings passed on to libcurl must be shorter than 8000000 bytes, otherwise
 \fIcurl_easy_setopt(3)\fP returns \fBCURLE_BAD_FUNCTION_ARGUMENT\fP (added in
 7.65.0).
 
+\fBCURLE_BAD_FUNCTION_ARGUMENT\fP is returned when the argument to an option
+is invalid, like perhaps out of range.a
+
 If you try to set an option that libcurl doesn't know about, perhaps because
 the library is too old to support it or the option was removed in a recent
 version, this function will return \fICURLE_UNKNOWN_OPTION\fP. If support for

lib/setopt.c

@@ -430,7 +430,7 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param)
       primary->version_max = version_max;
     }
 #else
-    result = CURLE_UNKNOWN_OPTION;
+    result = CURLE_NOT_BUILT_IN;
 #endif
     break;
 
@@ -1077,7 +1077,7 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param)
       break;
     default:
       /* reserve other values for future use */
-      result = CURLE_UNKNOWN_OPTION;
+      result = CURLE_BAD_FUNCTION_ARGUMENT;
       break;
     }
     break;
@@ -1230,21 +1230,13 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param)
      * An FTP/SFTP option that modifies an upload to create missing
      * directories on the server.
      */
-    switch(va_arg(param, long)) {
-    case 0:
-      data->set.ftp_create_missing_dirs = 0;
-      break;
-    case 1:
-      data->set.ftp_create_missing_dirs = 1;
-      break;
-    case 2:
-      data->set.ftp_create_missing_dirs = 2;
-      break;
-    default:
+    arg = va_arg(param, long);
     /* reserve other values for future use */
-      result = CURLE_UNKNOWN_OPTION;
-      break;
-    }
+    if((arg < CURLFTP_CREATE_DIR_NONE) ||
+       (arg > CURLFTP_CREATE_DIR_RETRY))
+      result = CURLE_BAD_FUNCTION_ARGUMENT;
+    else
+      data->set.ftp_create_missing_dirs = (int)arg;
     break;
   case CURLOPT_READDATA:
     /*