tests/server: add more SOCKS5 handshake error checking

- Add additional checking for missing and too-short SOCKS5 handshake
  messages.

Prior to this change the SOCKS5 test server did not check that all parts
of the handshake were received successfully. If those parts were missing
or too short then the server would access uninitialized memory.

This issue was discovered in CI job 'memory-sanitizer' test results.
Test 2055 was failing due to the SOCKS5 test server not running. It was
not running because either it crashed or memory sanitizer aborted it
during Test 728. Test 728 connects to the SOCKS5 test server on a
redirect but does not send any data on purpose. The test server was not
prepared for that.

Reported-by: Dan Fandrich

Fixes https://github.com/curl/curl/issues/12117
Closes https://github.com/curl/curl/pull/12118

tests/server/socksd.c

@@ -379,6 +379,10 @@ static curl_socket_t sockit(curl_socket_t fd)
   getconfig();
 
   rc = recv(fd, (char *)buffer, sizeof(buffer), 0);
+  if(rc <= 0) {
+    logmsg("SOCKS identifier message missing, recv returned %d", rc);
+    return CURL_SOCKET_BAD;
+  }
 
   logmsg("READ %d bytes", rc);
   loghex(buffer, rc);
@@ -386,6 +390,11 @@ static curl_socket_t sockit(curl_socket_t fd)
   if(buffer[SOCKS5_VERSION] == 4)
     return socks4(fd, buffer, rc);
 
+  if(rc < 3) {
+    logmsg("SOCKS5 identifier message too short: %d", rc);
+    return CURL_SOCKET_BAD;
+  }
+
   if(buffer[SOCKS5_VERSION] != config.version) {
     logmsg("VERSION byte not %d", config.version);
     return CURL_SOCKET_BAD;
@@ -417,6 +426,10 @@ static curl_socket_t sockit(curl_socket_t fd)
 
   /* expect the request or auth */
   rc = recv(fd, (char *)buffer, sizeof(buffer), 0);
+  if(rc <= 0) {
+    logmsg("SOCKS5 request or auth message missing, recv returned %d", rc);
+    return CURL_SOCKET_BAD;
+  }
 
   logmsg("READ %d bytes", rc);
   loghex(buffer, rc);
@@ -472,6 +485,10 @@ static curl_socket_t sockit(curl_socket_t fd)
 
     /* expect the request */
     rc = recv(fd, (char *)buffer, sizeof(buffer), 0);
+    if(rc <= 0) {
+      logmsg("SOCKS5 request message missing, recv returned %d", rc);
+      return CURL_SOCKET_BAD;
+    }
 
     logmsg("READ %d bytes", rc);
     loghex(buffer, rc);