diff --git a/lib/multi.c b/lib/multi.c
index 85097816d..518ceb51f 100644
--- a/lib/multi.c
+++ b/lib/multi.c
@@ -1052,11 +1052,17 @@ CURLMcode curl_multi_fdset(struct Curl_multi *multi,
     for(i = 0; i< MAX_SOCKSPEREASYHANDLE; i++) {
       curl_socket_t s = CURL_SOCKET_BAD;
 
-      if((bitmap & GETSOCK_READSOCK(i)) && VALID_SOCK((sockbunch[i]))) {
+      if((bitmap & GETSOCK_READSOCK(i)) && VALID_SOCK(sockbunch[i])) {
+        if(!FDSET_SOCK(sockbunch[i]))
+          /* pretend it doesn't exist */
+          continue;
         FD_SET(sockbunch[i], read_fd_set);
         s = sockbunch[i];
       }
-      if((bitmap & GETSOCK_WRITESOCK(i)) && VALID_SOCK((sockbunch[i]))) {
+      if((bitmap & GETSOCK_WRITESOCK(i)) && VALID_SOCK(sockbunch[i])) {
+        if(!FDSET_SOCK(sockbunch[i]))
+          /* pretend it doesn't exist */
+          continue;
         FD_SET(sockbunch[i], write_fd_set);
         s = sockbunch[i];
       }
diff --git a/lib/select.h b/lib/select.h
index 19da1e774..59a571dbb 100644
--- a/lib/select.h
+++ b/lib/select.h
@@ -97,8 +97,10 @@ int tpf_select_libcurl(int maxfds, fd_set* reads, fd_set* writes,
 #if defined(TPF)
 #define VALID_SOCK(x) 1
 #define VERIFY_SOCK(x) Curl_nop_stmt
+#define FDSET_SOCK(x) 1
 #elif defined(USE_WINSOCK)
 #define VALID_SOCK(s) ((s) < INVALID_SOCKET)
+#define FDSET_SOCK(x) 1
 #define VERIFY_SOCK(x) do { \
   if(!VALID_SOCK(x)) { \
     SET_SOCKERRNO(WSAEINVAL); \
@@ -106,17 +108,17 @@ int tpf_select_libcurl(int maxfds, fd_set* reads, fd_set* writes,
   } \
 } while(0)
 #else
-#ifdef HAVE_POLL_FINE
-#define VALID_SOCK(s) ((s) >= 0)  /* FD_SETSIZE is irrelevant for poll */
-#else
-#define VALID_SOCK(s) (((s) >= 0) && ((s) < FD_SETSIZE))
-#endif
-#define VERIFY_SOCK(x) do { \
-  if(!VALID_SOCK(x)) { \
-    SET_SOCKERRNO(EINVAL); \
-    return -1; \
-  } \
-} while(0)
+#define VALID_SOCK(s) ((s) >= 0)
+
+/* If the socket is small enough to get set or read from an fdset */
+#define FDSET_SOCK(s) ((s) < FD_SETSIZE)
+
+#define VERIFY_SOCK(x) do {                     \
+    if(!VALID_SOCK(x) || !FDSET_SOCK(x)) {      \
+      SET_SOCKERRNO(EINVAL);                    \
+      return -1;                                \
+    }                                           \
+  } while(0)
 #endif
 
 #endif /* HEADER_CURL_SELECT_H */