Compiler warning fix

tests/libtest/lib518.c

@@ -251,7 +251,7 @@ static int rlimit(int keep_open)
 
   /* verify that we won't overflow size_t in malloc() */
 
-  if (num_open.rlim_max > ((size_t)-1) / sizeof(*fd)) {
+  if ((size_t)(num_open.rlim_max) > ((size_t)-1) / sizeof(*fd)) {
     sprintf(strbuff1, fmt, num_open.rlim_max);
     sprintf(strbuff, "unable to allocate an array for %s "
             "file descriptors, would overflow size_t", strbuff1);

tests/libtest/lib537.c

@@ -233,7 +233,7 @@ static int rlimit(int keep_open)
 
   /* verify that we won't overflow size_t in malloc() */
 
-  if (num_open.rlim_max > ((size_t)-1) / sizeof(*fd)) {
+  if ((size_t)(num_open.rlim_max) > ((size_t)-1) / sizeof(*fd)) {
     sprintf(strbuff1, fmt, num_open.rlim_max);
     sprintf(strbuff, "unable to allocate an array for %s "
             "file descriptors, would overflow size_t", strbuff1);