Treat exit statuses that indicate failure to execute the test program

and exit statuses that indicate that the test program was terminated
by a signal as failures.

Also, report failures by writing a string to the output file which
will trigger a diff when it is compared with output files from other
runs of the test.

And, having reported any detected errors in a way that will be
visible to the test system, always exit "successfully". RunSafely.sh's
own exit status is not used to determine if a test has passed or failed.


git-svn-id: https://llvm.org/svn/llvm-project/test-suite/trunk@58154 91177308-0d34-0410-b5e6-96231b3b80d8

RunSafely.sh

@@ -11,11 +11,13 @@
 #           fourth argument specified, and outputs a <outfile>.time file which
 #           contains a timing of the program and the program's exit code.
 #          
-#           If the <exitok> parameter is 0 then this script always returns 0,
-#           regardless of the actual exit of the <program>.
-#           If the <exitok> parameter is non-zero then this script returns
-#           the exit code of the <program>. If there is an error in getting
-#           the <program>'s exit code, this script returns 99.
+#           The <exitok> parameter specifies how the program's exit status
+#           is interpreted.  If the <exitok> parameter is non-zero, any
+#           non-zero exit status from the program is considered to indicate
+#           a test failure.  If the <exitok> parameter is zero, only exit
+#           statuses that indicates that the program could not be executed
+#           normally or that the program was terminated as a signal are
+#           considered to indicate a test failure.
 #
 #           If optional parameters -r <remote host> -l <remote user> are
 #           specified, it execute the program remotely using rsh.
@@ -42,6 +44,10 @@ if [ $# -lt 4 ]; then
   exit 1
 fi
 
+# Save a copy of the original arguments in a string before we
+# clobber them with the shift command.
+ORIG_ARGS="$*"
+
 DIR=${0%%`basename $0`}
 
 RHOST=
@@ -145,18 +151,34 @@ rm -f $PWD/${OUTFILE}.remote.time
 fi
 
 exitval=`grep '^exit ' $OUTFILE.time | sed -e 's/^exit //'`
+fail=yes
 if [ -z "$exitval" ] ; then
   exitval=99
-  echo "TEST $PROGRAM FAILED:  CAN'T GET EXIT CODE!"
+  echo "TEST $PROGRAM FAILED: CAN'T GET EXIT CODE!"
+elif test "$exitval" -eq 126 ; then
+  echo "TEST $PROGRAM FAILED: command not executable (exit status 126)!"
+elif test "$exitval" -eq 127 ; then
+  echo "TEST $PROGRAM FAILED: command not found (exit status 127)!"
+elif test "$exitval" -eq 128 ; then
+  # Exit status 128 doesn't have a standard meaning, but it's unlikely
+  # to be expected program behavior.
+  echo "TEST $PROGRAM FAILED: exit status 128!"
+elif test "$exitval" -gt 128 ; then
+  echo "TEST $PROGRAM FAILED: process terminated by signal (exit status $exitval)!"
+elif [ "$EXITOK" -ne 0 -a "$exitval" -ne 0 ] ; then
+  echo "TEST $PROGRAM FAILED: EXIT != 0"
+else
+  fail=no
 fi
 echo "exit $exitval" >> $OUTFILE
 
-if [ "$EXITOK" -ne 0 ] ; then
-  if test "$exitval" -ne 0 ; then
-    echo "TEST $PROGRAM FAILED:  EXIT != 0"
-  fi
-else
-  exitval=0
+# If we detected a failure, print the name of the test executable to the
+# output file. This will cause it to compare as different with other runs
+# of the same test even if they fail in the same way, because they'll have
+# different command names.
+if [ "${fail}" != "no" ]; then
+  echo "RunSafely.sh detected a failure with these command-line arguments: " \
+       "$ORIG_ARGS" >> $OUTFILE
 fi
 
 if ls | egrep "^core" > /dev/null
@@ -176,4 +198,5 @@ then
     $GDB -q -batch --command=StackTrace.$$ --core=$corefile $PROGRAM < /dev/null
     rm -f StackTrace.$$ $corefile
 fi
-exit "$exitval"
+# Always return "successful" so that tests will continue to be run.
+exit 0