Improve Sisyphus error handling, bug 415911, not part of the build

testing/sisyphus/bin/build.sh

@@ -50,14 +50,14 @@ case $product in
         if ! make -f client.mk build 2>&1; then
 
             if [[ -z "$TEST_FORCE_CLOBBER_ON_ERROR" ]]; then
-                error "error during build"
+                error "error during build" $LINENO
             else
-                echo "error occured during build. attempting a clobber build"
+                echo "error occured during build. attempting a clobber build" $LINENO
                 if ! make -f client.mk distclean 2>&1; then
-                    error "error during forced clobber"
+                    error "error during forced clobber" $LINENO
                 fi
                 if ! make -f client.mk build 2>&1; then
-                    error "error during forced build"
+                    error "error during forced build" $LINENO
                 fi
             fi
         fi
@@ -90,7 +90,7 @@ case $product in
             #
             executable=`get_executable $product $branch $executablepath`
             if [[ -z "$executable" ]]; then
-                error "get_executable $product $branch $executablepath returned empty path"
+                error "get_executable $product $branch $executablepath returned empty path" $LINENO
             fi
 
 
@@ -120,11 +120,11 @@ case $product in
     fi
 
     if ! make -f Makefile.ref ${JSBUILDOPT} clean 2>&1; then
-        error "during js/src clean"
+        error "during js/src clean" $LINENO
     fi 
 
     if ! make -f Makefile.ref ${JSBUILDOPT} 2>&1; then
-        error "during js/src build"
+        error "during js/src build" $LINENO
     fi
     ;;
 esac

testing/sisyphus/bin/builder.sh

@@ -43,8 +43,6 @@ source ${TEST_BIN}/library.sh
 
 TEST_LOG=/dev/null
 
-trap 'echo -e "\n*** ERROR ***\n\b" && tail $TEST_LOG' ERR
-
 #
 # options processing
 #

testing/sisyphus/bin/check-spider.sh

@@ -41,6 +41,8 @@ TEST_DIR=${TEST_DIR:-/work/mozilla/mozilla.com/test.mozilla.com/www}
 TEST_BIN=${TEST_BIN:-$TEST_DIR/bin}
 source ${TEST_BIN}/library.sh
 
+TEST_STARTUP_TRIES=${TEST_STARTUP_TRIES:-3}
+
 #
 # options processing
 #
@@ -68,7 +70,7 @@ note that the environment variables should have the same names as in the
 Checks if the Spider extension is installed either in the named profile
 or as a global extension, by attempting up to 3 times to launch the Spider.
 
-If Spider fails to launch, the script returns exit code 66.
+If Spider fails to launch, the script returns exit code 2.
 
 EOF
     exit 1
@@ -101,26 +103,26 @@ if [[ -z "$product" || -z "$branch" || -z "$executablepath" || -z "$profilename"
 fi
 
 if [[ "$product" != "firefox" && "$product" != "thunderbird" ]]; then
-    error "product \"$product\" must be one of firefox or thunderbird"
+    error "product \"$product\" must be one of firefox or thunderbird" $LINENO
 fi
 
 if [[ "$branch" != "1.8.0" && "$branch" != "1.8.1" && "$branch" != "1.9.0" ]]; 
     then
-    error "branch \"$branch\" must be one of 1.8.0, 1.8.1, 1.9.0"
+    error "branch \"$branch\" must be one of 1.8.0, 1.8.1, 1.9.0" $LINENO
 fi
 
 executable=`get_executable $product $branch $executablepath`
 
 if [[ -z "$executable" ]]; then
-    error "get_executable $product $branch $executablepath returned empty path"
+    error "get_executable $product $branch $executablepath returned empty path" $LINENO
 fi
 
 if [[ ! -x "$executable" ]]; then 
-    error "executable \"$executable\" is not executable"
+    error "executable \"$executable\" is not executable" $LINENO
 fi
 
 if echo "$profilename" | egrep -qiv '[a-z0-9_]'; then
-    error "profile name must consist of letters, digits or _"
+    error "profile name must consist of letters, digits or _" $LINENO
 fi
 
 echo # attempt to force Spider to load
@@ -132,9 +134,8 @@ while ! $TEST_BIN/timed_run.py ${TEST_STARTUP_TIMEOUT} "Start Spider: try $tries
     -uri "http://${TEST_HTTP}/bin/start-spider.html" \
     -hook "http://${TEST_HTTP}/bin/userhook-checkspider.js"; do
   let tries=tries+1
-  if [ "$tries" -gt 3 ]; then
+  if [ "$tries" -gt $TEST_STARTUP_TRIES  ]; then
-      echo "Failed to start spider. Exiting..."
+      error "Failed to start spider. Exiting..." $LINENO
-      exit 66
   fi
 done
 

testing/sisyphus/bin/checkout.sh

@@ -44,7 +44,7 @@ source ${TEST_BIN}/library.sh
 source ${TEST_BIN}/set-build-env.sh $@
 
 if [[ -z "$TREE" ]]; then
-    error "source tree not specified!"
+    error "source tree not specified!" $LINENO
 fi
 
 cd $TREE
@@ -56,14 +56,14 @@ case $product in
             -e "mozilla/$project/config/mozconfig" ) ]]; then
             if ! eval cvs -z3 -q co $BRANCH_CO_FLAGS \
                 mozilla/client.mk mozilla/$project/config/mozconfig; then
-                error "during checkout of mozconfig"
+                error "during checkout of mozconfig" $LINENO
             fi
         fi
 
         cd mozilla
 
         if ! make -f client.mk checkout 2>&1; then
-            error "during checkout of tree"
+            error "during checkout of tree" $LINENO
         fi
         ;;
 
@@ -77,15 +77,15 @@ case $product in
     cd mozilla/js/src
 
     if ! eval cvs -z3 -q update $BRANCH_CO_FLAGS $DATE_CO_FLAGS -d -P 2>&1; then
-        error "during checkout of js/src"
+        error "during checkout of js/src" $LINENO
     fi
 
     if ! cvs -z3 -q update -d -P -A editline config  2>&1; then
-        error "during checkout of js/src"
+        error "during checkout of js/src" $LINENO
     fi
     # end for js shell
     ;;
     *)
-    error "unknown product $product"
+    error "unknown product $product" $LINENO
     ;;
 esac

testing/sisyphus/bin/clean.sh

@@ -48,7 +48,7 @@ case $product in
         cd $TREE/mozilla
 
         if ! make -f client.mk clean 2>&1; then
-            error "during client.mk clean"
+            error "during client.mk clean" $LINENO
         fi
         ;;
 
@@ -56,7 +56,7 @@ case $product in
         cd $TREE/mozilla/js/src
 
         if ! make -f Makefile.ref clean 2>&1; then
-            error "during SpiderMonkey clean"
+            error "during SpiderMonkey clean" $LINENO
         fi
         ;;
 esac

testing/sisyphus/bin/create-directory.sh

@@ -91,26 +91,26 @@ if [[ -z $directory ]]
 fi
 
 if [[ `whoami` == "root" ]]; then
-    error "can not be run as root"
+    error "can not be run as root" $LINENO
 fi
 
 # get the cannonical name directory name
 mkdir -p "$directory"
 if ! pushd "$directory" > /dev/null ; then 
-    error "$directory is not accessible"
+    error "$directory is not accessible" $LINENO
 fi
 directory=`pwd`
 popd > /dev/null
 
 if [[ "$directory" == "/" ]]; then
-    error "directory $directory can not be root"
+    error "directory $directory can not be root" $LINENO
 fi
 
 parent=`dirname "$directory"`
 grandparent=`dirname "$parent"`
 
 if [[ "$parent" != "/tmp" && ( "$parent" == "/" || "$grandparent" == "/" ) ]]; then
-    error "directory $directory can not be a subdirectory of $parent"
+    error "directory $directory can not be a subdirectory of $parent" $LINENO
 fi
 
 

testing/sisyphus/bin/create-profile.sh

@@ -101,34 +101,34 @@ if [[ -z "$product" || -z "$branch" || -z "$executablepath" || \
 fi
 
 if [[ "$product" != "firefox" && "$product" != "thunderbird" ]]; then
-    error "product \"$product\" must be one of firefox or thunderbird"
+    error "product \"$product\" must be one of firefox or thunderbird" $LINENO
 fi
 
 if [[ "$branch" != "1.8.0" && "$branch" != "1.8.1" && "$branch" != "1.9.0" ]]; 
     then
-    error "branch \"$branch\" must be one of 1.8.0, 1.8.1, 1.9.0"
+    error "branch \"$branch\" must be one of 1.8.0, 1.8.1, 1.9.0" $LINENO
 fi
 
 executable=`get_executable $product $branch $executablepath`
 
 if [[ -z "$executable" ]]; then
-    error "get_executable $product $branch $executablepath returned empty path"
+    error "get_executable $product $branch $executablepath returned empty path" $LINENO
 fi
 
 if [[ ! -x "$executable" ]]; then 
-    error "executable \"$executable\" is not executable"
+    error "executable \"$executable\" is not executable" $LINENO
 fi
 
 $TEST_BIN/create-directory.sh -d "$directory" -n 
 
 if echo "$profilename" | egrep -qiv '[a-z0-9_]'; then
-    error "profile name \"$profilename\" must consist of letters, digits or _"
+    error "profile name \"$profilename\" must consist of letters, digits or _" $LINENO
 fi
 
 if [ $OSID == "win32" ]; then
     directoryospath=`cygpath -a -w $directory`
     if [[ -z "$directoryospath" ]]; then
-	    error "unable to convert unix path to windows path"
+	    error "unable to convert unix path to windows path" $LINENO
     fi
 else
     directoryospath="$directory"
@@ -137,12 +137,12 @@ fi
 echo "creating profile $profilename in directory $directory"
 
 if ! $TEST_BIN/timed_run.py ${TEST_STARTUP_TIMEOUT} "-" $executable -CreateProfile "$profilename $directoryospath"; then
-	error "creating profile $directory"
+	error "creating profile $directory" $LINENO
 fi
 
 if [[ -n $profiletemplate ]]; then
 	if [[ ! -d $profiletemplate ]]; then
-	    error "profile template directory $profiletemplate does not exist"
+	    error "profile template directory $profiletemplate does not exist" $LINENO
 	fi
 	echo "copying template profile $profiletemplate to $directory"
 	cp -R $profiletemplate/* $directory

testing/sisyphus/bin/edit-talkback.sh

@@ -97,11 +97,11 @@ fi
 executable=`get_executable $product $branch $executablepath`
 
 if [[ -z "$executable" ]]; then
-    error "get_executable $product $branch $executablepath returned empty path"
+    error "get_executable $product $branch $executablepath returned empty path" $LINENO
 fi
 
 if [[ ! -x "$executable" ]]; then 
-    error "executable \"$executable\" is not executable"
+    error "executable \"$executable\" is not executable" $LINENO
 fi
 
 executablepath=`dirname $executable`
@@ -159,12 +159,12 @@ if [[ $talkback -eq 1 ]]; then
             IFS=:
             ;;
         *)
-            error "unknown os $OSID"
+            error "unknown os $OSID" $LINENO
             ;;
     esac
 
     if [[ -z "$talkbackdir" ]]; then
-        error "empty talkback directory"
+        error "empty talkback directory" $LINENO
     fi
 
     mkdir -p "$talkbackdir"
@@ -197,6 +197,6 @@ if [[ $talkback -eq 1 ]]; then
             sed -i.bak "s@URLEditControl .*@URLEditControl = \"mozqa:$talkbackid\"@" Talkback.ini
             ;;
         *)
-            error "unknown os=$OSID"
+            error "unknown os=$OSID" $LINENO
     esac
 fi

testing/sisyphus/bin/install-build.sh

@@ -111,7 +111,7 @@ if [[ $OSID == "win32" ]]; then
     elif echo  $filetype | grep -iq 'zip archive'; then
 	    unzip -o -d "$executablepath" "$filename"
     else
-	    error "$unknown file type $filetype"
+	    error "$unknown file type $filetype" $LINENO
     fi
 
 else
@@ -123,7 +123,7 @@ else
         elif echo $filetype | grep -iq 'gzip'; then
             tar -zxvf $filename -C "$executablepath" 
         else
-            error "unknown file type $filetype"
+            error "unknown file type $filetype" $LINENO
         fi
         ;; 
 
@@ -138,7 +138,7 @@ else
         echo "disk=$disk"
         echo "volume=$volume"
         if [[ -z "$disk" || -z "$volume" ]]; then
-            error "mounting disk image: $result"
+            error "mounting disk image: $result" $LINENO
         fi
 
         for app in $volume/*.app; do
@@ -155,7 +155,7 @@ else
     #
     executable=`get_executable $product $branch $executablepath`
     if [[ -z "$executable" ]]; then
-        error "get_executable $product $branch $executablepath returned empty directory"
+        error "get_executable $product $branch $executablepath returned empty directory" $LINENO
     fi
     executabledir=`dirname $executable`
 

testing/sisyphus/bin/install-extensions.sh

@@ -100,30 +100,30 @@ if [[ -z "$product" || -z "$branch" || \
 fi
 
 if [[ "$product" != "firefox" && "$product" != "thunderbird" ]]; then
-    error "product \"$product\" must be one of firefox or thunderbird"
+    error "product \"$product\" must be one of firefox or thunderbird" $LINENO
 fi
 
 if [[ "$branch" != "1.8.0" && "$branch" != "1.8.1" && "$branch" != "1.9.0" ]]; 
     then
-    error "branch \"$branch\" must be one of 1.8.0, 1.8.1, 1.9.0"
+    error "branch \"$branch\" must be one of 1.8.0, 1.8.1, 1.9.0" $LINENO
 fi
 
 executable=`get_executable $product $branch $executablepath`
 
 if [[ -z "$executable" ]]; then
-    error "get_executable $product $branch $executablepath returned empty path"
+    error "get_executable $product $branch $executablepath returned empty path" $LINENO
 fi
 
 if [[ ! -x "$executable" ]]; then 
-    error "executable \"$executable\" is not executable"
+    error "executable \"$executable\" is not executable" $LINENO
 fi
 
 if echo $profilename | egrep -qiv '[a-z0-9_]'; then
-    error "profile name must consist of letters, digits or _"
+    error "profile name must consist of letters, digits or _" $LINENO
 fi
 
-for extension in $extensions/all/*; do 
+for extension in $extensions/all/*.xpi; do 
-    if [[ $extension == "$extensions/all/*" ]]; then
+    if [[ $extension == "$extensions/all/*.xpi" ]]; then
 	    break
     fi
     if [[ "$OSID" == "win32" ]]; then
@@ -150,7 +150,7 @@ for extension in $extensions/$OSID/*; do
 
     echo installing $extension
     if ! $TEST_BIN/timed_run.py ${TEST_STARTUP_TIMEOUT} "-" $executable -P $profilename -install-global-extension "$extensionos"; then
-        error "Failed to install $extensionos"
+        error "Failed to install $extensionos" $LINENO
     fi
 
 done
@@ -164,6 +164,6 @@ fi
 
 if ! $TEST_BIN/timed_run.py ${TEST_STARTUP_TIMEOUT} "install extensions - second restart" \
     $executable -P $profilename "http://${TEST_HTTP}/bin/install-extensions-2.html"; then
-    error "Fatal 2nd failure to load the install-extensions page"
+    error "Fatal 2nd failure to load the install-extensions page" $LINENO
 fi
 

testing/sisyphus/bin/install-plugins.sh

@@ -95,17 +95,17 @@ if [[ -z "$product" || -z "$branch" || \
 fi
 
 if [[ "$product" != "firefox" && "$product" != "thunderbird" ]]; then
-    error "product \"$product\" must be one of firefox or thunderbird"
+    error "product \"$product\" must be one of firefox or thunderbird" $LINENO
 fi
 
 executable=`get_executable $product $branch $executablepath`
 
 if [[ -z "$executable" ]]; then
-    error "get_executable $product $branch $executablepath returned empty path"
+    error "get_executable $product $branch $executablepath returned empty path" $LINENO
 fi
 
 if [[ ! -x "$executable" ]]; then 
-    error "executable \"$executable\" is not executable"
+    error "executable \"$executable\" is not executable" $LINENO
 fi
 
 executablepath=`dirname $executable`

testing/sisyphus/bin/library.sh

@@ -43,23 +43,73 @@
 # TEST_DIR=${TEST_DIR:-/work/mozilla/mozilla.com/test.mozilla.com/www}
 # TEST_BIN=${TEST_BIN:-$TEST_DIR/bin}
 # source ${TEST_BIN}/library.sh
-#
-#trap "echo error $0 `caller 1`; exit" ERR
-
-# skip remainder of script if it has already 
-# included
 
 if [[ -n "$DEBUG" ]]; then
     echo "calling $0 $@" 1>&2
 fi
 
+# export variables
+set -a 
+
+# in the event of an untrapped script error tail the test log, 
+# if it exists, to stderr then echo a FATAL ERROR message to the 
+# test log and stderr.
+
+function _err()
+{
+    local rc=$?
+    debug "_err: $0"
+
+    if [[ "$rc" -gt 0 ]]; then
+        if [[ -n "$TEST_LOG" ]]; then
+            echo -e "\nFATAL ERROR in $0 exit code $rc\n" >> $TEST_LOG
+        else
+            echo -e "\nFATAL ERROR in $0 exit code $rc\n" 1>&2
+        fi
+    fi
+    exit $rc
+}
+
+trap "_err" ERR
+
+function _exit()
+{
+    local rc=$?
+    local currscript=`get_scriptname $0`
+
+    debug "_exit: $0"
+
+    if [[ "$rc" -gt 0 && -n "$TEST_LOG" && "$SCRIPT" == "$currscript" ]]; then
+        # only tail the log once at the top level script
+        tail $TEST_LOG 1>&2
+    fi
+}
+
+trap "_exit" EXIT
+
+    # error message
+    # output error message end exit 2
+
+    error()
+    {
+        local message=$1
+        local lineno=$2
+
+        debug "error: $0:$LINENO"
+    
+        echo -e "FATAL ERROR in script $0:$lineno $message\n" 1>&2
+        if [[ "$0" == "-bash" || "$0" == "bash" ]]; then
+            return 0
+        fi
+        exit 2
+    } 
+
+
 if [[ -z "$LIBRARYSH" ]]; then
+    # skip remainder of script if it has already included
 
     LIBRARYSH=1
 
-    # export variables
-    set -a 
-
     # set time format for pipeline timing reports
     TIMEFORMAT="Elapsed time %0R seconds, User %0U seconds, System %0S seconds, CPU %P%%"
 
@@ -86,22 +136,10 @@ if [[ -z "$LIBRARYSH" ]]; then
 
     console()
     {
-        echo "$@" 1>&2
+        echo -e "$@" 1>&2
     }
 
 
-    # error message
-    # output error message end exit 2
-
-    error()
-    {
-        echo "error in script $SCRIPT: $1"
-        if [[ "$0" == "-bash" || "$0" == "bash" ]]; then
-            return 0
-        fi
-        exit 2
-    } 
-
     # dumpenvironment
     #
     # output environment to stdout
@@ -167,11 +205,18 @@ if [[ -z "$LIBRARYSH" ]]; then
         fi
     }
 
-    if [[ "$0" == "-bash" || "$0" == "bash" ]]; then
+    function get_scriptname()
-        SCRIPT="library.sh"
+    {
-    else
+        local script
-        SCRIPT=`basename $0`
+        if [[ "$0" == "-bash" || "$0" == "bash" ]]; then
-    fi
+            script="library.sh"
+        else
+            script=`basename $0`
+        fi
+        echo $script
+    }
+
+    SCRIPT=`get_scriptname $0`
 
     TEST_DIR=${TEST_DIR:-/work/mozilla/mozilla.com/test.mozilla.com/www}
     TEST_BIN=${TEST_BIN:-$TEST_DIR/bin}

testing/sisyphus/bin/smoke-build.sh

@@ -63,7 +63,7 @@ for filepath in $@; do
 
     if ! install-build.sh  -p "$product" -b "$branch" -x "/tmp/$product-$branch" \
 	-f "$filepath"; then
-	error "installing build $product $branch into /tmp/$product-$branch"
+	error "installing build $product $branch into /tmp/$product-$branch" $LINENO
     fi
 
     if [[ "$product" == "thunderbird" ]]; then
@@ -77,20 +77,20 @@ for filepath in $@; do
 	-D "/tmp/$product-$branch-profile" -N "$product-$branch-profile" \
 	-U ${TEST_DIR}/prefs/test-user.js \
 	$template; then
-	error "creating profile $product-$branch-profile at /tmp/$product-$branch"
+	error "creating profile $product-$branch-profile at /tmp/$product-$branch" $LINENO
     fi
 
     if ! install-extensions.sh -p "$product" -b "$branch" \
 	-x "/tmp/$product-$branch" \
 	-N "$product-$branch-profile" \
 	-E ${TEST_DIR}/xpi; then
-        error "installing extensions from ${TEST_DIR}/xpi"
+        error "installing extensions from ${TEST_DIR}/xpi" $LINENO
     fi
 
     if ! check-spider.sh -p "$product" -b "$branch" \
 	-x "/tmp/$product-$branch" \
 	-N "$product-$branch-profile"; then
-        error "check-spider.sh failed."
+        error "check-spider.sh failed." $LINENO
     fi
 
     uninstall-build.sh  -p "$product" -b "$branch" -x "/tmp/$product-$branch"

testing/sisyphus/bin/test-setup.sh

@@ -255,18 +255,10 @@ fi
 
 if [[ -n "$extensiondir" ]]; then
 
-    if ! install-extensions.sh -p $product -b $branch \
+    install-extensions.sh -p $product -b $branch \
-        -x $executablepath -N $profilename -E $extensiondir; then
+        -x $executablepath -N $profilename -E $extensiondir
-        exit 2
-    fi
 
-    if ! check-spider.sh -p $product -b $branch \
+    check-spider.sh -p $product -b $branch \
-        -x $executablepath -N $profilename; then
+        -x $executablepath -N $profilename
-        exit 2
-    fi
 
 fi
-
-
-
-

testing/sisyphus/bin/tester.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e
 # -*- Mode: Shell-script; tab-width: 4; indent-tabs-mode: nil; -*-
 # ***** BEGIN LICENSE BLOCK *****
 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
@@ -43,8 +43,6 @@ source ${TEST_BIN}/library.sh
 
 TEST_LOG=/dev/null
 
-#trap 'echo -e "\n*** ERROR ***\n\b" && tail $TEST_LOG' ERR
-
 #
 # options processing
 #
@@ -53,7 +51,7 @@ function usage()
 {
     cat<<EOF
 usage: 
-$SCRIPT -t testscript datalist1 [datalist2 [datalist3 [datalist4]]]
+$SCRIPT -t testscript [-v ] datalist1 [datalist2 [datalist3 [datalist4]]]
 
 variable            description
 ===============     ===========================================================
@@ -109,14 +107,10 @@ for data in $datalist; do
     echo "log: $TEST_LOG "
 
     if [[ "$verbose" == "1" ]]; then
-        if ! test-setup.sh -d $TEST_DIR/data/$data.data 2>&1 | tee -a $TEST_LOG; then
+        test-setup.sh -d $TEST_DIR/data/$data.data 2>&1 | tee -a $TEST_LOG
-            error "test-setup.sh failed"
-        fi
         $testscript $testargs -d $TEST_DIR/data/$data.data 2>&1 | tee -a $TEST_LOG
     else
-        if ! test-setup.sh -d $TEST_DIR/data/$data.data >> $TEST_LOG 2>&1; then
+        test-setup.sh -d $TEST_DIR/data/$data.data >> $TEST_LOG 2>&1
-            error "test-setup.sh failed"
-        fi
         $testscript $testargs -d $TEST_DIR/data/$data.data >> $TEST_LOG 2>&1
     fi
 

testing/sisyphus/bin/uninstall-build.sh

@@ -142,7 +142,7 @@ if [[ $OSID == "win32" ]]; then
                 fi
             fi
         else
-            error "Unknown branch $branch"
+            error "Unknown branch $branch" $LINENO
         fi
         # the NSIS uninstaller will copy itself, then fork to the new
         # copy so that it can delete itself. This causes a race condition