JavaScript Test Library - tweak reporting to improve automated test comparisons, no bug

js/tests/Maketests

@@ -38,6 +38,7 @@
 include config.mk
 
 TEST_PROFILE?=TEST
+TEST_HTTP ?= test.mozilla.com
 
 
 ifndef TEST_JSBROWSER_TIMEOUT
@@ -45,12 +46,16 @@ TEST_JSBROWSER_TIMEOUT=900
 endif
 
 ifndef TEST_JSEACH_TIMEOUT
-TEST_JSEACH_TIMEOUT=120
+TEST_JSEACH_TIMEOUT=900
+endif
+
+ifndef TEST_JSSHELL_TIMEOUT
+TEST_JSSHELL_TIMEOUT=900
 endif
 
 CURDIR := $(shell pwd)
-
-LOG ?= $(subst /www/tests/,/www/results/,$(CURDIR)/$(TEST_DATE)-$(TEST_NAME)-$(TEST_BUILDID)-$(TEST_OPTDEBUG))
+LOGPATH = $(subst /www/tests/,/www/results/,$(CURDIR))
+LOGFILE = $(LOGPATH)/$(TEST_DATE)-$(TEST_NAME)-$(TEST_BUILDID)-$(TEST_OPTDEBUG)
 
 JS_FILES=$(shell find . -mindepth 2 -name '*.js' -print | grep -v shell.js | grep -v browser.js | grep -v template.js | sort)
 
@@ -72,8 +77,9 @@ jsshell:
 		-s $(TEST_JS_EXE) \
 		-e sm$(TEST_OPTDEBUG) \
 		-o '-S 524288' \
-		-k \
-		-f $(LOG)-results-js.html 2>&1 | sed 's/^/jsshell: /'
+		-K \
+		-T $(TEST_JSSHELL_TIMEOUT) \
+		-f $(LOGFILE)-results-js.html 2>&1 | sed 's/^/jsshell: /'
 
 e4xshell:
 	-perl jsDriver.pl \
@@ -81,8 +87,9 @@ e4xshell:
 		-s $(TEST_JS_EXE) \
 		-o '-S 524288 -x' \
 		-e sm$(TEST_OPTDEBUG) \
-		-k \
-		-f $(LOG)-results-e4x.html 2>&1 | sed 's/^/e4xshell: /'
+		-K \
+		-T $(TEST_JSSHELL_TIMEOUT) \
+		-f $(LOGFILE)-results-e4x.html 2>&1 | sed 's/^/e4xshell: /'
 
 #
 # browser based tests
@@ -90,23 +97,23 @@ e4xshell:
 
 jsbrowser: menu.html
 	$(TEST_BIN)/pipeline.sh \
-		"$(TEST_BIN)/spider.pl $(TEST_BIN) $(TEST_JSBROWSER_TIMEOUT) $(TEST_EXE) $(TEST_PROFILE) /tests/mozilla.org/js/userhook-js.js menu-list.txt 0 js 2>&1" \
-	"sed 's/^/jsbrowser: /'"
+		"$(TEST_BIN)/spider.pl $(TEST_BIN) $(TEST_JSBROWSER_TIMEOUT) $(TEST_EXE) $(TEST_PROFILE) /tests/mozilla.org/js/userhook-js.js menu-list.txt 0 js 2>&1" | \
+	sed 's/^/jsbrowser: /' 
 
 e4xbrowser: menu.html
 	$(TEST_BIN)/pipeline.sh \
-		"$(TEST_BIN)/spider.pl $(TEST_BIN) $(TEST_JSBROWSER_TIMEOUT) $(TEST_EXE) $(TEST_PROFILE) /tests/mozilla.org/js/userhook-e4x.js menu-list.txt 0 e4x 2>&1" \
-		"sed 's/^/e4xbrowser: /'"
+		"$(TEST_BIN)/spider.pl $(TEST_BIN) $(TEST_JSBROWSER_TIMEOUT) $(TEST_EXE) $(TEST_PROFILE) /tests/mozilla.org/js/userhook-e4x.js menu-list.txt 0 e4x 2>&1" | \
+	sed 's/^/e4xbrowser: /' 
 
 jsbrowsereach: menu.html
 	$(TEST_BIN)/pipeline.sh \
-		"$(TEST_BIN)/spider.pl $(TEST_BIN) $(TEST_JSEACH_TIMEOUT) $(TEST_EXE) $(TEST_PROFILE) /tests/mozilla.org/js/userhookeach-js.js js-list.txt 0 js 2>&1" \
-	"sed 's/^/jsbrowsereach: /'"
+		"$(TEST_BIN)/spider.pl $(TEST_BIN) $(TEST_JSEACH_TIMEOUT) $(TEST_EXE) $(TEST_PROFILE) /tests/mozilla.org/js/userhookeach-js.js js-list.txt 0 js 2>&1" | \
+	sed 's/^/jsbrowsereach: /' 
 
 e4xbrowsereach: menu.html
 	$(TEST_BIN)/pipeline.sh \
-		"$(TEST_BIN)/spider.pl $(TEST_BIN) $(TEST_JSEACH_TIMEOUT) $(TEST_EXE) $(TEST_PROFILE) /tests/mozilla.org/js/userhookeach-e4x.js e4x-list.txt 0 e4x 2>&1" \
-	"sed 's/^/e4xbrowsereach: /'"
+		"$(TEST_BIN)/spider.pl $(TEST_BIN) $(TEST_JSEACH_TIMEOUT) $(TEST_EXE) $(TEST_PROFILE) /tests/mozilla.org/js/userhookeach-e4x.js e4x-list.txt 0 e4x 2>&1" | \
+	sed 's/^/e4xbrowsereach: /' 
 
 menu.html: menuhead.html menufoot.html $(JS_FILES) Makefile
 	$(MAKE) -f Makefile

js/tests/userhook-e4x.js

@@ -75,7 +75,8 @@ function userOnAfterPage()
         var testcase = testcases[i];
         cdump('test: '    + testcase.path + ' ' +
               'bug: '         + testcase.bugnumber + ' ' + 
-              (testcase.passed ? 'PASSED':'FAILED') + ' ' +
+              'result: ' + (testcase.passed ? 'PASSED':'FAILED') + ' ' +
+              'type: browser ' + 
               'description: ' + testcase.description + ' ' + 
               'expected: '    + testcase.expect + ' ' + 
               'actual: '      + testcase.actual + ' ' +

js/tests/userhook-js.js

@@ -75,7 +75,8 @@ function userOnAfterPage()
         var testcase = testcases[i];
         cdump('test: '    + testcase.path + ' ' +
               'bug: '         + testcase.bugnumber + ' ' + 
-              (testcase.passed ? 'PASSED':'FAILED') + ' ' +
+              'result: ' + (testcase.passed ? 'PASSED':'FAILED') + ' ' +
+              'type: browser ' + 
               'description: ' + testcase.description + ' ' + 
               'expected: '    + testcase.expect + ' ' + 
               'actual: '      + testcase.actual + ' ' +

js/tests/userhookeach-e4x.js

@@ -74,9 +74,10 @@ function checkTestCompleted()
     for (var i = 0; i < testcases.length; i++)
     {
       var testcase = testcases[i];
-      cdump('test: '    + testcase.path + ' ' + 
+      cdump('test: '    + testcase.path + ' ' +
             'bug: '         + testcase.bugnumber + ' ' + 
-            (testcase.passed ? 'PASSED':'FAILED') + ' ' +
+            'result: ' + (testcase.passed ? 'PASSED':'FAILED') + ' ' +
+            'type: browser ' + 
             'description: ' + testcase.description + ' ' + 
             'expected: '    + testcase.expect + ' ' + 
             'actual: '      + testcase.actual + ' ' +

js/tests/userhookeach-js.js

@@ -75,7 +75,8 @@ function checkTestCompleted()
       var testcase = testcases[i];
       cdump('test: '    + testcase.path + ' ' +
             'bug: '         + testcase.bugnumber + ' ' + 
-            (testcase.passed ? 'PASSED':'FAILED') + ' ' +
+            'result: ' + (testcase.passed ? 'PASSED':'FAILED') + ' ' +
+            'type: browser ' + 
             'description: ' + testcase.description + ' ' + 
             'expected: '    + testcase.expect + ' ' + 
             'actual: '      + testcase.actual + ' ' +