зеркало из https://github.com/mozilla/pjs.git
Bug 499315 - add support of "script" tests to reftests, r=dbaron.
This commit is contained in:
Bob Clary
Родитель
170f622732
Коммит
cc3d4a2668
|
|
@ -164,14 +164,31 @@ must be one of the following:
|
|||
|
||||
c. <type> is one of the following:
|
||||
|
||||
== The test passes if the images of the two renderings are the
|
||||
SAME.
|
||||
!= The test passes if the images of the two renderings are
|
||||
DIFFERENT.
|
||||
load The test passes unconditionally if the page loads. url_ref
|
||||
must be omitted, and the test cannot be marked as fails or
|
||||
random. (Used to test for crashes, hangs, assertions, and
|
||||
leaks.)
|
||||
== The test passes if the images of the two renderings are the
|
||||
SAME.
|
||||
!= The test passes if the images of the two renderings are
|
||||
DIFFERENT.
|
||||
load The test passes unconditionally if the page loads. url_ref
|
||||
must be omitted, and the test cannot be marked as fails or
|
||||
random. (Used to test for crashes, hangs, assertions, and
|
||||
leaks.)
|
||||
script The loaded page records the test's pass or failure status
|
||||
in a JavaScript data structure accessible through the following
|
||||
API.
|
||||
|
||||
getTestCases() returns an array of test result objects
|
||||
representing the results of the tests performed by the page.
|
||||
|
||||
Each test result object has two methods:
|
||||
|
||||
testPassed() returns true if the test result object passed,
|
||||
otherwise it returns false.
|
||||
|
||||
testDescription() returns a string describing the test
|
||||
result.
|
||||
|
||||
url_ref must be omitted. The test may be marked as fails or
|
||||
random. (Used to test the JavaScript Engine.)
|
||||
|
||||
d. <url> is either a relative file path or an absolute URL for the
|
||||
test page
|
||||
|
|
|
|||
|
|
@ -72,7 +72,7 @@ def parseManifest(manifest, dirs):
|
|||
if items[0] == "include":
|
||||
parseManifest(os.path.join(manifestdir, items[1]), dirs)
|
||||
continue
|
||||
elif items[0] == "load":
|
||||
elif items[0] == "load" or items[0] == "script":
|
||||
testURLs = [items[1]]
|
||||
elif items[0] == "==" or items[0] == "!=":
|
||||
testURLs = items[1:3]
|
||||
|
|
|
|||
|
|
@ -107,12 +107,16 @@ var gSlowestTestTime = 0;
|
|||
var gSlowestTestURL;
|
||||
var gClearingForAssertionCheck = false;
|
||||
|
||||
const TYPE_REFTEST_EQUAL = '==';
|
||||
const TYPE_REFTEST_NOTEQUAL = '!=';
|
||||
const TYPE_LOAD = 'load'; // test without a reference (just test that it does
|
||||
// not assert, crash, hang, or leak)
|
||||
const TYPE_SCRIPT = 'script'; // test contains individual test results
|
||||
|
||||
const EXPECTED_PASS = 0;
|
||||
const EXPECTED_FAIL = 1;
|
||||
const EXPECTED_RANDOM = 2;
|
||||
const EXPECTED_DEATH = 3; // test must be skipped to avoid e.g. crash/hang
|
||||
const EXPECTED_LOAD = 4; // test without a reference (just test that it does
|
||||
// not assert, crash, hang, or leak)
|
||||
|
||||
var HTTP_SERVER_PORT = 4444;
|
||||
const HTTP_SERVER_PORTS_TO_TRY = 50;
|
||||
|
|
@ -146,7 +150,7 @@ function OnRefTestLoad()
|
|||
var prefs = Components.classes["@mozilla.org/preferences-service;1"].
|
||||
getService(Components.interfaces.nsIPrefBranch2);
|
||||
gLoadTimeout = prefs.getIntPref("reftest.timeout");
|
||||
}
|
||||
}
|
||||
catch(e) {
|
||||
gLoadTimeout = 5 * 60 * 1000; //5 minutes as per bug 479518
|
||||
}
|
||||
|
|
@ -369,11 +373,9 @@ function ReadManifest(aURL)
|
|||
secMan.checkLoadURI(aURL, incURI,
|
||||
CI.nsIScriptSecurityManager.DISALLOW_SCRIPT);
|
||||
ReadManifest(incURI);
|
||||
} else if (items[0] == "load") {
|
||||
if (expected_status == EXPECTED_PASS)
|
||||
expected_status = EXPECTED_LOAD;
|
||||
} else if (items[0] == TYPE_LOAD) {
|
||||
if (items.length != 2 ||
|
||||
(expected_status != EXPECTED_LOAD &&
|
||||
(expected_status != EXPECTED_PASS &&
|
||||
expected_status != EXPECTED_DEATH))
|
||||
throw "Error 3 in manifest file " + aURL.spec + " line " + lineNo;
|
||||
var [testURI] = runHttp
|
||||
|
|
@ -385,16 +387,35 @@ function ReadManifest(aURL)
|
|||
: testURI.spec;
|
||||
secMan.checkLoadURI(aURL, testURI,
|
||||
CI.nsIScriptSecurityManager.DISALLOW_SCRIPT);
|
||||
gURLs.push( { equal: true /* meaningless */,
|
||||
gURLs.push( { type: TYPE_LOAD,
|
||||
expected: expected_status,
|
||||
prettyPath: prettyPath,
|
||||
minAsserts: minAsserts,
|
||||
maxAsserts: maxAsserts,
|
||||
url1: testURI,
|
||||
url2: null } );
|
||||
} else if (items[0] == "==" || items[0] == "!=") {
|
||||
if (items.length != 3)
|
||||
} else if (items[0] == TYPE_SCRIPT) {
|
||||
if (items.length != 2)
|
||||
throw "Error 4 in manifest file " + aURL.spec + " line " + lineNo;
|
||||
var [testURI] = runHttp
|
||||
? ServeFiles(aURL, httpDepth,
|
||||
listURL.file.parent, [items[1]])
|
||||
: [gIOService.newURI(items[1], null, listURL)];
|
||||
var prettyPath = runHttp
|
||||
? gIOService.newURI(items[1], null, listURL).spec
|
||||
: testURI.spec;
|
||||
secMan.checkLoadURI(aURL, testURI,
|
||||
CI.nsIScriptSecurityManager.DISALLOW_SCRIPT);
|
||||
gURLs.push( { type: TYPE_SCRIPT,
|
||||
expected: expected_status,
|
||||
prettyPath: prettyPath,
|
||||
minAsserts: minAsserts,
|
||||
maxAsserts: maxAsserts,
|
||||
url1: testURI,
|
||||
url2: null } );
|
||||
} else if (items[0] == TYPE_REFTEST_EQUAL || items[0] == TYPE_REFTEST_NOTEQUAL) {
|
||||
if (items.length != 3)
|
||||
throw "Error 5 in manifest file " + aURL.spec + " line " + lineNo;
|
||||
var [testURI, refURI] = runHttp
|
||||
? ServeFiles(aURL, httpDepth,
|
||||
listURL.file.parent, [items[1], items[2]])
|
||||
|
|
@ -407,7 +428,7 @@ function ReadManifest(aURL)
|
|||
CI.nsIScriptSecurityManager.DISALLOW_SCRIPT);
|
||||
secMan.checkLoadURI(aURL, refURI,
|
||||
CI.nsIScriptSecurityManager.DISALLOW_SCRIPT);
|
||||
gURLs.push( { equal: (items[0] == "=="),
|
||||
gURLs.push( { type: items[0],
|
||||
expected: expected_status,
|
||||
prettyPath: prettyPath,
|
||||
minAsserts: minAsserts,
|
||||
|
|
@ -415,7 +436,7 @@ function ReadManifest(aURL)
|
|||
url1: testURI,
|
||||
url2: refURI } );
|
||||
} else {
|
||||
throw "Error 5 in manifest file " + aURL.spec + " line " + lineNo;
|
||||
throw "Error 6 in manifest file " + aURL.spec + " line " + lineNo;
|
||||
}
|
||||
} while (more);
|
||||
}
|
||||
|
|
@ -437,8 +458,10 @@ function BuildUseCounts()
|
|||
{
|
||||
gURIUseCounts = {};
|
||||
for (var i = 0; i < gURLs.length; ++i) {
|
||||
var expected = gURLs[i].expected;
|
||||
if (expected != EXPECTED_DEATH && expected != EXPECTED_LOAD) {
|
||||
var url = gURLs[i];
|
||||
if (url.expected != EXPECTED_DEATH &&
|
||||
(url.type == TYPE_REFTEST_EQUAL ||
|
||||
url.type == TYPE_REFTEST_NOTEQUAL)) {
|
||||
AddURIUseCount(gURLs[i].url1);
|
||||
AddURIUseCount(gURLs[i].url2);
|
||||
}
|
||||
|
|
@ -517,7 +540,9 @@ function StartCurrentURI(aState)
|
|||
gState = aState;
|
||||
gCurrentURL = gURLs[0]["url" + aState].spec;
|
||||
|
||||
if (gURICanvases[gCurrentURL] && gURLs[0].expected != EXPECTED_LOAD &&
|
||||
if (gURICanvases[gCurrentURL] &&
|
||||
(gURLs[0].type == TYPE_REFTEST_EQUAL ||
|
||||
gURLs[0].type == TYPE_REFTEST_NOTEQUAL) &&
|
||||
gURLs[0].maxAsserts == 0) {
|
||||
// Pretend the document loaded --- DocumentLoaded will notice
|
||||
// there's already a canvas for this URL
|
||||
|
|
@ -840,12 +865,96 @@ function DocumentLoaded()
|
|||
gFailureReason = null;
|
||||
gFailureTimeout = null;
|
||||
|
||||
if (gURLs[0].expected == EXPECTED_LOAD) {
|
||||
// Not 'const ...' because of 'EXPECTED_*' value dependency.
|
||||
var outputs = {};
|
||||
const randomMsg = "(EXPECTED RANDOM)";
|
||||
outputs[EXPECTED_PASS] = {
|
||||
true: {s: "TEST-PASS" , n: "Pass"},
|
||||
false: {s: "TEST-UNEXPECTED-FAIL" , n: "UnexpectedFail"}
|
||||
};
|
||||
outputs[EXPECTED_FAIL] = {
|
||||
true: {s: "TEST-UNEXPECTED-PASS" , n: "UnexpectedPass"},
|
||||
false: {s: "TEST-KNOWN-FAIL" , n: "KnownFail"}
|
||||
};
|
||||
outputs[EXPECTED_RANDOM] = {
|
||||
true: {s: "TEST-PASS" + randomMsg , n: "Random"},
|
||||
false: {s: "TEST-KNOWN-FAIL" + randomMsg, n: "Random"}
|
||||
};
|
||||
var output;
|
||||
|
||||
if (gURLs[0].type == TYPE_LOAD) {
|
||||
++gTestResults.LoadOnly;
|
||||
dump("REFTEST TEST-PASS | " + gURLs[0].prettyPath + " | (LOAD ONLY)\n");
|
||||
FinishTestItem();
|
||||
return;
|
||||
}
|
||||
if (gURLs[0].type == TYPE_SCRIPT) {
|
||||
var missing_msg = false;
|
||||
var testwindow = gBrowser.contentWindow;
|
||||
expected = gURLs[0].expected;
|
||||
|
||||
if (testwindow.wrappedJSObject)
|
||||
testwindow = testwindow.wrappedJSObject;
|
||||
|
||||
var testcases;
|
||||
|
||||
if (!testwindow.getTestCases || typeof testwindow.getTestCases != "function") {
|
||||
// Force an unexpected failure to alert the test author to fix the test.
|
||||
expected = EXPECTED_PASS;
|
||||
missing_msg = "test must provide a function getTestCases(). (SCRIPT)\n";
|
||||
}
|
||||
else if (!(testcases = testwindow.getTestCases())) {
|
||||
// Force an unexpected failure to alert the test author to fix the test.
|
||||
expected = EXPECTED_PASS;
|
||||
missing_msg = "test's getTestCases() must return an Array-like Object. (SCRIPT)\n";
|
||||
}
|
||||
else if (testcases.length == 0) {
|
||||
// This failure may be due to a JavaScript Engine bug causing
|
||||
// early termination of the test.
|
||||
missing_msg = "No test results reported. (SCRIPT)\n";
|
||||
}
|
||||
|
||||
if (missing_msg) {
|
||||
output = outputs[expected][false];
|
||||
++gTestResults[output.n];
|
||||
var result = "REFTEST " + output.s + " | " +
|
||||
gURLs[0].prettyPath + " | " + // the URL being tested
|
||||
missing_msg;
|
||||
|
||||
dump(result);
|
||||
FinishTestItem();
|
||||
return;
|
||||
}
|
||||
|
||||
var results = testcases.map(function(test) {
|
||||
return { passed: test.testPassed(), description: test.testDescription()};
|
||||
});
|
||||
var anyFailed = results.some(function(result) { return !result.passed; });
|
||||
var outputPair;
|
||||
if (anyFailed && expected == EXPECTED_FAIL) {
|
||||
// If we're marked as expected to fail, and some (but not all) tests
|
||||
// passed, treat those tests as though they were marked random
|
||||
// (since we can't tell whether they were really intended to be
|
||||
// marked failing or not).
|
||||
outputPair = { true: outputs[EXPECTED_RANDOM][true],
|
||||
false: outputs[expected][false] };
|
||||
} else {
|
||||
outputPair = outputs[expected];
|
||||
}
|
||||
var index = 0;
|
||||
results.forEach(function(result) {
|
||||
var output = outputPair[result.passed];
|
||||
|
||||
++gTestResults[output.n];
|
||||
result = "REFTEST " + output.s + " | " +
|
||||
gURLs[0].prettyPath + " | " + // the URL being tested
|
||||
result.description + " item " + (++index) + "\n";
|
||||
dump(result);
|
||||
});
|
||||
|
||||
FinishTestItem();
|
||||
return;
|
||||
}
|
||||
|
||||
if (gURICanvases[gCurrentURL]) {
|
||||
gCurrentCanvas = gURICanvases[gCurrentURL];
|
||||
|
|
@ -889,31 +998,16 @@ function DocumentLoaded()
|
|||
}
|
||||
|
||||
// whether the comparison result matches what is in the manifest
|
||||
var test_passed = (equal == gURLs[0].equal);
|
||||
var test_passed = (equal == (gURLs[0].type == TYPE_REFTEST_EQUAL));
|
||||
// what is expected on this platform (PASS, FAIL, or RANDOM)
|
||||
var expected = gURLs[0].expected;
|
||||
output = outputs[expected][test_passed];
|
||||
|
||||
// Not 'const ...' because of 'EXPECTED_*' value dependency.
|
||||
var outputs = {};
|
||||
const randomMsg = "(EXPECTED RANDOM)";
|
||||
outputs[EXPECTED_PASS] = {
|
||||
true: {s: "TEST-PASS" , n: "Pass"},
|
||||
false: {s: "TEST-UNEXPECTED-FAIL" , n: "UnexpectedFail"}
|
||||
};
|
||||
outputs[EXPECTED_FAIL] = {
|
||||
true: {s: "TEST-UNEXPECTED-PASS" , n: "UnexpectedPass"},
|
||||
false: {s: "TEST-KNOWN-FAIL" , n: "KnownFail"}
|
||||
};
|
||||
outputs[EXPECTED_RANDOM] = {
|
||||
true: {s: "TEST-PASS" + randomMsg , n: "Random"},
|
||||
false: {s: "TEST-KNOWN-FAIL" + randomMsg, n: "Random"}
|
||||
};
|
||||
++gTestResults[output.n];
|
||||
|
||||
++gTestResults[outputs[expected][test_passed].n];
|
||||
|
||||
var result = "REFTEST " + outputs[expected][test_passed].s + " | " +
|
||||
var result = "REFTEST " + output.s + " | " +
|
||||
gURLs[0].prettyPath + " | "; // the URL being tested
|
||||
if (!gURLs[0].equal) {
|
||||
if (gURLs[0].type == TYPE_REFTEST_NOTEQUAL) {
|
||||
result += "(!=) ";
|
||||
}
|
||||
dump(result + "\n");
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче