зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1517264 - Add a mechanism to regression-test the reftest parsing code. r=dbaron
This code has gotten tweaked a number of times over the years to support new log formats, but there's always the risk that a tweak will break old log formats. So this patch adds a small test function that can be run by anybody making changes to ensure old logs still parse fine. A few current log samples are tested thusly. Differential Revision: https://phabricator.services.mozilla.com/D15576 --HG-- extra : moz-landing-system : lando
This commit is contained in:
Kartikaya Gupta
Родитель
b03f03b25b
Коммит
c136391d92
|
|
@ -215,10 +215,120 @@ function log_pasted() {
|
|||
|
||||
var gTestItems;
|
||||
|
||||
// This function is not used in production code, but can be invoked manually
|
||||
// from the devtools console in order to test changes to the parsing regexes
|
||||
// in process_log.
|
||||
function test_parsing() {
|
||||
// Note that the logs in these testcases have been manually edited to strip
|
||||
// out stuff for brevity.
|
||||
var testcases = [
|
||||
{ "name": "empty log",
|
||||
"log": "",
|
||||
"expected": { "pass": 0, "unexpected": 0, "random": 0, "skip": 0 },
|
||||
},
|
||||
{ "name": "android log",
|
||||
"log": `[task 2018-12-28T10:36:45.718Z] 10:36:45 INFO - REFTEST TEST-START | a == b
|
||||
[task 2018-12-28T10:36:45.719Z] 10:36:45 INFO - REFTEST TEST-LOAD | a | 78 / 275 (28%)
|
||||
[task 2018-12-28T10:36:56.138Z] 10:36:56 INFO - REFTEST TEST-LOAD | b | 78 / 275 (28%)
|
||||
[task 2018-12-28T10:37:06.559Z] 10:37:06 INFO - REFTEST TEST-UNEXPECTED-FAIL | a == b | image comparison, max difference: 255, number of differing pixels: 5950
|
||||
[task 2018-12-28T10:37:06.568Z] 10:37:06 INFO - REFTEST IMAGE 1 (TEST): data:image/png;base64,
|
||||
[task 2018-12-28T10:37:06.577Z] 10:37:06 INFO - REFTEST IMAGE 2 (REFERENCE): data:image/png;base64,
|
||||
[task 2018-12-28T10:37:06.577Z] 10:37:06 INFO - REFTEST INFO | Saved log: stuff trimmed here
|
||||
[task 2018-12-28T10:37:06.582Z] 10:37:06 INFO - REFTEST TEST-END | a == b
|
||||
[task 2018-12-28T10:37:06.583Z] 10:37:06 INFO - REFTEST TEST-START | a2 == b2
|
||||
[task 2018-12-28T10:37:06.583Z] 10:37:06 INFO - REFTEST TEST-LOAD | a2 | 79 / 275 (28%)
|
||||
[task 2018-12-28T10:37:06.584Z] 10:37:06 INFO - REFTEST TEST-LOAD | b2 | 79 / 275 (28%)
|
||||
[task 2018-12-28T10:37:16.982Z] 10:37:16 INFO - REFTEST TEST-PASS | a2 == b2 | image comparison, max difference: 0, number of differing pixels: 0
|
||||
[task 2018-12-28T10:37:16.982Z] 10:37:16 INFO - REFTEST TEST-END | a2 == b2`,
|
||||
"expected": { "pass": 1, "unexpected": 1, "random": 0, "skip": 0 },
|
||||
},
|
||||
{ "name": "local reftest run (Linux)",
|
||||
"log": `REFTEST TEST-START | file:///a == file:///b
|
||||
REFTEST TEST-LOAD | file:///a | 73 / 86 (84%)
|
||||
REFTEST TEST-LOAD | file:///b | 73 / 86 (84%)
|
||||
REFTEST TEST-PASS | file:///a == file:///b | image comparison, max difference: 0, number of differing pixels: 0
|
||||
REFTEST TEST-END | file:///a == file:///b`,
|
||||
"expected": { "pass": 1, "unexpected": 0, "random": 0, "skip": 0 },
|
||||
},
|
||||
{ "name": "wpt reftests (Linux automation)",
|
||||
"log": `16:50:43 INFO - TEST-START | /a
|
||||
16:50:43 INFO - PID 4276 | 1548694243694 Marionette INFO Testing http://web-platform.test:8000/a == http://web-platform.test:8000/b
|
||||
16:50:43 INFO - PID 4276 | 1548694243963 Marionette INFO No differences allowed
|
||||
16:50:44 INFO - TEST-PASS | /a | took 370ms
|
||||
16:50:44 INFO - TEST-START | /a2
|
||||
16:50:44 INFO - PID 4276 | 1548694244066 Marionette INFO Testing http://web-platform.test:8000/a2 == http://web-platform.test:8000/b2
|
||||
16:50:44 INFO - PID 4276 | 1548694244792 Marionette INFO No differences allowed
|
||||
16:50:44 INFO - PID 4276 | 1548694244792 Marionette INFO Found 28 pixels different, maximum difference per channel 14
|
||||
16:50:44 INFO - TEST-UNEXPECTED-FAIL | /a2 | Testing http://web-platform.test:8000/a2 == http://web-platform.test:8000/b2
|
||||
16:50:44 INFO - REFTEST IMAGE 1 (TEST): data:image/png;base64,
|
||||
16:50:44 INFO - REFTEST IMAGE 2 (REFERENCE): data:image/png;base64,
|
||||
16:50:44 INFO - TEST-INFO took 840ms`,
|
||||
"expected": { "pass": 1, "unexpected": 1, "random": 0, "skip": 0 },
|
||||
},
|
||||
{ "name": "windows log",
|
||||
"log": `12:17:14 INFO - REFTEST TEST-START | a == b
|
||||
12:17:14 INFO - REFTEST TEST-LOAD | a | 1603 / 2053 (78%)
|
||||
12:17:14 INFO - REFTEST TEST-LOAD | b | 1603 / 2053 (78%)
|
||||
12:17:14 INFO - REFTEST TEST-PASS(EXPECTED RANDOM) | a == b | image comparison, max difference: 0, number of differing pixels: 0
|
||||
12:17:14 INFO - REFTEST TEST-END | a == b
|
||||
12:17:14 INFO - REFTEST TEST-START | a2 == b2
|
||||
12:17:14 INFO - REFTEST TEST-LOAD | a2 | 1604 / 2053 (78%)
|
||||
12:17:14 INFO - REFTEST TEST-LOAD | b2 | 1604 / 2053 (78%)
|
||||
12:17:14 INFO - REFTEST TEST-UNEXPECTED-FAIL | a2 == b2 | image comparison, max difference: 255, number of differing pixels: 9976
|
||||
12:17:14 INFO - REFTEST IMAGE 1 (TEST): data:image/png;base64,
|
||||
12:17:14 INFO - REFTEST IMAGE 2 (REFERENCE): data:image/png;base64,
|
||||
12:17:14 INFO - REFTEST INFO | Saved log: stuff trimmed here
|
||||
12:17:14 INFO - REFTEST TEST-END | a2 == b2
|
||||
12:01:09 INFO - REFTEST TEST-START | a3 == b3
|
||||
12:01:09 INFO - REFTEST TEST-LOAD | a3 | 66 / 189 (34%)
|
||||
12:01:09 INFO - REFTEST TEST-LOAD | b3 | 66 / 189 (34%)
|
||||
12:01:09 INFO - REFTEST TEST-KNOWN-FAIL | a3 == b3 | image comparison, max difference: 255, number of differing pixels: 9654
|
||||
12:01:09 INFO - REFTEST TEST-END | a3 == b3`,
|
||||
"expected": { "pass": 1, "unexpected": 1, "random": 1, "skip": 0 },
|
||||
},
|
||||
];
|
||||
|
||||
var current_test = 0;
|
||||
|
||||
// Override the build_viewer function invoked at the end of process_log to
|
||||
// actually just check the results of parsing.
|
||||
build_viewer = function() {
|
||||
var expected = testcases[current_test].expected;
|
||||
for (var result of gTestItems) {
|
||||
for (let type in expected) { // type is "pass", "unexpected" etc.
|
||||
if (result[type]) {
|
||||
expected[type]--;
|
||||
}
|
||||
}
|
||||
}
|
||||
var failed = false;
|
||||
for (let type in expected) {
|
||||
if (expected[type] != 0) {
|
||||
console.log(`Failure: for testcase ${testcases[current_test].name} got ${expected[type]} fewer ${type} results than expected!`);
|
||||
failed = true;
|
||||
}
|
||||
}
|
||||
if (!failed) {
|
||||
console.log(`Success for testcase ${testcases[current_test].name}`);
|
||||
}
|
||||
};
|
||||
|
||||
while (current_test < testcases.length) {
|
||||
process_log(testcases[current_test].log);
|
||||
current_test++;
|
||||
}
|
||||
}
|
||||
|
||||
function process_log(contents) {
|
||||
var lines = contents.split(/[\r\n]+/);
|
||||
gTestItems = [];
|
||||
for (var j in lines) {
|
||||
|
||||
// !!!!!!
|
||||
// When making any changes to this code, please add a test to the
|
||||
// test_parsing function above, and ensure all existing tests pass.
|
||||
// !!!!!!
|
||||
|
||||
var line = lines[j];
|
||||
// Ignore duplicated output in logcat.
|
||||
if (line.match(/I\/Gecko.*?REFTEST/))
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче