зеркало из https://github.com/mozilla/gecko-dev.git
Bug 820154 - Implement todo() in marionette-simpletest. r=jgriffin
This commit is contained in:
Julien Pagès
Родитель
5b25273c1b
Коммит
2c45aa6d65
|
|
@ -495,7 +495,7 @@ setReq.onerror = function() {
|
|||
'expected timeout not triggered')
|
||||
|
||||
if 'fail' in self.jsFile:
|
||||
self.assertTrue(results['failed'] > 0,
|
||||
self.assertTrue(len(results['failures']) > 0,
|
||||
"expected test failures didn't occur")
|
||||
else:
|
||||
logger = get_default_logger()
|
||||
|
|
@ -504,10 +504,27 @@ setReq.onerror = function() {
|
|||
name = "got false, expected true" if failure.get('name') is None else failure['name']
|
||||
logger.test_status(self.test_name, name, 'FAIL',
|
||||
message=diag)
|
||||
self.assertEqual(0, results['failed'],
|
||||
'%d tests failed' % (results['failed']))
|
||||
for failure in results['expectedFailures']:
|
||||
diag = "" if failure.get('diag') is None else failure['diag']
|
||||
name = "got false, expected false" if failure.get('name') is None else failure['name']
|
||||
logger.test_status(self.test_name, name, 'FAIL',
|
||||
expected='FAIL', message=diag)
|
||||
for failure in results['unexpectedSuccesses']:
|
||||
diag = "" if failure.get('diag') is None else failure['diag']
|
||||
name = "got true, expected false" if failure.get('name') is None else failure['name']
|
||||
logger.test_status(self.test_name, name, 'PASS',
|
||||
expected='FAIL', message=diag)
|
||||
self.assertEqual(0, len(results['failures']),
|
||||
'%d tests failed' % len(results['failures']))
|
||||
if len(results['unexpectedSuccesses']) > 0:
|
||||
raise _UnexpectedSuccess('')
|
||||
if len(results['expectedFailures']) > 0:
|
||||
raise _ExpectedFailure((AssertionError, AssertionError(''), None))
|
||||
|
||||
self.assertTrue(results['passed'] + results['failed'] > 0,
|
||||
self.assertTrue(results['passed']
|
||||
+ len(results['failures'])
|
||||
+ len(results['expectedFailures'])
|
||||
+ len(results['unexpectedSuccesses']) > 0,
|
||||
'no tests run')
|
||||
|
||||
except ScriptTimeoutException:
|
||||
|
|
|
|||
|
|
@ -16,16 +16,16 @@ class SimpletestSanityTest(MarionetteTestCase):
|
|||
sentPass1 = "is(true, true, 'isTest3');" + self.callFinish
|
||||
sentPass2 = "is(true, true, 'isTest4');" + self.callFinish
|
||||
|
||||
self.assertEqual(1, self.marionette.execute_script(sentFail1)["failed"])
|
||||
self.assertEqual(1, len(self.marionette.execute_script(sentFail1)["failures"]))
|
||||
self.assertEqual(0, self.marionette.execute_script(sentFail2)["passed"])
|
||||
self.assertEqual(1, self.marionette.execute_script(sentPass1)["passed"])
|
||||
self.assertEqual(0, self.marionette.execute_script(sentPass2)["failed"])
|
||||
self.assertEqual(0, len(self.marionette.execute_script(sentPass2)["failures"]))
|
||||
|
||||
self.marionette.set_script_timeout(1000)
|
||||
self.assertEqual(1, self.marionette.execute_async_script(sentFail1)["failed"])
|
||||
self.assertEqual(1, len(self.marionette.execute_async_script(sentFail1)["failures"]))
|
||||
self.assertEqual(0, self.marionette.execute_async_script(sentFail2)["passed"])
|
||||
self.assertEqual(1, self.marionette.execute_async_script(sentPass1)["passed"])
|
||||
self.assertEqual(0, self.marionette.execute_async_script(sentPass2)["failed"])
|
||||
self.assertEqual(0, len(self.marionette.execute_async_script(sentPass2)["failures"]))
|
||||
|
||||
self.marionette.set_context("content")
|
||||
runtests()
|
||||
|
|
@ -39,15 +39,15 @@ class SimpletestSanityTest(MarionetteTestCase):
|
|||
sentPass1 = "isnot(true, false, 'isnotTest1');" + self.callFinish
|
||||
sentPass2 = "isnot(true, false, 'isnotTest2');" + self.callFinish
|
||||
|
||||
self.assertEqual(1, self.marionette.execute_script(sentFail1)["failed"]);
|
||||
self.assertEqual(1, len(self.marionette.execute_script(sentFail1)["failures"]));
|
||||
self.assertEqual(0, self.marionette.execute_script(sentFail2)["passed"]);
|
||||
self.assertEqual(0, self.marionette.execute_script(sentPass1)["failed"]);
|
||||
self.assertEqual(0, len(self.marionette.execute_script(sentPass1)["failures"]));
|
||||
self.assertEqual(1, self.marionette.execute_script(sentPass2)["passed"]);
|
||||
|
||||
self.marionette.set_script_timeout(1000)
|
||||
self.assertEqual(1, self.marionette.execute_async_script(sentFail1)["failed"]);
|
||||
self.assertEqual(1, len(self.marionette.execute_async_script(sentFail1)["failures"]));
|
||||
self.assertEqual(0, self.marionette.execute_async_script(sentFail2)["passed"]);
|
||||
self.assertEqual(0, self.marionette.execute_async_script(sentPass1)["failed"]);
|
||||
self.assertEqual(0, len(self.marionette.execute_async_script(sentPass1)["failures"]));
|
||||
self.assertEqual(1, self.marionette.execute_async_script(sentPass2)["passed"]);
|
||||
|
||||
self.marionette.set_context("content")
|
||||
|
|
@ -62,15 +62,15 @@ class SimpletestSanityTest(MarionetteTestCase):
|
|||
sentPass1 = "ok(1==1, 'testOk3');" + self.callFinish
|
||||
sentPass2 = "ok(1==1, 'testOk4');" + self.callFinish
|
||||
|
||||
self.assertEqual(1, self.marionette.execute_script(sentFail1)["failed"]);
|
||||
self.assertEqual(1, len(self.marionette.execute_script(sentFail1)["failures"]));
|
||||
self.assertEqual(0, self.marionette.execute_script(sentFail2)["passed"]);
|
||||
self.assertEqual(0, self.marionette.execute_script(sentPass1)["failed"]);
|
||||
self.assertEqual(0, len(self.marionette.execute_script(sentPass1)["failures"]));
|
||||
self.assertEqual(1, self.marionette.execute_script(sentPass2)["passed"]);
|
||||
|
||||
self.marionette.set_script_timeout(1000)
|
||||
self.assertEqual(1, self.marionette.execute_async_script(sentFail1)["failed"]);
|
||||
self.assertEqual(1, len(self.marionette.execute_async_script(sentFail1)["failures"]));
|
||||
self.assertEqual(0, self.marionette.execute_async_script(sentFail2)["passed"]);
|
||||
self.assertEqual(0, self.marionette.execute_async_script(sentPass1)["failed"]);
|
||||
self.assertEqual(0, len(self.marionette.execute_async_script(sentPass1)["failures"]));
|
||||
self.assertEqual(1, self.marionette.execute_async_script(sentPass2)["passed"]);
|
||||
|
||||
self.marionette.set_context("content")
|
||||
|
|
@ -78,3 +78,25 @@ class SimpletestSanityTest(MarionetteTestCase):
|
|||
self.marionette.set_context("chrome")
|
||||
runtests()
|
||||
|
||||
def test_todo(self):
|
||||
def runtests():
|
||||
sentFail1 = "todo(1==1, 'testTodo1', TEST_UNEXPECTED_PASS, TEST_KNOWN_FAIL);" + self.callFinish
|
||||
sentFail2 = "todo(1==1, 'testTodo2', TEST_UNEXPECTED_PASS, TEST_KNOWN_FAIL);" + self.callFinish
|
||||
sentPass1 = "todo(1==2, 'testTodo3');" + self.callFinish
|
||||
sentPass2 = "todo(1==2, 'testTodo4');" + self.callFinish
|
||||
|
||||
self.assertEqual(1, len(self.marionette.execute_script(sentFail1)["unexpectedSuccesses"]));
|
||||
self.assertEqual(0, len(self.marionette.execute_script(sentFail2)["expectedFailures"]));
|
||||
self.assertEqual(0, len(self.marionette.execute_script(sentPass1)["unexpectedSuccesses"]));
|
||||
self.assertEqual(1, len(self.marionette.execute_script(sentPass2)["expectedFailures"]));
|
||||
|
||||
self.marionette.set_script_timeout(1000)
|
||||
self.assertEqual(1, len(self.marionette.execute_async_script(sentFail1)["unexpectedSuccesses"]));
|
||||
self.assertEqual(0, len(self.marionette.execute_async_script(sentFail2)["expectedFailures"]));
|
||||
self.assertEqual(0, len(self.marionette.execute_async_script(sentPass1)["unexpectedSuccesses"]));
|
||||
self.assertEqual(1, len(self.marionette.execute_async_script(sentPass2)["expectedFailures"]));
|
||||
|
||||
self.marionette.set_context("content")
|
||||
runtests()
|
||||
self.marionette.set_context("chrome")
|
||||
runtests()
|
||||
|
|
|
|||
|
|
@ -16,33 +16,37 @@ this.Marionette = function Marionette(scope, window, context, logObj, timeout,
|
|||
this.heartbeatCallback = heartbeatCallback;
|
||||
this.testName = testName;
|
||||
this.TEST_UNEXPECTED_FAIL = "TEST-UNEXPECTED-FAIL";
|
||||
this.TEST_UNEXPECTED_PASS = "TEST-UNEXPECTED-PASS";
|
||||
this.TEST_PASS = "TEST-PASS";
|
||||
this.TEST_KNOWN_FAIL = "TEST-KNOWN-FAIL";
|
||||
}
|
||||
|
||||
Marionette.prototype = {
|
||||
exports: ['ok', 'is', 'isnot', 'log', 'getLogs', 'generate_results', 'waitFor',
|
||||
exports: ['ok', 'is', 'isnot', 'todo', 'log', 'getLogs', 'generate_results', 'waitFor',
|
||||
'runEmulatorCmd', 'runEmulatorShell', 'TEST_PASS', 'TEST_KNOWN_FAIL',
|
||||
'TEST_UNEXPECTED_FAIL'],
|
||||
'TEST_UNEXPECTED_FAIL', 'TEST_UNEXPECTED_PASS'],
|
||||
|
||||
ok: function Marionette__ok(condition, name, passString, failString, diag) {
|
||||
this.heartbeatCallback();
|
||||
if (typeof(diag) == "undefined") {
|
||||
diag = this.repr(condition) + " was " + !!condition + ", expected true";
|
||||
}
|
||||
let test = {'result': !!condition, 'name': name, 'diag': diag};
|
||||
addTest: function Marionette__addTest(condition, name, passString, failString, diag, state) {
|
||||
|
||||
let test = {'result': !!condition, 'name': name, 'diag': diag, 'state': state};
|
||||
this.logResult(test,
|
||||
typeof(passString) == "undefined" ? this.TEST_PASS : passString,
|
||||
typeof(failString) == "undefined" ? this.TEST_UNEXPECTED_FAIL : failString);
|
||||
this.tests.push(test);
|
||||
},
|
||||
|
||||
ok: function Marionette__ok(condition, name, passString, failString) {
|
||||
this.heartbeatCallback();
|
||||
let diag = this.repr(condition) + " was " + !!condition + ", expected true";
|
||||
this.addTest(condition, name, passString, failString, diag);
|
||||
},
|
||||
|
||||
is: function Marionette__is(a, b, name, passString, failString) {
|
||||
this.heartbeatCallback();
|
||||
let pass = (a == b);
|
||||
let diag = pass ? this.repr(a) + " should equal " + this.repr(b)
|
||||
: "got " + this.repr(a) + ", expected " + this.repr(b);
|
||||
this.ok(pass, name, passString, failString, diag);
|
||||
this.addTest(pass, name, passString, failString, diag);
|
||||
},
|
||||
|
||||
isnot: function Marionette__isnot (a, b, name, passString, failString) {
|
||||
|
|
@ -50,7 +54,18 @@ Marionette.prototype = {
|
|||
let pass = (a != b);
|
||||
let diag = pass ? this.repr(a) + " should not equal " + this.repr(b)
|
||||
: "didn't expect " + this.repr(a) + ", but got it";
|
||||
this.ok(pass, name, passString, failString, diag);
|
||||
this.addTest(pass, name, passString, failString, diag);
|
||||
},
|
||||
|
||||
todo: function Marionette__todo(condition, name, passString, failString) {
|
||||
this.heartbeatCallback();
|
||||
let diag = this.repr(condition) + " was expected false";
|
||||
this.addTest(!condition,
|
||||
name,
|
||||
typeof(passString) == "undefined" ? this.TEST_KNOWN_FAIL : passString,
|
||||
typeof(failString) == "undefined" ? this.TEST_UNEXPECTED_FAIL : failString,
|
||||
diag,
|
||||
"todo");
|
||||
},
|
||||
|
||||
log: function Marionette__log(msg, level) {
|
||||
|
|
@ -71,21 +86,32 @@ Marionette.prototype = {
|
|||
generate_results: function Marionette__generate_results() {
|
||||
this.heartbeatCallback();
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
let failures = [];
|
||||
let expectedFailures = [];
|
||||
let unexpectedSuccesses = [];
|
||||
for (let i in this.tests) {
|
||||
let isTodo = (this.tests[i].state == "todo");
|
||||
if(this.tests[i].result) {
|
||||
passed++;
|
||||
if (isTodo) {
|
||||
expectedFailures.push({'name': this.tests[i].name, 'diag': this.tests[i].diag});
|
||||
}
|
||||
else {
|
||||
passed++;
|
||||
}
|
||||
}
|
||||
else {
|
||||
failed++;
|
||||
failures.push({'name': this.tests[i].name,
|
||||
'diag': this.tests[i].diag});
|
||||
if (isTodo) {
|
||||
unexpectedSuccesses.push({'name': this.tests[i].name, 'diag': this.tests[i].diag});
|
||||
}
|
||||
else {
|
||||
failures.push({'name': this.tests[i].name, 'diag': this.tests[i].diag});
|
||||
}
|
||||
}
|
||||
}
|
||||
// Reset state in case this object is reused for more tests.
|
||||
this.tests = [];
|
||||
return {"passed": passed, "failed": failed, "failures": failures};
|
||||
return {"passed": passed, "failures": failures, "expectedFailures": expectedFailures,
|
||||
"unexpectedSuccesses": unexpectedSuccesses};
|
||||
},
|
||||
|
||||
logToFile: function Marionette__logToFile(file) {
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче