зеркало из https://github.com/mozilla/gecko-dev.git
288 строки
11 KiB
Python
288 строки
11 KiB
Python
import math
|
|
|
|
import mozinfo
|
|
|
|
|
|
class Bisect(object):
|
|
|
|
"Class for creating, bisecting and summarizing for --bisect-chunk option."
|
|
|
|
def __init__(self, harness):
|
|
super(Bisect, self).__init__()
|
|
self.summary = []
|
|
self.contents = {}
|
|
self.repeat = 10
|
|
self.failcount = 0
|
|
self.max_failures = 3
|
|
|
|
def setup(self, tests):
|
|
"""This method is used to initialize various variables that are required
|
|
for test bisection"""
|
|
status = 0
|
|
self.contents.clear()
|
|
# We need totalTests key in contents for sanity check
|
|
self.contents["totalTests"] = tests
|
|
self.contents["tests"] = tests
|
|
self.contents["loop"] = 0
|
|
return status
|
|
|
|
def reset(self, expectedError, result):
|
|
"""This method is used to initialize self.expectedError and self.result
|
|
for each loop in runtests."""
|
|
self.expectedError = expectedError
|
|
self.result = result
|
|
|
|
def get_tests_for_bisection(self, options, tests):
|
|
"""Make a list of tests for bisection from a given list of tests"""
|
|
bisectlist = []
|
|
for test in tests:
|
|
bisectlist.append(test)
|
|
if test.endswith(options.bisectChunk):
|
|
break
|
|
|
|
return bisectlist
|
|
|
|
def pre_test(self, options, tests, status):
|
|
"""This method is used to call other methods for setting up variables and
|
|
getting the list of tests for bisection."""
|
|
if options.bisectChunk == "default":
|
|
return tests
|
|
# The second condition in 'if' is required to verify that the failing
|
|
# test is the last one.
|
|
elif "loop" not in self.contents or not self.contents["tests"][-1].endswith(
|
|
options.bisectChunk
|
|
):
|
|
tests = self.get_tests_for_bisection(options, tests)
|
|
status = self.setup(tests)
|
|
|
|
return self.next_chunk_binary(options, status)
|
|
|
|
def post_test(self, options, expectedError, result):
|
|
"""This method is used to call other methods to summarize results and check whether a
|
|
sanity check is done or not."""
|
|
self.reset(expectedError, result)
|
|
status = self.summarize_chunk(options)
|
|
# Check whether sanity check has to be done. Also it is necessary to check whether
|
|
# options.bisectChunk is present in self.expectedError as we do not want to run
|
|
# if it is "default".
|
|
if status == -1 and options.bisectChunk in self.expectedError:
|
|
# In case we have a debug build, we don't want to run a sanity
|
|
# check, will take too much time.
|
|
if mozinfo.info["debug"]:
|
|
return status
|
|
|
|
testBleedThrough = self.contents["testsToRun"][0]
|
|
tests = self.contents["totalTests"]
|
|
tests.remove(testBleedThrough)
|
|
# To make sure that the failing test is dependent on some other
|
|
# test.
|
|
if options.bisectChunk in testBleedThrough:
|
|
return status
|
|
|
|
status = self.setup(tests)
|
|
self.summary.append("Sanity Check:")
|
|
|
|
return status
|
|
|
|
def next_chunk_reverse(self, options, status):
|
|
"This method is used to bisect the tests in a reverse search fashion."
|
|
|
|
# Base Cases.
|
|
if self.contents["loop"] <= 1:
|
|
self.contents["testsToRun"] = self.contents["tests"]
|
|
if self.contents["loop"] == 1:
|
|
self.contents["testsToRun"] = [self.contents["tests"][-1]]
|
|
self.contents["loop"] += 1
|
|
return self.contents["testsToRun"]
|
|
|
|
if "result" in self.contents:
|
|
if self.contents["result"] == "PASS":
|
|
chunkSize = self.contents["end"] - self.contents["start"]
|
|
self.contents["end"] = self.contents["start"] - 1
|
|
self.contents["start"] = self.contents["end"] - chunkSize
|
|
|
|
# self.contents['result'] will be expected error only if it fails.
|
|
elif self.contents["result"] == "FAIL":
|
|
self.contents["tests"] = self.contents["testsToRun"]
|
|
status = 1 # for initializing
|
|
|
|
# initialize
|
|
if status:
|
|
totalTests = len(self.contents["tests"])
|
|
chunkSize = int(math.ceil(totalTests / 10.0))
|
|
self.contents["start"] = totalTests - chunkSize - 1
|
|
self.contents["end"] = totalTests - 2
|
|
|
|
start = self.contents["start"]
|
|
end = self.contents["end"] + 1
|
|
self.contents["testsToRun"] = self.contents["tests"][start:end]
|
|
self.contents["testsToRun"].append(self.contents["tests"][-1])
|
|
self.contents["loop"] += 1
|
|
|
|
return self.contents["testsToRun"]
|
|
|
|
def next_chunk_binary(self, options, status):
|
|
"This method is used to bisect the tests in a binary search fashion."
|
|
|
|
# Base cases.
|
|
if self.contents["loop"] <= 1:
|
|
self.contents["testsToRun"] = self.contents["tests"]
|
|
if self.contents["loop"] == 1:
|
|
self.contents["testsToRun"] = [self.contents["tests"][-1]]
|
|
self.contents["loop"] += 1
|
|
return self.contents["testsToRun"]
|
|
|
|
# Initialize the contents dict.
|
|
if status:
|
|
totalTests = len(self.contents["tests"])
|
|
self.contents["start"] = 0
|
|
self.contents["end"] = totalTests - 2
|
|
|
|
# pylint --py3k W1619
|
|
mid = (self.contents["start"] + self.contents["end"]) / 2
|
|
if "result" in self.contents:
|
|
if self.contents["result"] == "PASS":
|
|
self.contents["end"] = mid
|
|
|
|
elif self.contents["result"] == "FAIL":
|
|
self.contents["start"] = mid + 1
|
|
|
|
mid = (self.contents["start"] + self.contents["end"]) / 2
|
|
start = mid + 1
|
|
end = self.contents["end"] + 1
|
|
self.contents["testsToRun"] = self.contents["tests"][start:end]
|
|
if not self.contents["testsToRun"]:
|
|
self.contents["testsToRun"].append(self.contents["tests"][mid])
|
|
self.contents["testsToRun"].append(self.contents["tests"][-1])
|
|
self.contents["loop"] += 1
|
|
|
|
return self.contents["testsToRun"]
|
|
|
|
def summarize_chunk(self, options):
|
|
"This method is used summarize the results after the list of tests is run."
|
|
if options.bisectChunk == "default":
|
|
# if no expectedError that means all the tests have successfully
|
|
# passed.
|
|
if len(self.expectedError) == 0:
|
|
return -1
|
|
options.bisectChunk = self.expectedError.keys()[0]
|
|
self.summary.append("\tFound Error in test: %s" % options.bisectChunk)
|
|
return 0
|
|
|
|
# If options.bisectChunk is not in self.result then we need to move to
|
|
# the next run.
|
|
if options.bisectChunk not in self.result:
|
|
return -1
|
|
|
|
self.summary.append("\tPass %d:" % self.contents["loop"])
|
|
if len(self.contents["testsToRun"]) > 1:
|
|
self.summary.append(
|
|
"\t\t%d test files(start,end,failing). [%s, %s, %s]"
|
|
% (
|
|
len(self.contents["testsToRun"]),
|
|
self.contents["testsToRun"][0],
|
|
self.contents["testsToRun"][-2],
|
|
self.contents["testsToRun"][-1],
|
|
)
|
|
)
|
|
else:
|
|
self.summary.append("\t\t1 test file [%s]" % self.contents["testsToRun"][0])
|
|
return self.check_for_intermittent(options)
|
|
|
|
if self.result[options.bisectChunk] == "PASS":
|
|
self.summary.append("\t\tno failures found.")
|
|
if self.contents["loop"] == 1:
|
|
status = -1
|
|
else:
|
|
self.contents["result"] = "PASS"
|
|
status = 0
|
|
|
|
elif self.result[options.bisectChunk] == "FAIL":
|
|
if "expectedError" not in self.contents:
|
|
self.summary.append("\t\t%s failed." % self.contents["testsToRun"][-1])
|
|
self.contents["expectedError"] = self.expectedError[options.bisectChunk]
|
|
status = 0
|
|
|
|
elif (
|
|
self.expectedError[options.bisectChunk]
|
|
== self.contents["expectedError"]
|
|
):
|
|
self.summary.append(
|
|
"\t\t%s failed with expected error."
|
|
% self.contents["testsToRun"][-1]
|
|
)
|
|
self.contents["result"] = "FAIL"
|
|
status = 0
|
|
|
|
# This code checks for test-bleedthrough. Should work for any
|
|
# algorithm.
|
|
numberOfTests = len(self.contents["testsToRun"])
|
|
if numberOfTests < 3:
|
|
# This means that only 2 tests are run. Since the last test
|
|
# is the failing test itself therefore the bleedthrough
|
|
# test is the first test
|
|
self.summary.append(
|
|
"TEST-UNEXPECTED-FAIL | %s | Bleedthrough detected, this test is the "
|
|
"root cause for many of the above failures"
|
|
% self.contents["testsToRun"][0]
|
|
)
|
|
status = -1
|
|
else:
|
|
self.summary.append(
|
|
"\t\t%s failed with different error."
|
|
% self.contents["testsToRun"][-1]
|
|
)
|
|
status = -1
|
|
|
|
return status
|
|
|
|
def check_for_intermittent(self, options):
|
|
"This method is used to check whether a test is an intermittent."
|
|
if self.result[options.bisectChunk] == "PASS":
|
|
self.summary.append(
|
|
"\t\tThe test %s passed." % self.contents["testsToRun"][0]
|
|
)
|
|
if self.repeat > 0:
|
|
# loop is set to 1 to again run the single test.
|
|
self.contents["loop"] = 1
|
|
self.repeat -= 1
|
|
return 0
|
|
else:
|
|
if self.failcount > 0:
|
|
# -1 is being returned as the test is intermittent, so no need to bisect
|
|
# further.
|
|
return -1
|
|
# If the test does not fail even once, then proceed to next chunk for bisection.
|
|
# loop is set to 2 to proceed on bisection.
|
|
self.contents["loop"] = 2
|
|
return 1
|
|
elif self.result[options.bisectChunk] == "FAIL":
|
|
self.summary.append(
|
|
"\t\tThe test %s failed." % self.contents["testsToRun"][0]
|
|
)
|
|
self.failcount += 1
|
|
self.contents["loop"] = 1
|
|
self.repeat -= 1
|
|
# self.max_failures is the maximum number of times a test is allowed
|
|
# to fail to be called an intermittent. If a test fails more than
|
|
# limit set, it is a perma-fail.
|
|
if self.failcount < self.max_failures:
|
|
if self.repeat == 0:
|
|
# -1 is being returned as the test is intermittent, so no need to bisect
|
|
# further.
|
|
return -1
|
|
return 0
|
|
else:
|
|
self.summary.append(
|
|
"TEST-UNEXPECTED-FAIL | %s | Bleedthrough detected, this test is the "
|
|
"root cause for many of the above failures"
|
|
% self.contents["testsToRun"][0]
|
|
)
|
|
return -1
|
|
|
|
def print_summary(self):
|
|
"This method is used to print the recorded summary."
|
|
print("Bisection summary:")
|
|
for line in self.summary:
|
|
print(line)
|