From 2eca56c5af36ce377828413bc96b44a214c629d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20Pag=C3=A8s?= Date: Tue, 16 Dec 2014 22:38:58 +0100 Subject: [PATCH] create bisector and main modules from regression --- .coveragerc | 4 + mozregression/bisector.py | 429 ++++++++++++++++++++++++++++++++++++ mozregression/main.py | 200 +++++++++++++++++ tests/unit/test_bisector.py | 277 +++++++++++++++++++++++ tests/unit/test_main.py | 120 ++++++++++ 5 files changed, 1030 insertions(+) create mode 100644 mozregression/bisector.py create mode 100644 mozregression/main.py create mode 100644 tests/unit/test_bisector.py create mode 100644 tests/unit/test_main.py diff --git a/.coveragerc b/.coveragerc index b2e7c111..463d9ed5 100644 --- a/.coveragerc +++ b/.coveragerc @@ -6,3 +6,7 @@ source = mozregression exclude_lines = # Don't complain if tests don't hit defensive assertion code raise NotImplementedError + # pass instructions does need to be tested + pass + # Don't complain if non-runnable code isn't run + if __name__ == .__main__.: diff --git a/mozregression/bisector.py b/mozregression/bisector.py new file mode 100644 index 00000000..60acc9fd --- /dev/null +++ b/mozregression/bisector.py @@ -0,0 +1,429 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import math +import sys +import datetime +from mozlog.structured import get_default_logger +from mozregression.launchers import create_launcher +from mozregression.utils import format_date +from mozregression.build_data import NightlyBuildData +from mozregression.inboundfinder import BuildsFinder + +def compute_steps_left(steps): + if steps <= 1: + return 0 + return math.trunc(math.log(steps, 2)) + +class BisectorHandler(object): + """ + React to events of a :class:`Bisector`. This is intended to be subclassed. + + A BisectorHandler keep the state of the current bisection process. + """ + build_type = 'unknown' + + def __init__(self, fetch_config, persist=None, launcher_kwargs=None): + self.fetch_config = fetch_config + self.persist = persist + self.launcher_kwargs = launcher_kwargs or {} + self.found_repo = None + self.build_data = None + self.last_good_revision = None + self.first_bad_revision = None + self.found_repo = None + self.app_info = None + self._logger = get_default_logger('Bisector') + + def set_build_data(self, build_data): + """ + Save a reference of the :class:`mozregression.build_data.BuildData` + instance. + + This is called by the bisector before each step of the bisection + process. + """ + self.build_data = build_data + + def launcher_persist_prefix(self, index): + """ + Returns an appropriate prefix for a downloaded build. + """ + raise NotImplementedError + + def _print_progress(self, new_data): + """ + Log the current state of the bisection process. + """ + raise NotImplementedError + + def start_launcher(self, index): + """ + Create and returns a :class:`mozregression.launchers.Launcher` + that has been started. + + This is called by the bisector when a build needs to be tested. + """ + build_url = self.build_data[index]['build_url'] + launcher = create_launcher(self.fetch_config.app_name, + build_url, + persist=self.persist, + persist_prefix=self.launcher_persist_prefix(index)) + launcher.start(**self.launcher_kwargs) + self.app_info = launcher.get_app_info() + self.found_repo = self.app_info['application_repository'] + return launcher + + def get_pushlog_url(self): + return "%s/pushloghtml?fromchange=%s&tochange=%s" % ( + self.found_repo, self.last_good_revision, self.first_bad_revision) + + def _get_str_range(self): + return ('revision: %s' % self.last_good_revision, + 'revision: %s' % self.first_bad_revision) + + def print_range(self): + """ + Log the state of the current state of the bisection process, with an + appropriate pushlog url. + """ + good, bad = self._get_str_range() + self._logger.info("Last good %s" % good) + self._logger.info("First bad %s" % bad) + self._logger.info("Pushlog:\n%s\n" % self.get_pushlog_url()) + + def build_good(self, mid, new_data): + """ + Called by the Bisector when a build is good. + """ + self.last_good_revision = self.app_info['application_changeset'] + if len(new_data) > 1: + self._print_progress(new_data) + + def build_bad(self, mid, new_data): + """ + Called by the Bisector when a build is bad. + """ + self.first_bad_revision = self.app_info['application_changeset'] + if len(new_data) > 1: + self._print_progress(new_data) + + def build_retry(self, mid): + pass + + def build_skip(self, mid): + pass + + def no_data(self): + pass + + def finished(self): + pass + + def user_exit(self, mid): + pass + +class NightlyHandler(BisectorHandler): + build_type = 'nightly' + good_date = None + bad_date = None + mid_date = None + + def launcher_persist_prefix(self, index): + return '%s--%s--' % (self.mid_date, + self.fetch_config.get_nightly_repo(self.mid_date)) + + def start_launcher(self, mid): + # register dates + self.good_date = self.build_data.get_date_for_index(0) + self.bad_date = self.build_data.get_date_for_index(-1) + self.mid_date = self.build_data.get_date_for_index(mid) + self._logger.info("Running nightly for %s" % self.mid_date) + return BisectorHandler.start_launcher(self, mid) + + def _print_progress(self, new_data): + next_good_date = new_data.get_date_for_index(0) + next_bad_date = new_data.get_date_for_index(-1) + next_days_range = (next_bad_date - next_good_date).days + self._logger.info("Narrowed nightly regression window from" + " [%s, %s] (%d days) to [%s, %s] (%d days)" + " (~%d steps left)" + % (format_date(self.good_date), + format_date(self.bad_date), + (self.bad_date - self.good_date).days, + format_date(next_good_date), + format_date(next_bad_date), + next_days_range, + compute_steps_left(next_days_range))) + + def ensure_metadata(self): + if not self.last_good_revision: + date = self.build_data.get_date_for_index(0) + infos = self.build_data.get_build_infos_for_date(date) + self.found_repo = infos['repository'] + self.last_good_revision = infos['changeset'] + + if not self.first_bad_revision: + date = self.build_data.get_date_for_index(-1) + infos = self.build_data.get_build_infos_for_date(date) + self.found_repo = infos['repository'] + self.first_bad_revision = infos['changeset'] + + def _get_str_range(self): + good, bad = BisectorHandler._get_str_range(self) + good += ' (%s)' % self.good_date + bad += ' (%s)' % self.bad_date + return good, bad + +class InboundHandler(BisectorHandler): + build_type = 'inbound' + + def launcher_persist_prefix(self, index): + return '%s--%s--' % (self.build_data[index]['timestamp'], + self.fetch_config.inbound_branch) + + def start_launcher(self, mid): + data = self.build_data[mid] + self._logger.info("Testing inbound build with timestamp %s," + " revision %s" + % (data['timestamp'], + data['revision'])) + return BisectorHandler.start_launcher(self, mid) + + def _print_progress(self, new_data): + self._logger.info("Narrowed inbound regression window from [%s, %s]" + " (%d revisions) to [%s, %s] (%d revisions)" + " (~%d steps left)" + % (self.build_data[0]['revision'], + self.build_data[-1]['revision'], + len(self.build_data), + new_data[0]['revision'], + new_data[-1]['revision'], + len(new_data), + compute_steps_left(len(new_data)))) + +class Bisector(object): + """ + Handle the logic of the bisection process, and report events to a given + :class:`BisectorHandler`. + """ + NO_DATA = 1 + FINISHED = 2 + USER_EXIT = 3 + + def __init__(self, handler): + self.handler = handler + + def get_verdict(self, offer_skip=True): + options = ['good', 'bad'] + if offer_skip: + options.append('skip') + options += ['retry', 'exit'] + # allow user to just type one letter + allowed_inputs = options + [o[0] for o in options] + # format options to nice printing + formatted_options = (', '.join(["'%s'" % o for o in options[:-1]]) + + " or '%s'" % options[-1]) + verdict = "" + while verdict not in allowed_inputs: + verdict = raw_input("Was this %s build good, bad, or broken?" + " (type %s and press Enter): " + % (self.handler.build_type, + formatted_options)) + + # shorten verdict to one character for processing... + return verdict[0] + + def bisect(self, build_data): + """ + Starts a bisection for a :class:`mozregression.build_data.BuildData`. + """ + while True: + self.handler.set_build_data(build_data) + mid = build_data.mid_point() + + if len(build_data) == 0: + self.handler.no_data() + return self.NO_DATA + + if mid == 0: + self.handler.finished() + return self.FINISHED + + launcher = self.handler.start_launcher(mid) + verdict = self.get_verdict() + launcher.stop() + + if verdict == 'g': + # if build is good, we have to split from + # [G, ?, ?, G, ?, B] + # to + # [G, ?, B] + build_data = build_data[mid:] + build_data.ensure_limits() + self.handler.build_good(mid, build_data) + elif verdict == 'b': + # if build is bad, we have to split from + # [G, ?, ?, B, ?, B] + # to + # [G, ?, ?, B] + build_data = build_data[:mid+1] + build_data.ensure_limits() + self.handler.build_bad(mid, build_data) + elif verdict == 'r': + self.handler.build_retry(mid) + elif verdict == 's': + self.handler.build_skip(mid) + del build_data[mid] + else: + # user exit + self.handler.user_exit(mid) + return self.USER_EXIT + +class BisectRunner(object): + def __init__(self, fetch_config, options): + self.fetch_config = fetch_config + self.options = options + self.launcher_kwargs = dict( + addons=options.addons, + profile=options.profile, + cmdargs=options.cmdargs, + ) + self._logger = get_default_logger('Bisector') + + def bisect_nightlies(self, good_date, bad_date): + build_data = NightlyBuildData(good_date, bad_date, self.fetch_config) + handler = NightlyHandler(self.fetch_config, + persist=self.options.persist, + launcher_kwargs=self.launcher_kwargs) + bisector = Bisector(handler) + result = bisector.bisect(build_data) + if result == Bisector.FINISHED: + self._logger.info("Got as far as we can go bisecting nightlies...") + self._logger.info("Ensuring we have enough metadata to get a pushlog...") + handler.ensure_metadata() + handler.print_range() + if self.fetch_config.can_go_inbound(): + self._logger.info("... attempting to bisect inbound builds" + " (starting from previous week, to make" + " sure no inbound revision is missed)") + infos = {} + days = 6 + while not 'changeset' in infos: + days += 1 + prev_date = good_date - datetime.timedelta(days=days) + infos = handler.build_data.get_build_infos_for_date(prev_date) + if days > 7: + self._logger.info("At least one build folder was" + " invalid, we have to start from" + " %d days ago." % days) + return self.bisect_inbound(infos['changeset'], + handler.first_bad_revision) + else: + message = ("Can not bissect inbound for application `%s`" + % self.fetch_config.app_name) + if self.fetch_config.is_inbound(): + # the config is able to bissect inbound but not + # for this repo. + message += (" because the repo `%s` was specified" + % self.options.repo) + self._logger.info(message + '.') + elif result == Bisector.USER_EXIT: + self._logger.info('Newest known good nightly: %s' + % handler.good_date) + self._logger.info('Oldest known bad nightly: %s' + % handler.bad_date) + self.print_resume_info(handler) + else: + # NO_DATA + self._logger.info("Unable to get valid builds within the given" + " range. You should try to launch mozregression" + " again with a larger date range.") + return 1 + return 0 + + def bisect_inbound(self, good_rev, bad_rev): + self._logger.info("Getting inbound builds between %s and %s" + % (good_rev, bad_rev)) + # anything within twelve hours is potentially within the range + # (should be a tighter but some older builds have wrong timestamps, + # see https://bugzilla.mozilla.org/show_bug.cgi?id=1018907 ... + # we can change this at some point in the future, after those builds + # expire) + build_finder = BuildsFinder(self.fetch_config) + inbound_data = build_finder.get_build_infos(good_rev, + bad_rev, + range=60*60*12) + handler = InboundHandler(self.fetch_config, + persist=self.options.persist, + launcher_kwargs=self.launcher_kwargs) + handler.last_good_revision = good_rev + handler.first_bad_revision = bad_rev + bisector = Bisector(handler) + result = bisector.bisect(inbound_data) + if result == Bisector.FINISHED: + self._logger.info("Oh noes, no (more) inbound revisions :(") + handler.print_range() + self.offer_build(handler.last_good_revision, + handler.first_bad_revision) + elif result == Bisector.USER_EXIT: + self._logger.info('Newest known good inbound revision: %s' + % handler.last_good_revision) + self._logger.info('Oldest known bad inbound revision: %s' + % handler.first_bad_revision) + + self.print_resume_info(handler) + else: + # NO_DATA + self._logger.info("No inbound data found.") + return 1 + return 0 + + def print_resume_info(self, handler): + if isinstance(handler, NightlyHandler): + info = '--good=%s --bad=%s' % (handler.good_date, handler.bad_date) + else: + info = ('--inbound --good-rev=%s --bad-rev=%s' + % (handler.last_good_revision, handler.first_bad_revision)) + options = self.options + info += ' --app=%s' % options.app + if len(options.addons) > 0: + info += ' --addons=%s' % ",".join(options.addons) + if options.profile is not None: + info += ' --profile=%s' % options.profile + if options.inbound_branch is not None: + info += ' --inbound-branch=%s' % options.inbound_branch + info += ' --bits=%s' % options.bits + if options.persist is not None: + info += ' --persist=%s' % options.persist + + self._logger.info('To resume, run:') + self._logger.info('mozregression %s' % info) + + def find_regression_chset(self, last_good_revision, first_bad_revision): + # Uses mozcommitbuilder to bisect on changesets + # Only needed if they want to bisect, so we'll put the dependency here. + from mozcommitbuilder import builder + commit_builder = builder.Builder() + + self._logger.info(" Narrowed changeset range from %s to %s" + % (last_good_revision, first_bad_revision)) + + self._logger.info("Time to do some bisecting and building!") + commit_builder.bisect(last_good_revision, first_bad_revision) + quit() + + def offer_build(self, last_good_revision, first_bad_revision): + verdict = raw_input("do you want to bisect further by fetching" + " the repository and building? (y or n) ") + if verdict != "y": + sys.exit() + + if self.fetch_config.app_name == "firefox": + self.find_regression_chset(last_good_revision, first_bad_revision) + else: + sys.exit("Bisection on anything other than firefox is not" + " currently supported.") diff --git a/mozregression/main.py b/mozregression/main.py new file mode 100644 index 00000000..d0b918be --- /dev/null +++ b/mozregression/main.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import mozinfo +import datetime +import sys +from argparse import ArgumentParser +from mozlog.structured import commandline, get_default_logger + +from mozregression import errors +from mozregression import limitedfilecache +from mozregression import __version__ +from mozregression.utils import (parse_date, date_of_release, format_date, + parse_bits, set_http_cache_session, + one_gigabyte, formatted_valid_release_dates) +from mozregression.fetch_configs import create_config +from mozregression.bisector import BisectRunner + +def parse_args(argv=None): + usage = ("\n" + " %(prog)s [OPTIONS]" + " [[--bad BAD_DATE]|[--bad-release BAD_RELEASE]]" + " [[--good GOOD_DATE]|[--good-release GOOD_RELEASE]]" + "\n" + " %(prog)s [OPTIONS]" + " --inbound --bad-rev BAD_REV --good-rev GOOD_REV") + + parser = ArgumentParser(usage=usage) + parser.add_argument("--version", action="version", version=__version__, + help=("print the mozregression version number and" + " exits.")) + + parser.add_argument("-b", "--bad", + metavar="YYYY-MM-DD", + dest="bad_date", + help="first known bad nightly build, default is today.") + + parser.add_argument("-g", "--good", + metavar="YYYY-MM-DD", + dest="good_date", + help="last known good nightly build.") + + parser.add_argument("--list-releases", + action="store_true", + help="list all known releases and exit") + + parser.add_argument("--bad-release", + type=int, + help=("first known bad nightly build. This option is" + " incompatible with --bad.")) + + parser.add_argument("--good-release", + type=int, + help=("last known good nightly build. This option is" + " incompatible with --good.")) + + parser.add_argument("--inbound", + action="store_true", + help=("use inbound instead of nightlies (use --good-rev" + " and --bad-rev options.")) + + parser.add_argument("--bad-rev", dest="first_bad_revision", + help="first known bad revision (use with --inbound).") + + parser.add_argument("--good-rev", dest="last_good_revision", + help="last known good revision (use with --inbound).") + + parser.add_argument("-e", "--addon", + dest="addons", + action='append', + default=[], + metavar="PATH1", + help="an addon to install; repeat for multiple addons.") + + parser.add_argument("-p", "--profile", + metavar="PATH", + help="profile to use with nightlies.") + + parser.add_argument("-a", "--arg", + dest="cmdargs", + action='append', + default=[], + metavar="ARG1", + help=("a command-line argument to pass to the" + " application; repeat for multiple arguments.")) + + parser.add_argument("-n", "--app", + choices=('firefox', 'fennec', 'thunderbird', 'b2g'), + default="firefox", + help="application name. Default: %(default)s.") + + parser.add_argument("--repo", + metavar="[mozilla-aurora|mozilla-beta|...]", + help="repository name used for nightly hunting.") + + parser.add_argument("--inbound-branch", + metavar="[b2g-inbound|fx-team|...]", + help="inbound branch name on ftp.mozilla.org.") + + parser.add_argument("--bits", + choices=("32", "64"), + default=mozinfo.bits, + help=("force 32 or 64 bit version (only applies to" + " x86_64 boxes). Default: %(default)s bits.")) + + parser.add_argument("--persist", + help=("the directory in which downloaded files are" + " to persist.")) + + parser.add_argument("--http-cache-dir", + help=("the directory for caching http requests." + " If not set there will be an in-memory cache" + " used.")) + + commandline.add_logging_group(parser) + options = parser.parse_args(argv) + options.bits = parse_bits(options.bits) + return options + + +def cli(argv=None): + default_bad_date = str(datetime.date.today()) + default_good_date = "2009-01-01" + options = parse_args(argv) + logger = commandline.setup_logging("mozregression", options, {"mach": sys.stdout}) + + if options.list_releases: + print(formatted_valid_release_dates()) + sys.exit() + + cache_session = limitedfilecache.get_cache( + options.http_cache_dir, one_gigabyte, + logger=get_default_logger('Limited File Cache')) + set_http_cache_session(cache_session) + + fetch_config = create_config(options.app, mozinfo.os, options.bits) + runner = BisectRunner(fetch_config, options) + + if fetch_config.is_inbound(): + # this can be useful for both inbound and nightly, because we + # can go to inbound from nightly. + fetch_config.set_inbound_branch(options.inbound_branch) + + if options.inbound: + if not fetch_config.is_inbound(): + sys.exit('Unable to bissect inbound for `%s`' % fetch_config.app_name) + if not options.last_good_revision or not options.first_bad_revision: + sys.exit("If bisecting inbound, both --good-rev and --bad-rev" + " must be set") + app = lambda: runner.bisect_inbound(options.last_good_revision, + options.first_bad_revision) + else: + # TODO: currently every fetch_config is nightly aware. Shoud we test + # for this to be sure here ? + fetch_config.set_nightly_repo(options.repo) + if not options.bad_release and not options.bad_date: + options.bad_date = default_bad_date + logger.info("No 'bad' date specified, using %s" % options.bad_date) + elif options.bad_release and options.bad_date: + sys.exit("Options '--bad_release' and '--bad_date' are" + " incompatible.") + elif options.bad_release: + options.bad_date = date_of_release(options.bad_release) + if options.bad_date is None: + sys.exit("Unable to find a matching date for release " + + str(options.bad_release) + + "\n" + formatted_valid_release_dates()) + logger.info("Using 'bad' date %s for release %s" + % (options.bad_date, options.bad_release)) + if not options.good_release and not options.good_date: + options.good_date = default_good_date + logger.info("No 'good' date specified, using %s" + % options.good_date) + elif options.good_release and options.good_date: + sys.exit("Options '--good_release' and '--good_date'" + " are incompatible.") + elif options.good_release: + options.good_date = date_of_release(options.good_release) + if options.good_date is None: + sys.exit("Unable to find a matching date for release " + + str(options.good_release) + + "\n" + formatted_valid_release_dates()) + logger.info("Using 'good' date %s for release %s" + % (options.good_date, options.good_release)) + + app = lambda: runner.bisect_nightlies(parse_date(options.good_date), + parse_date(options.bad_date)) + try: + sys.exit(app()) + except KeyboardInterrupt: + sys.exit("\nInterrupted.") + except errors.MozRegressionError as exc: + sys.exit(str(exc)) + + +if __name__ == "__main__": + cli() diff --git a/tests/unit/test_bisector.py b/tests/unit/test_bisector.py new file mode 100644 index 00000000..f08aa34e --- /dev/null +++ b/tests/unit/test_bisector.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest +from mock import patch, Mock, call +import datetime + +from mozregression.fetch_configs import create_config +from mozregression.bisector import (BisectorHandler, NightlyHandler, + InboundHandler, Bisector) + +class TestBisectorHandler(unittest.TestCase): + def setUp(self): + self.handler = BisectorHandler(create_config('firefox', 'linux', 64)) + self.handler.set_build_data([ + {'build_url': 'http://build_url_0'} + ]) + + @patch('mozregression.bisector.BisectorHandler.launcher_persist_prefix') + @patch('mozregression.bisector.create_launcher') + def test_start_launcher(self, create_launcher, launcher_persist_prefix): + app_info = {'application_repository': 'something'} + launcher = Mock(get_app_info=Mock(return_value=app_info)) + create_launcher.return_value = launcher + launcher_persist_prefix.return_value = 'something-' + + result = self.handler.start_launcher(0) + # create launcher is well called + create_launcher.assert_called_with('firefox', 'http://build_url_0', + persist=None, + persist_prefix='something-') + # launcher is started + launcher.start.assert_called_with(**self.handler.launcher_kwargs) + # app_info and found_repo are set + self.assertEqual(self.handler.app_info, app_info) + self.assertEqual(self.handler.found_repo, 'something') + # and launvher instance is returned + self.assertEqual(result, launcher) + + def test_get_pushlog_url(self): + self.handler.found_repo = 'https://hg.mozilla.repo' + self.handler.last_good_revision = '2' + self.handler.first_bad_revision = '6' + self.assertEqual(self.handler.get_pushlog_url(), + "https://hg.mozilla.repo/pushloghtml?fromchange=2&tochange=6") + + def test_print_range(self): + self.handler.found_repo = 'https://hg.mozilla.repo' + self.handler.last_good_revision = '2' + self.handler.first_bad_revision = '6' + log = [] + self.handler._logger = Mock(info = log.append) + + self.handler.print_range() + self.assertEqual(log[0], "Last good revision: 2") + self.assertEqual(log[1], "First bad revision: 6") + self.assertIn(self.handler.get_pushlog_url(), log[2]) + + @patch('mozregression.bisector.BisectorHandler._print_progress') + def test_build_good(self, _print_progress): + self.handler.app_info = {"application_changeset": '123'} + # call build_good with no new data points + self.handler.build_good(0, []) + self.assertEqual(self.handler.last_good_revision, '123') + _print_progress.assert_not_called() + # with at least two, _print_progress will be called + self.handler.build_good(0, [1, 2]) + _print_progress.assert_called_with([1, 2]) + + @patch('mozregression.bisector.BisectorHandler._print_progress') + def test_build_bad(self, _print_progress): + self.handler.app_info = {"application_changeset": '123'} + # call build_bad with no new data points + self.handler.build_bad(0, []) + self.assertEqual(self.handler.first_bad_revision, '123') + _print_progress.assert_not_called() + # with at least two, _print_progress will be called + self.handler.build_bad(0, [1, 2]) + _print_progress.assert_called_with([1, 2]) + +class TestNightlyHandler(unittest.TestCase): + def setUp(self): + self.handler = NightlyHandler(create_config('firefox', 'linux', 64)) + + def test_launcher_persist_prefix(self): + # define a repo and a mid_date + self.handler.fetch_config.set_nightly_repo('myrepo') + self.handler.mid_date = datetime.date(2014, 11, 10) + + prefix = self.handler.launcher_persist_prefix(1) + self.assertEqual(prefix, '2014-11-10--myrepo--') + + @patch('mozregression.bisector.NightlyHandler.launcher_persist_prefix') + @patch('mozregression.bisector.BisectorHandler.start_launcher') + def test_start_launcher(self, start_launcher, launcher_persist_prefix): + get_date_for_index = Mock(side_effect=lambda i: i) + self.handler.build_data = Mock(get_date_for_index=get_date_for_index) + start_launcher.return_value = 'my_launcher' + + launcher = self.handler.start_launcher(3) + # check we have called get_date_for_index + get_date_for_index.assert_has_calls([call(0), call(3), call(-1)], + any_order=True) + # dates are well set + self.assertEqual(self.handler.mid_date, 3) + self.assertEqual(self.handler.good_date, 0) + self.assertEqual(self.handler.bad_date, -1) + # base BisectorHandler.start_launcher has been called and is returned + start_launcher.assert_called_with(self.handler, 3) + self.assertEqual(launcher, 'my_launcher') + + def test_print_progress(self): + log = [] + self.handler._logger = Mock(info = log.append) + self.handler.good_date = datetime.date(2014, 11, 10) + self.handler.bad_date = datetime.date(2014, 11, 20) + def get_date_for_index(index): + if index == 0: + return datetime.date(2014, 11, 15) + elif index == -1: + return datetime.date(2014, 11, 20) + new_data = Mock(get_date_for_index=get_date_for_index) + + self.handler._print_progress(new_data) + self.assertIn('from [2014-11-10, 2014-11-20] (10 days)', log[0]) + self.assertIn('to [2014-11-15, 2014-11-20] (5 days)', log[0]) + self.assertIn('2 steps left', log[0]) + +class TestInboundHandler(unittest.TestCase): + def setUp(self): + self.handler = InboundHandler(create_config('firefox', 'linux', 64)) + + def test_launcher_persist_prefix(self): + # define a repo and a mid_date + self.handler.fetch_config.set_inbound_branch('mybranch') + self.handler.set_build_data([{'timestamp': 123456789}]) + + prefix = self.handler.launcher_persist_prefix(0) + self.assertEqual(prefix, '123456789--mybranch--') + + @patch('mozregression.bisector.NightlyHandler.launcher_persist_prefix') + @patch('mozregression.bisector.BisectorHandler.start_launcher') + def test_start_launcher(self, start_launcher, launcher_persist_prefix): + start_launcher.return_value = 'my_launcher' + self.handler.set_build_data([{'timestamp': 123456789, 'revision':'12'}]) + launcher = self.handler.start_launcher(0) + + # base BisectorHandler.start_launcher has been called and is returned + start_launcher.assert_called_with(self.handler, 0) + self.assertEqual(launcher, 'my_launcher') + + def test_print_progress(self): + log = [] + self.handler._logger = Mock(info = log.append) + self.handler.set_build_data([ + {'revision':'12'}, + {'revision':'123'}, + {'revision':'1234'}, + {'revision':'12345'}, + ]) + new_data = [{'revision': '1234'}, {'revision': '12345'}] + + self.handler._print_progress(new_data) + self.assertIn('from [12, 12345] (4 revisions)', log[0]) + self.assertIn('to [1234, 12345] (2 revisions)', log[0]) + self.assertIn('1 steps left', log[0]) + +class MyBuildData(list): + def mid_point(self): + if len(self) < 3: + return 0 + return len(self) / 2 + + def ensure_limits(self): + pass + + def __getslice__(self, smin, smax): + return MyBuildData(list.__getslice__(self, smin, smax)) + +class TestBisector(unittest.TestCase): + def setUp(self): + self.handler = Mock() + self.bisector = Bisector(self.handler) + + @patch('__builtin__.raw_input') + def test_get_verdict(self, raw_input): + raw_input.return_value = 'g' + verdict = self.bisector.get_verdict() + self.assertEqual(verdict, 'g') + + def test_bisect_no_data(self): + build_data = MyBuildData() + result = self.bisector.bisect(build_data) + # test that handler methods where called + self.handler.set_build_data.assert_called_with(build_data) + self.handler.no_data.assert_called() + # check return code + self.assertEqual(result, Bisector.NO_DATA) + + def test_bisect_finished(self): + build_data = MyBuildData([1]) + result = self.bisector.bisect(build_data) + # test that handler methods where called + self.handler.set_build_data.assert_called_with(build_data) + self.handler.finished.assert_called() + # check return code + self.assertEqual(result, Bisector.FINISHED) + + def do_bisect(self, build_data, verdicts): + iter_verdict = iter(verdicts) + self.bisector.get_verdict = Mock(side_effect=iter_verdict.next) + launcher = Mock() + self.handler.start_launcher.return_value = launcher + result = self.bisector.bisect(build_data) + return { + 'result': result, + 'launcher': launcher, + } + + def test_bisect_case1(self): + test_result = self.do_bisect(MyBuildData([1, 2, 3, 4, 5]), ['g', 'b']) + # check that set_build_data was called + self.handler.set_build_data.assert_has_calls([ + # first call + call(MyBuildData([1, 2, 3, 4, 5])), + # we answered good + call(MyBuildData([3, 4, 5])), + # we answered bad + call(MyBuildData([3, 4])), + ]) + # ensure that the launcher was stopped + test_result['launcher'].stop.assert_called() + # ensure that we called the handler's methods + self.handler.build_good.assert_called_with(2, MyBuildData([3, 4, 5])) + self.handler.build_bad.assert_called_with(1, MyBuildData([3, 4])) + self.handler.build_data.ensure_limits.assert_called() + # bisection is finished + self.assertEqual(test_result['result'], Bisector.FINISHED) + + def test_bisect_case2(self): + test_result = self.do_bisect(MyBuildData([1, 2, 3]), ['r', 's']) + # check that set_build_data was called + self.handler.set_build_data.assert_has_calls([ + # first call + # this should be call(MyBuildData([1, 2, 3])), + # but as the code delete the index in place when we skip + # (with a del statement) our build_data is impacted. + # well, we just have to know that for the test. + call(MyBuildData([1, 3])), + # we asked for a retry (same comment as above) + call(MyBuildData([1, 3])), + # we skipped one + call(MyBuildData([1, 3])), + ]) + # ensure that the launcher was stopped + test_result['launcher'].stop.assert_called() + # ensure that we called the handler's methods + self.handler.build_retry.assert_called_with(1) + self.handler.build_skip.assert_called_with(1) + self.handler.build_data.ensure_limits.assert_called() + # bisection is finished + self.assertEqual(test_result['result'], Bisector.FINISHED) + + def test_bisect_user_exit(self): + test_result = self.do_bisect(MyBuildData(range(20)), ['e']) + # check that set_build_data was called + self.handler.set_build_data.assert_has_calls([call(MyBuildData(range(20)))]) + # ensure that we called the handler's method + self.handler.user_exit.assert_called_with(10) + # user exit + self.assertEqual(test_result['result'], Bisector.USER_EXIT) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py new file mode 100644 index 00000000..b55365bc --- /dev/null +++ b/tests/unit/test_main.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest +from mock import patch, Mock +import datetime + +from mozregression import main +from mozregression import errors + +class TestMainCli(unittest.TestCase): + def setUp(self): + self.runner = Mock() + + @patch('mozlog.structured.commandline.setup_logging') + @patch('mozregression.main.set_http_cache_session') + @patch('mozregression.limitedfilecache.get_cache') + @patch('mozregression.main.BisectRunner') + def do_cli(self, argv, BisectRunner, get_cache, set_http_cache_session, + setup_logging): + def create_runner(fetch_config, options): + self.runner.fetch_config = fetch_config + self.runner.options = options + return self.runner + BisectRunner.side_effect = create_runner + try: + main.cli(argv) + except SystemExit as exc: + return exc.code + else: + self.fail('mozregression.main.cli did not call sys.exit') + + @patch('sys.stdout') + def test_get_usage(self, stdout): + output = [] + stdout.write.side_effect = output.append + + exitcode = self.do_cli(['-h']) + output = ''.join(output) + self.assertEqual(exitcode, 0) + self.assertIn('usage:', output) + + def test_without_args(self): + self.runner.bisect_nightlies.return_value = 0 + exitcode = self.do_cli([]) + # application is by default firefox + self.assertEqual(self.runner.fetch_config.app_name, 'firefox') + # bisect_nightlies has been called + self.runner.bisect_nightlies.assert_called_with(datetime.date(2009, 1, 1), + datetime.date.today()) + # we exited with the return value of bisect_nightlies + self.assertEquals(exitcode, 0) + + def test_basic_inbound(self): + self.runner.bisect_inbound.return_value = 0 + exitcode = self.do_cli(['--inbound', '--good-rev=1', '--bad-rev=5']) + # application is by default firefox + self.assertEqual(self.runner.fetch_config.app_name, 'firefox') + # bisect_inbound has been called + self.runner.bisect_inbound.assert_called_with('1', '5') + # we exited with the return value of bisect_inbound + self.assertEquals(exitcode, 0) + + def test_inbound_revs_must_be_given(self): + argslist = [ + ['--inbound'], + ['--inbound', '--good-rev=1'], + ['--inbound', '--bad-rev=5'], + ] + for args in argslist: + exitcode = self.do_cli(args) + self.assertIn('--good-rev and --bad-rev must be set', exitcode) + + @patch('mozregression.fetch_configs.FirefoxConfig.is_inbound') + def test_inbound_must_be_doable(self, is_inbound): + is_inbound.return_value = False + exitcode = self.do_cli(['--inbound', '--good-rev=1', '--bad-rev=5']) + self.assertIn('Unable to bissect inbound', exitcode) + + @patch('mozregression.main.formatted_valid_release_dates') + def test_list_releases(self, formatted_valid_release_dates): + exitcode = self.do_cli(['--list-releases']) + formatted_valid_release_dates.assert_called_once() + self.assertIn(exitcode, (0, None)) + + def test_bad_date_and_bad_release_are_incompatible(self): + exitcode = self.do_cli(['--bad=2014-11-10', '--bad-release=1']) + self.assertIn('incompatible', exitcode) + + def test_bad_release_invalid(self): + exitcode = self.do_cli(['--bad-release=-1']) + self.assertIn('Unable to find a matching date for release', exitcode) + + def test_good_date_and_good_release_are_incompatible(self): + exitcode = self.do_cli(['--good=2014-11-10', '--good-release=1']) + self.assertIn('incompatible', exitcode) + + def test_good_release_invalid(self): + exitcode = self.do_cli(['--good-release=-1']) + self.assertIn('Unable to find a matching date for release', exitcode) + + def test_handle_keyboard_interrupt(self): + # KeyboardInterrupt are handled with a nice error message. + self.runner.bisect_nightlies.side_effect = KeyboardInterrupt + exitcode = self.do_cli([]) + self.assertIn('Interrupted', exitcode) + + def test_handle_mozregression_errors(self): + # Any MozRegressionError subclass is handled with a nice error message + self.runner.bisect_nightlies.side_effect = errors.MozRegressionError('my error') + exitcode = self.do_cli([]) + self.assertIn('my error', exitcode) + + def test_handle_other_errors(self): + # other exceptions are just thrown as usual, so we have complete stacktrace + self.runner.bisect_nightlies.side_effect = NameError + self.assertRaises(NameError, self.do_cli, [])