create bisector and main modules from regression

.coveragerc

@@ -6,3 +6,7 @@ source = mozregression
 exclude_lines =
     # Don't complain if tests don't hit defensive assertion code
     raise NotImplementedError
+    # pass instructions does need to be tested
+    pass
+    # Don't complain if non-runnable code isn't run
+    if __name__ == .__main__.:

mozregression/bisector.py

@@ -0,0 +1,429 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import math
+import sys
+import datetime
+from mozlog.structured import get_default_logger
+from mozregression.launchers import create_launcher
+from mozregression.utils import format_date
+from mozregression.build_data import NightlyBuildData
+from mozregression.inboundfinder import BuildsFinder
+
+def compute_steps_left(steps):
+    if steps <= 1:
+        return 0
+    return math.trunc(math.log(steps, 2))
+
+class BisectorHandler(object):
+    """
+    React to events of a :class:`Bisector`. This is intended to be subclassed.
+
+    A BisectorHandler keep the state of the current bisection process.
+    """
+    build_type = 'unknown'
+
+    def __init__(self, fetch_config, persist=None, launcher_kwargs=None):
+        self.fetch_config = fetch_config
+        self.persist = persist
+        self.launcher_kwargs = launcher_kwargs or {}
+        self.found_repo = None
+        self.build_data = None
+        self.last_good_revision = None
+        self.first_bad_revision = None
+        self.found_repo = None
+        self.app_info = None
+        self._logger = get_default_logger('Bisector')
+
+    def set_build_data(self, build_data):
+        """
+        Save a reference of the :class:`mozregression.build_data.BuildData`
+        instance.
+
+        This is called by the bisector before each step of the bisection
+        process.
+        """
+        self.build_data = build_data
+
+    def launcher_persist_prefix(self, index):
+        """
+        Returns an appropriate prefix for a downloaded build.
+        """
+        raise NotImplementedError
+
+    def _print_progress(self, new_data):
+        """
+        Log the current state of the bisection process.
+        """
+        raise NotImplementedError
+
+    def start_launcher(self, index):
+        """
+        Create and returns a :class:`mozregression.launchers.Launcher`
+        that has been started.
+
+        This is called by the bisector when a build needs to be tested.
+        """
+        build_url = self.build_data[index]['build_url']
+        launcher = create_launcher(self.fetch_config.app_name,
+                                   build_url,
+                                   persist=self.persist,
+                                   persist_prefix=self.launcher_persist_prefix(index))
+        launcher.start(**self.launcher_kwargs)
+        self.app_info = launcher.get_app_info()
+        self.found_repo = self.app_info['application_repository']
+        return launcher
+
+    def get_pushlog_url(self):
+        return "%s/pushloghtml?fromchange=%s&tochange=%s" % (
+            self.found_repo, self.last_good_revision, self.first_bad_revision)
+
+    def _get_str_range(self):
+        return ('revision: %s' % self.last_good_revision,
+                'revision: %s' % self.first_bad_revision)
+
+    def print_range(self):
+        """
+        Log the state of the current state of the bisection process, with an
+        appropriate pushlog url.
+        """
+        good, bad = self._get_str_range()
+        self._logger.info("Last good %s" % good)
+        self._logger.info("First bad %s" % bad)
+        self._logger.info("Pushlog:\n%s\n" % self.get_pushlog_url())
+
+    def build_good(self, mid, new_data):
+        """
+        Called by the Bisector when a build is good.
+        """
+        self.last_good_revision = self.app_info['application_changeset']
+        if len(new_data) > 1:
+            self._print_progress(new_data)
+
+    def build_bad(self, mid, new_data):
+        """
+        Called by the Bisector when a build is bad.
+        """
+        self.first_bad_revision = self.app_info['application_changeset']
+        if len(new_data) > 1:
+            self._print_progress(new_data)
+
+    def build_retry(self, mid):
+        pass
+
+    def build_skip(self, mid):
+        pass
+
+    def no_data(self):
+        pass
+
+    def finished(self):
+        pass
+
+    def user_exit(self, mid):
+        pass
+
+class NightlyHandler(BisectorHandler):
+    build_type = 'nightly'
+    good_date = None
+    bad_date = None
+    mid_date = None
+
+    def launcher_persist_prefix(self, index):
+        return '%s--%s--' % (self.mid_date,
+                             self.fetch_config.get_nightly_repo(self.mid_date))
+
+    def start_launcher(self, mid):
+        # register dates
+        self.good_date = self.build_data.get_date_for_index(0)
+        self.bad_date = self.build_data.get_date_for_index(-1)
+        self.mid_date = self.build_data.get_date_for_index(mid)
+        self._logger.info("Running nightly for %s" % self.mid_date)
+        return BisectorHandler.start_launcher(self, mid)
+
+    def _print_progress(self, new_data):
+        next_good_date = new_data.get_date_for_index(0)
+        next_bad_date = new_data.get_date_for_index(-1)
+        next_days_range = (next_bad_date - next_good_date).days
+        self._logger.info("Narrowed nightly regression window from"
+                          " [%s, %s] (%d days) to [%s, %s] (%d days)"
+                          " (~%d steps left)"
+                          % (format_date(self.good_date),
+                             format_date(self.bad_date),
+                             (self.bad_date - self.good_date).days,
+                             format_date(next_good_date),
+                             format_date(next_bad_date),
+                             next_days_range,
+                             compute_steps_left(next_days_range)))
+
+    def ensure_metadata(self):
+        if not self.last_good_revision:
+            date = self.build_data.get_date_for_index(0)
+            infos = self.build_data.get_build_infos_for_date(date)
+            self.found_repo = infos['repository']
+            self.last_good_revision = infos['changeset']
+
+        if not self.first_bad_revision:
+            date = self.build_data.get_date_for_index(-1)
+            infos = self.build_data.get_build_infos_for_date(date)
+            self.found_repo = infos['repository']
+            self.first_bad_revision = infos['changeset']
+
+    def _get_str_range(self):
+        good, bad = BisectorHandler._get_str_range(self)
+        good += ' (%s)' % self.good_date
+        bad += ' (%s)' % self.bad_date
+        return good, bad
+
+class InboundHandler(BisectorHandler):
+    build_type = 'inbound'
+
+    def launcher_persist_prefix(self, index):
+        return '%s--%s--' % (self.build_data[index]['timestamp'],
+                             self.fetch_config.inbound_branch)
+
+    def start_launcher(self, mid):
+        data = self.build_data[mid]
+        self._logger.info("Testing inbound build with timestamp %s,"
+                          " revision %s"
+                          % (data['timestamp'],
+                             data['revision']))
+        return BisectorHandler.start_launcher(self, mid)
+
+    def _print_progress(self, new_data):
+        self._logger.info("Narrowed inbound regression window from [%s, %s]"
+                          " (%d revisions) to [%s, %s] (%d revisions)"
+                          " (~%d steps left)"
+                          % (self.build_data[0]['revision'],
+                             self.build_data[-1]['revision'],
+                             len(self.build_data),
+                             new_data[0]['revision'],
+                             new_data[-1]['revision'],
+                             len(new_data),
+                             compute_steps_left(len(new_data))))
+
+class Bisector(object):
+    """
+    Handle the logic of the bisection process, and report events to a given
+    :class:`BisectorHandler`.
+    """
+    NO_DATA = 1
+    FINISHED = 2
+    USER_EXIT = 3
+
+    def __init__(self, handler):
+        self.handler = handler
+
+    def get_verdict(self, offer_skip=True):
+        options = ['good', 'bad']
+        if offer_skip:
+            options.append('skip')
+        options += ['retry', 'exit']
+        # allow user to just type one letter
+        allowed_inputs = options + [o[0] for o in options]
+        # format options to nice printing
+        formatted_options = (', '.join(["'%s'" % o for o in options[:-1]])
+                             + " or '%s'" % options[-1])
+        verdict = ""
+        while verdict not in allowed_inputs:
+            verdict = raw_input("Was this %s build good, bad, or broken?"
+                                " (type %s and press Enter): "
+                                % (self.handler.build_type,
+                                   formatted_options))
+
+        # shorten verdict to one character for processing...
+        return verdict[0]
+
+    def bisect(self, build_data):
+        """
+        Starts a bisection for a :class:`mozregression.build_data.BuildData`.
+        """
+        while True:
+            self.handler.set_build_data(build_data)
+            mid = build_data.mid_point()
+
+            if len(build_data) == 0:
+                self.handler.no_data()
+                return self.NO_DATA
+
+            if mid == 0:
+                self.handler.finished()
+                return self.FINISHED
+
+            launcher = self.handler.start_launcher(mid)
+            verdict = self.get_verdict()
+            launcher.stop()
+
+            if verdict == 'g':
+                # if build is good, we have to split from
+                # [G, ?, ?, G, ?, B]
+                # to
+                #          [G, ?, B]
+                build_data = build_data[mid:]
+                build_data.ensure_limits()
+                self.handler.build_good(mid, build_data)
+            elif verdict == 'b':
+                # if build is bad, we have to split from
+                # [G, ?, ?, B, ?, B]
+                # to
+                # [G, ?, ?, B]
+                build_data = build_data[:mid+1]
+                build_data.ensure_limits()
+                self.handler.build_bad(mid, build_data)
+            elif verdict == 'r':
+                self.handler.build_retry(mid)
+            elif verdict == 's':
+                self.handler.build_skip(mid)
+                del build_data[mid]
+            else:
+                # user exit
+                self.handler.user_exit(mid)
+                return self.USER_EXIT
+
+class BisectRunner(object):
+    def __init__(self, fetch_config, options):
+        self.fetch_config = fetch_config
+        self.options = options
+        self.launcher_kwargs = dict(
+            addons=options.addons,
+            profile=options.profile,
+            cmdargs=options.cmdargs,
+        )
+        self._logger = get_default_logger('Bisector')
+
+    def bisect_nightlies(self, good_date, bad_date):
+        build_data = NightlyBuildData(good_date, bad_date, self.fetch_config)
+        handler = NightlyHandler(self.fetch_config,
+                                 persist=self.options.persist,
+                                 launcher_kwargs=self.launcher_kwargs)
+        bisector = Bisector(handler)
+        result = bisector.bisect(build_data)
+        if result == Bisector.FINISHED:
+            self._logger.info("Got as far as we can go bisecting nightlies...")
+            self._logger.info("Ensuring we have enough metadata to get a pushlog...")
+            handler.ensure_metadata()
+            handler.print_range()
+            if self.fetch_config.can_go_inbound():
+                self._logger.info("... attempting to bisect inbound builds"
+                                  " (starting from previous week, to make"
+                                  " sure no inbound revision is missed)")
+                infos = {}
+                days = 6
+                while not 'changeset' in infos:
+                    days += 1
+                    prev_date = good_date - datetime.timedelta(days=days)
+                    infos = handler.build_data.get_build_infos_for_date(prev_date)
+                if days > 7:
+                    self._logger.info("At least one build folder was"
+                                      " invalid, we have to start from"
+                                      " %d days ago." % days)
+                return self.bisect_inbound(infos['changeset'],
+                                           handler.first_bad_revision)
+            else:
+                message = ("Can not bissect inbound for application `%s`"
+                           % self.fetch_config.app_name)
+                if self.fetch_config.is_inbound():
+                    # the config is able to bissect inbound but not
+                    # for this repo.
+                    message += (" because the repo `%s` was specified"
+                                % self.options.repo)
+                self._logger.info(message + '.')
+        elif result == Bisector.USER_EXIT:
+            self._logger.info('Newest known good nightly: %s'
+                              % handler.good_date)
+            self._logger.info('Oldest known bad nightly: %s'
+                              % handler.bad_date)
+            self.print_resume_info(handler)
+        else:
+            # NO_DATA
+            self._logger.info("Unable to get valid builds within the given"
+                              " range. You should try to launch mozregression"
+                              " again with a larger date range.")
+            return 1
+        return 0
+
+    def bisect_inbound(self, good_rev, bad_rev):
+        self._logger.info("Getting inbound builds between %s and %s"
+                          % (good_rev, bad_rev))
+        # anything within twelve hours is potentially within the range
+        # (should be a tighter but some older builds have wrong timestamps,
+        # see https://bugzilla.mozilla.org/show_bug.cgi?id=1018907 ...
+        # we can change this at some point in the future, after those builds
+        # expire)
+        build_finder = BuildsFinder(self.fetch_config)
+        inbound_data = build_finder.get_build_infos(good_rev,
+                                                    bad_rev,
+                                                    range=60*60*12)
+        handler = InboundHandler(self.fetch_config,
+                                 persist=self.options.persist,
+                                 launcher_kwargs=self.launcher_kwargs)
+        handler.last_good_revision = good_rev
+        handler.first_bad_revision = bad_rev
+        bisector = Bisector(handler)
+        result = bisector.bisect(inbound_data)
+        if result == Bisector.FINISHED:
+            self._logger.info("Oh noes, no (more) inbound revisions :(")
+            handler.print_range()
+            self.offer_build(handler.last_good_revision,
+                             handler.first_bad_revision)
+        elif result == Bisector.USER_EXIT:
+            self._logger.info('Newest known good inbound revision: %s'
+                              % handler.last_good_revision)
+            self._logger.info('Oldest known bad inbound revision: %s'
+                              % handler.first_bad_revision)
+
+            self.print_resume_info(handler)
+        else:
+            # NO_DATA
+            self._logger.info("No inbound data found.")
+            return 1
+        return 0
+
+    def print_resume_info(self, handler):
+        if isinstance(handler, NightlyHandler):
+            info = '--good=%s --bad=%s' % (handler.good_date, handler.bad_date)
+        else:
+            info = ('--inbound --good-rev=%s --bad-rev=%s'
+                    % (handler.last_good_revision, handler.first_bad_revision))
+        options = self.options
+        info += ' --app=%s' % options.app
+        if len(options.addons) > 0:
+            info += ' --addons=%s' % ",".join(options.addons)
+        if options.profile is not None:
+            info += ' --profile=%s' % options.profile
+        if options.inbound_branch is not None:
+            info += ' --inbound-branch=%s' % options.inbound_branch
+        info += ' --bits=%s' % options.bits
+        if options.persist is not None:
+            info += ' --persist=%s' % options.persist
+
+        self._logger.info('To resume, run:')
+        self._logger.info('mozregression %s' % info)
+
+    def find_regression_chset(self, last_good_revision, first_bad_revision):
+        # Uses mozcommitbuilder to bisect on changesets
+        # Only needed if they want to bisect, so we'll put the dependency here.
+        from mozcommitbuilder import builder
+        commit_builder = builder.Builder()
+
+        self._logger.info(" Narrowed changeset range from %s to %s"
+                          % (last_good_revision, first_bad_revision))
+
+        self._logger.info("Time to do some bisecting and building!")
+        commit_builder.bisect(last_good_revision, first_bad_revision)
+        quit()
+
+    def offer_build(self, last_good_revision, first_bad_revision):
+        verdict = raw_input("do you want to bisect further by fetching"
+                            " the repository and building? (y or n) ")
+        if verdict != "y":
+            sys.exit()
+
+        if self.fetch_config.app_name == "firefox":
+            self.find_regression_chset(last_good_revision, first_bad_revision)
+        else:
+            sys.exit("Bisection on anything other than firefox is not"
+                     " currently supported.")

mozregression/main.py

@@ -0,0 +1,200 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import mozinfo
+import datetime
+import sys
+from argparse import ArgumentParser
+from mozlog.structured import commandline, get_default_logger
+
+from mozregression import errors
+from mozregression import limitedfilecache
+from mozregression import __version__
+from mozregression.utils import (parse_date, date_of_release, format_date,
+                                 parse_bits, set_http_cache_session,
+                                 one_gigabyte, formatted_valid_release_dates)
+from mozregression.fetch_configs import create_config
+from mozregression.bisector import BisectRunner
+
+def parse_args(argv=None):
+    usage = ("\n"
+             " %(prog)s [OPTIONS]"
+             " [[--bad BAD_DATE]|[--bad-release BAD_RELEASE]]"
+             " [[--good GOOD_DATE]|[--good-release GOOD_RELEASE]]"
+             "\n"
+             " %(prog)s [OPTIONS]"
+             " --inbound --bad-rev BAD_REV --good-rev GOOD_REV")
+
+    parser = ArgumentParser(usage=usage)
+    parser.add_argument("--version", action="version", version=__version__,
+                        help=("print the mozregression version number and"
+                              " exits."))
+
+    parser.add_argument("-b", "--bad",
+                        metavar="YYYY-MM-DD",
+                        dest="bad_date",
+                        help="first known bad nightly build, default is today.")
+
+    parser.add_argument("-g", "--good",
+                        metavar="YYYY-MM-DD",
+                        dest="good_date",
+                        help="last known good nightly build.")
+
+    parser.add_argument("--list-releases",
+                        action="store_true",
+                        help="list all known releases and exit")
+
+    parser.add_argument("--bad-release",
+                        type=int,
+                        help=("first known bad nightly build. This option is"
+                              " incompatible with --bad."))
+
+    parser.add_argument("--good-release",
+                        type=int,
+                        help=("last known good nightly build. This option is"
+                              " incompatible with --good."))
+
+    parser.add_argument("--inbound",
+                        action="store_true",
+                        help=("use inbound instead of nightlies (use --good-rev"
+                              " and --bad-rev options."))
+
+    parser.add_argument("--bad-rev", dest="first_bad_revision",
+                        help="first known bad revision (use with --inbound).")
+
+    parser.add_argument("--good-rev", dest="last_good_revision",
+                        help="last known good revision (use with --inbound).")
+
+    parser.add_argument("-e", "--addon",
+                        dest="addons",
+                        action='append',
+                        default=[],
+                        metavar="PATH1",
+                        help="an addon to install; repeat for multiple addons.")
+
+    parser.add_argument("-p", "--profile",
+                        metavar="PATH",
+                        help="profile to use with nightlies.")
+
+    parser.add_argument("-a", "--arg",
+                        dest="cmdargs",
+                        action='append',
+                        default=[],
+                        metavar="ARG1",
+                        help=("a command-line argument to pass to the"
+                              " application; repeat for multiple arguments."))
+
+    parser.add_argument("-n", "--app",
+                        choices=('firefox', 'fennec', 'thunderbird', 'b2g'),
+                        default="firefox",
+                        help="application name. Default: %(default)s.")
+
+    parser.add_argument("--repo",
+                        metavar="[mozilla-aurora|mozilla-beta|...]",
+                        help="repository name used for nightly hunting.")
+
+    parser.add_argument("--inbound-branch",
+                        metavar="[b2g-inbound|fx-team|...]",
+                        help="inbound branch name on ftp.mozilla.org.")
+
+    parser.add_argument("--bits",
+                        choices=("32", "64"),
+                        default=mozinfo.bits,
+                        help=("force 32 or 64 bit version (only applies to"
+                              " x86_64 boxes). Default: %(default)s bits."))
+
+    parser.add_argument("--persist",
+                        help=("the directory in which downloaded files are"
+                              " to persist."))
+
+    parser.add_argument("--http-cache-dir",
+                        help=("the directory for caching http requests."
+                              " If not set there will be an in-memory cache"
+                              " used."))
+
+    commandline.add_logging_group(parser)
+    options = parser.parse_args(argv)
+    options.bits = parse_bits(options.bits)
+    return options
+
+
+def cli(argv=None):
+    default_bad_date = str(datetime.date.today())
+    default_good_date = "2009-01-01"
+    options = parse_args(argv)
+    logger = commandline.setup_logging("mozregression", options, {"mach": sys.stdout})
+
+    if options.list_releases:
+        print(formatted_valid_release_dates())
+        sys.exit()
+
+    cache_session = limitedfilecache.get_cache(
+        options.http_cache_dir, one_gigabyte,
+        logger=get_default_logger('Limited File Cache'))
+    set_http_cache_session(cache_session)
+
+    fetch_config = create_config(options.app, mozinfo.os, options.bits)
+    runner = BisectRunner(fetch_config, options)
+
+    if fetch_config.is_inbound():
+        # this can be useful for both inbound and nightly, because we
+        # can go to inbound from nightly.
+        fetch_config.set_inbound_branch(options.inbound_branch)
+
+    if options.inbound:
+        if not fetch_config.is_inbound():
+            sys.exit('Unable to bissect inbound for `%s`' % fetch_config.app_name)
+        if not options.last_good_revision or not options.first_bad_revision:
+            sys.exit("If bisecting inbound, both --good-rev and --bad-rev"
+                     " must be set")
+        app = lambda: runner.bisect_inbound(options.last_good_revision,
+                                            options.first_bad_revision)
+    else:
+        # TODO: currently every fetch_config is nightly aware. Shoud we test
+        # for this to be sure here ?
+        fetch_config.set_nightly_repo(options.repo)
+        if not options.bad_release and not options.bad_date:
+            options.bad_date = default_bad_date
+            logger.info("No 'bad' date specified, using %s" % options.bad_date)
+        elif options.bad_release and options.bad_date:
+            sys.exit("Options '--bad_release' and '--bad_date' are"
+                     " incompatible.")
+        elif options.bad_release:
+            options.bad_date = date_of_release(options.bad_release)
+            if options.bad_date is None:
+                sys.exit("Unable to find a matching date for release "
+                         + str(options.bad_release)
+                         + "\n" + formatted_valid_release_dates())
+            logger.info("Using 'bad' date %s for release %s"
+                        % (options.bad_date, options.bad_release))
+        if not options.good_release and not options.good_date:
+            options.good_date = default_good_date
+            logger.info("No 'good' date specified, using %s"
+                        % options.good_date)
+        elif options.good_release and options.good_date:
+            sys.exit("Options '--good_release' and '--good_date'"
+                     " are incompatible.")
+        elif options.good_release:
+            options.good_date = date_of_release(options.good_release)
+            if options.good_date is None:
+                sys.exit("Unable to find a matching date for release "
+                         + str(options.good_release)
+                         + "\n" + formatted_valid_release_dates())
+            logger.info("Using 'good' date %s for release %s"
+                        % (options.good_date, options.good_release))
+
+        app = lambda: runner.bisect_nightlies(parse_date(options.good_date),
+                                              parse_date(options.bad_date))
+    try:
+        sys.exit(app())
+    except KeyboardInterrupt:
+        sys.exit("\nInterrupted.")
+    except errors.MozRegressionError as exc:
+        sys.exit(str(exc))
+
+
+if __name__ == "__main__":
+    cli()

tests/unit/test_bisector.py

@@ -0,0 +1,277 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import unittest
+from mock import patch, Mock, call
+import datetime
+
+from mozregression.fetch_configs import create_config
+from mozregression.bisector import (BisectorHandler, NightlyHandler,
+                                    InboundHandler, Bisector)
+
+class TestBisectorHandler(unittest.TestCase):
+    def setUp(self):
+        self.handler = BisectorHandler(create_config('firefox', 'linux', 64))
+        self.handler.set_build_data([
+            {'build_url': 'http://build_url_0'}
+        ])
+
+    @patch('mozregression.bisector.BisectorHandler.launcher_persist_prefix')
+    @patch('mozregression.bisector.create_launcher')
+    def test_start_launcher(self, create_launcher, launcher_persist_prefix):
+        app_info = {'application_repository': 'something'}
+        launcher = Mock(get_app_info=Mock(return_value=app_info))
+        create_launcher.return_value = launcher
+        launcher_persist_prefix.return_value = 'something-'
+
+        result = self.handler.start_launcher(0)
+        # create launcher is well called
+        create_launcher.assert_called_with('firefox', 'http://build_url_0',
+                                           persist=None,
+                                           persist_prefix='something-')
+        # launcher is started
+        launcher.start.assert_called_with(**self.handler.launcher_kwargs)
+        # app_info and found_repo are set
+        self.assertEqual(self.handler.app_info, app_info)
+        self.assertEqual(self.handler.found_repo, 'something')
+        # and launvher instance is returned
+        self.assertEqual(result, launcher)
+
+    def test_get_pushlog_url(self):
+        self.handler.found_repo = 'https://hg.mozilla.repo'
+        self.handler.last_good_revision = '2'
+        self.handler.first_bad_revision = '6'
+        self.assertEqual(self.handler.get_pushlog_url(),
+                         "https://hg.mozilla.repo/pushloghtml?fromchange=2&tochange=6")
+
+    def test_print_range(self):
+        self.handler.found_repo = 'https://hg.mozilla.repo'
+        self.handler.last_good_revision = '2'
+        self.handler.first_bad_revision = '6'
+        log = []
+        self.handler._logger = Mock(info = log.append)
+
+        self.handler.print_range()
+        self.assertEqual(log[0], "Last good revision: 2")
+        self.assertEqual(log[1], "First bad revision: 6")
+        self.assertIn(self.handler.get_pushlog_url(), log[2])
+
+    @patch('mozregression.bisector.BisectorHandler._print_progress')
+    def test_build_good(self, _print_progress):
+        self.handler.app_info = {"application_changeset": '123'}
+        # call build_good with no new data points
+        self.handler.build_good(0, [])
+        self.assertEqual(self.handler.last_good_revision, '123')
+        _print_progress.assert_not_called()
+        # with at least two, _print_progress will be called
+        self.handler.build_good(0, [1, 2])
+        _print_progress.assert_called_with([1, 2])
+
+    @patch('mozregression.bisector.BisectorHandler._print_progress')
+    def test_build_bad(self, _print_progress):
+        self.handler.app_info = {"application_changeset": '123'}
+        # call build_bad with no new data points
+        self.handler.build_bad(0, [])
+        self.assertEqual(self.handler.first_bad_revision, '123')
+        _print_progress.assert_not_called()
+        # with at least two, _print_progress will be called
+        self.handler.build_bad(0, [1, 2])
+        _print_progress.assert_called_with([1, 2])
+
+class TestNightlyHandler(unittest.TestCase):
+    def setUp(self):
+        self.handler = NightlyHandler(create_config('firefox', 'linux', 64))
+
+    def test_launcher_persist_prefix(self):
+        # define a repo and a mid_date
+        self.handler.fetch_config.set_nightly_repo('myrepo')
+        self.handler.mid_date = datetime.date(2014, 11, 10)
+
+        prefix = self.handler.launcher_persist_prefix(1)
+        self.assertEqual(prefix, '2014-11-10--myrepo--')
+
+    @patch('mozregression.bisector.NightlyHandler.launcher_persist_prefix')
+    @patch('mozregression.bisector.BisectorHandler.start_launcher')
+    def test_start_launcher(self, start_launcher, launcher_persist_prefix):
+        get_date_for_index = Mock(side_effect=lambda i: i)
+        self.handler.build_data = Mock(get_date_for_index=get_date_for_index)
+        start_launcher.return_value = 'my_launcher'
+
+        launcher = self.handler.start_launcher(3)
+        # check we have called get_date_for_index
+        get_date_for_index.assert_has_calls([call(0), call(3), call(-1)],
+                                            any_order=True)
+        #  dates are well set
+        self.assertEqual(self.handler.mid_date, 3)
+        self.assertEqual(self.handler.good_date, 0)
+        self.assertEqual(self.handler.bad_date, -1)
+        # base BisectorHandler.start_launcher has been called and is returned
+        start_launcher.assert_called_with(self.handler, 3)
+        self.assertEqual(launcher, 'my_launcher')
+
+    def test_print_progress(self):
+        log = []
+        self.handler._logger = Mock(info = log.append)
+        self.handler.good_date = datetime.date(2014, 11, 10)
+        self.handler.bad_date = datetime.date(2014, 11, 20)
+        def get_date_for_index(index):
+            if index == 0:
+                return datetime.date(2014, 11, 15)
+            elif index == -1:
+                return datetime.date(2014, 11, 20)
+        new_data = Mock(get_date_for_index=get_date_for_index)
+
+        self.handler._print_progress(new_data)
+        self.assertIn('from [2014-11-10, 2014-11-20] (10 days)', log[0])
+        self.assertIn('to [2014-11-15, 2014-11-20] (5 days)', log[0])
+        self.assertIn('2 steps left', log[0])
+
+class TestInboundHandler(unittest.TestCase):
+    def setUp(self):
+        self.handler = InboundHandler(create_config('firefox', 'linux', 64))
+
+    def test_launcher_persist_prefix(self):
+        # define a repo and a mid_date
+        self.handler.fetch_config.set_inbound_branch('mybranch')
+        self.handler.set_build_data([{'timestamp': 123456789}])
+
+        prefix = self.handler.launcher_persist_prefix(0)
+        self.assertEqual(prefix, '123456789--mybranch--')
+
+    @patch('mozregression.bisector.NightlyHandler.launcher_persist_prefix')
+    @patch('mozregression.bisector.BisectorHandler.start_launcher')
+    def test_start_launcher(self, start_launcher, launcher_persist_prefix):
+        start_launcher.return_value = 'my_launcher'
+        self.handler.set_build_data([{'timestamp': 123456789, 'revision':'12'}])
+        launcher = self.handler.start_launcher(0)
+
+        # base BisectorHandler.start_launcher has been called and is returned
+        start_launcher.assert_called_with(self.handler, 0)
+        self.assertEqual(launcher, 'my_launcher')
+
+    def test_print_progress(self):
+        log = []
+        self.handler._logger = Mock(info = log.append)
+        self.handler.set_build_data([
+            {'revision':'12'},
+            {'revision':'123'},
+            {'revision':'1234'},
+            {'revision':'12345'},
+        ])
+        new_data = [{'revision': '1234'}, {'revision': '12345'}]
+
+        self.handler._print_progress(new_data)
+        self.assertIn('from [12, 12345] (4 revisions)', log[0])
+        self.assertIn('to [1234, 12345] (2 revisions)', log[0])
+        self.assertIn('1 steps left', log[0])
+
+class MyBuildData(list):
+    def mid_point(self):
+        if len(self) < 3:
+            return 0
+        return len(self) / 2
+
+    def ensure_limits(self):
+        pass
+
+    def __getslice__(self, smin, smax):
+        return MyBuildData(list.__getslice__(self, smin, smax))
+
+class TestBisector(unittest.TestCase):
+    def setUp(self):
+        self.handler = Mock()
+        self.bisector = Bisector(self.handler)
+
+    @patch('__builtin__.raw_input')
+    def test_get_verdict(self, raw_input):
+        raw_input.return_value = 'g'
+        verdict = self.bisector.get_verdict()
+        self.assertEqual(verdict, 'g')
+
+    def test_bisect_no_data(self):
+        build_data = MyBuildData()
+        result = self.bisector.bisect(build_data)
+        # test that handler methods where called
+        self.handler.set_build_data.assert_called_with(build_data)
+        self.handler.no_data.assert_called()
+        # check return code
+        self.assertEqual(result, Bisector.NO_DATA)
+
+    def test_bisect_finished(self):
+        build_data = MyBuildData([1])
+        result = self.bisector.bisect(build_data)
+        # test that handler methods where called
+        self.handler.set_build_data.assert_called_with(build_data)
+        self.handler.finished.assert_called()
+        # check return code
+        self.assertEqual(result, Bisector.FINISHED)
+
+    def do_bisect(self, build_data, verdicts):
+        iter_verdict = iter(verdicts)
+        self.bisector.get_verdict = Mock(side_effect=iter_verdict.next)
+        launcher = Mock()
+        self.handler.start_launcher.return_value = launcher
+        result = self.bisector.bisect(build_data)
+        return {
+            'result': result,
+            'launcher': launcher,
+        }
+
+    def test_bisect_case1(self):
+        test_result = self.do_bisect(MyBuildData([1, 2, 3, 4, 5]), ['g', 'b'])
+        # check that set_build_data was called
+        self.handler.set_build_data.assert_has_calls([
+            # first call
+            call(MyBuildData([1, 2, 3, 4, 5])),
+            # we answered good
+            call(MyBuildData([3, 4, 5])),
+            # we answered bad
+            call(MyBuildData([3, 4])),
+        ])
+        # ensure that the launcher was stopped
+        test_result['launcher'].stop.assert_called()
+        # ensure that we called the handler's methods
+        self.handler.build_good.assert_called_with(2, MyBuildData([3, 4, 5]))
+        self.handler.build_bad.assert_called_with(1, MyBuildData([3, 4]))
+        self.handler.build_data.ensure_limits.assert_called()
+        # bisection is finished
+        self.assertEqual(test_result['result'], Bisector.FINISHED)
+
+    def test_bisect_case2(self):
+        test_result = self.do_bisect(MyBuildData([1, 2, 3]), ['r', 's'])
+        # check that set_build_data was called
+        self.handler.set_build_data.assert_has_calls([
+            # first call
+            # this should be call(MyBuildData([1, 2, 3])),
+            # but as the code delete the index in place when we skip
+            # (with a del statement) our build_data is impacted.
+            # well, we just have to know that for the test.
+            call(MyBuildData([1, 3])),
+            # we asked for a retry (same comment as above)
+            call(MyBuildData([1, 3])),
+            # we skipped one
+            call(MyBuildData([1, 3])),
+        ])
+        # ensure that the launcher was stopped
+        test_result['launcher'].stop.assert_called()
+        # ensure that we called the handler's methods
+        self.handler.build_retry.assert_called_with(1)
+        self.handler.build_skip.assert_called_with(1)
+        self.handler.build_data.ensure_limits.assert_called()
+        # bisection is finished
+        self.assertEqual(test_result['result'], Bisector.FINISHED)
+
+    def test_bisect_user_exit(self):
+        test_result = self.do_bisect(MyBuildData(range(20)), ['e'])
+        # check that set_build_data was called
+        self.handler.set_build_data.assert_has_calls([call(MyBuildData(range(20)))])
+        # ensure that we called the handler's method
+        self.handler.user_exit.assert_called_with(10)
+        # user exit
+        self.assertEqual(test_result['result'], Bisector.USER_EXIT)
+
+if __name__ == '__main__':
+    unittest.main()

tests/unit/test_main.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import unittest
+from mock import patch, Mock
+import datetime
+
+from mozregression import main
+from mozregression import errors
+
+class TestMainCli(unittest.TestCase):
+    def setUp(self):
+        self.runner = Mock()
+
+    @patch('mozlog.structured.commandline.setup_logging')
+    @patch('mozregression.main.set_http_cache_session')
+    @patch('mozregression.limitedfilecache.get_cache')
+    @patch('mozregression.main.BisectRunner')
+    def do_cli(self, argv, BisectRunner, get_cache, set_http_cache_session,
+               setup_logging):
+        def create_runner(fetch_config, options):
+            self.runner.fetch_config = fetch_config
+            self.runner.options = options
+            return self.runner
+        BisectRunner.side_effect = create_runner
+        try:
+            main.cli(argv)
+        except SystemExit as exc:
+            return exc.code
+        else:
+            self.fail('mozregression.main.cli did not call sys.exit')
+
+    @patch('sys.stdout')
+    def test_get_usage(self, stdout):
+        output = []
+        stdout.write.side_effect = output.append
+
+        exitcode = self.do_cli(['-h'])
+        output = ''.join(output)
+        self.assertEqual(exitcode, 0)
+        self.assertIn('usage:', output)
+
+    def test_without_args(self):
+        self.runner.bisect_nightlies.return_value = 0
+        exitcode = self.do_cli([])
+        # application is by default firefox
+        self.assertEqual(self.runner.fetch_config.app_name, 'firefox')
+        # bisect_nightlies has been called
+        self.runner.bisect_nightlies.assert_called_with(datetime.date(2009, 1, 1),
+                                                        datetime.date.today())
+        # we exited with the return value of bisect_nightlies
+        self.assertEquals(exitcode, 0)
+
+    def test_basic_inbound(self):
+        self.runner.bisect_inbound.return_value = 0
+        exitcode = self.do_cli(['--inbound', '--good-rev=1', '--bad-rev=5'])
+        # application is by default firefox
+        self.assertEqual(self.runner.fetch_config.app_name, 'firefox')
+        # bisect_inbound has been called
+        self.runner.bisect_inbound.assert_called_with('1', '5')
+        # we exited with the return value of bisect_inbound
+        self.assertEquals(exitcode, 0)
+
+    def test_inbound_revs_must_be_given(self):
+        argslist = [
+            ['--inbound'],
+            ['--inbound', '--good-rev=1'],
+            ['--inbound', '--bad-rev=5'],
+        ]
+        for args in argslist:
+            exitcode = self.do_cli(args)
+            self.assertIn('--good-rev and --bad-rev must be set', exitcode)
+
+    @patch('mozregression.fetch_configs.FirefoxConfig.is_inbound')
+    def test_inbound_must_be_doable(self, is_inbound):
+        is_inbound.return_value = False
+        exitcode = self.do_cli(['--inbound', '--good-rev=1', '--bad-rev=5'])
+        self.assertIn('Unable to bissect inbound', exitcode)
+
+    @patch('mozregression.main.formatted_valid_release_dates')
+    def test_list_releases(self, formatted_valid_release_dates):
+        exitcode = self.do_cli(['--list-releases'])
+        formatted_valid_release_dates.assert_called_once()
+        self.assertIn(exitcode, (0, None))
+
+    def test_bad_date_and_bad_release_are_incompatible(self):
+        exitcode = self.do_cli(['--bad=2014-11-10', '--bad-release=1'])
+        self.assertIn('incompatible', exitcode)
+
+    def test_bad_release_invalid(self):
+        exitcode = self.do_cli(['--bad-release=-1'])
+        self.assertIn('Unable to find a matching date for release', exitcode)
+
+    def test_good_date_and_good_release_are_incompatible(self):
+        exitcode = self.do_cli(['--good=2014-11-10', '--good-release=1'])
+        self.assertIn('incompatible', exitcode)
+
+    def test_good_release_invalid(self):
+        exitcode = self.do_cli(['--good-release=-1'])
+        self.assertIn('Unable to find a matching date for release', exitcode)
+
+    def test_handle_keyboard_interrupt(self):
+        # KeyboardInterrupt are handled with a nice error message.
+        self.runner.bisect_nightlies.side_effect = KeyboardInterrupt
+        exitcode = self.do_cli([])
+        self.assertIn('Interrupted', exitcode)
+
+    def test_handle_mozregression_errors(self):
+        # Any MozRegressionError subclass is handled with a nice error message
+        self.runner.bisect_nightlies.side_effect = errors.MozRegressionError('my error')
+        exitcode = self.do_cli([])
+        self.assertIn('my error', exitcode)
+
+    def test_handle_other_errors(self):
+        # other exceptions are just thrown as usual, so we have complete stacktrace
+        self.runner.bisect_nightlies.side_effect = NameError
+        self.assertRaises(NameError, self.do_cli, [])