gecko-dev/taskcluster/mach_commands.py

# -*- coding: utf-8 -*-

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.


from __future__ import absolute_import, print_function, unicode_literals
import argparse
import json
import logging
import os
import sys
import time
import traceback
from functools import partial

from mach.decorators import (
    Command,
    CommandArgument,
    SettingsProvider,
    SubCommand,
)

import gecko_taskgraph.main
from gecko_taskgraph.main import commands as taskgraph_commands

logger = logging.getLogger("taskcluster")


@SettingsProvider
class TaskgraphConfig(object):
    @classmethod
    def config_settings(cls):
        return [
            (
                "taskgraph.diffcmd",
                "string",
                "The command to run with `./mach taskgraph --diff`",
                "diff --report-identical-files "
                "--label={attr}@{base} --label={attr}@{cur} -U20",
                {},
            )
        ]


def strtobool(value):
    """Convert string to boolean.

    Wraps "distutils.util.strtobool", deferring the import of the package
    in case it's not installed. Otherwise, we have a "chicken and egg problem" where
    |mach bootstrap| would install the required package to enable "distutils.util", but
    it can't because mach fails to interpret this file.
    """
    from distutils.util import strtobool

    return bool(strtobool(value))


def get_taskgraph_command_parser(name):
    """Given a command name, obtain its argument parser.

    Args:
        name (str): Name of the command.

    Returns:
        ArgumentParser: An ArgumentParser instance.
    """
    command = taskgraph_commands[name]
    parser = argparse.ArgumentParser()
    for arg in command.func.args:
        parser.add_argument(*arg[0], **arg[1])

    parser.set_defaults(func=command.func, **command.defaults)
    return parser


def get_taskgraph_decision_parser():
    parser = get_taskgraph_command_parser("decision")

    extra_args = [
        (
            ["--optimize-target-tasks"],
            {
                "type": lambda flag: strtobool(flag),
                "nargs": "?",
                "const": "true",
                "help": "If specified, this indicates whether the target "
                "tasks are eligible for optimization. Otherwise, the default "
                "for the project is used.",
            },
        ),
        (
            ["--include-push-tasks"],
            {
                "action": "store_true",
                "help": "Whether tasks from the on-push graph should be re-used "
                "in this graph. This allows cron graphs to avoid rebuilding "
                "jobs that were built on-push.",
            },
        ),
        (
            ["--rebuild-kind"],
            {
                "dest": "rebuild_kinds",
                "action": "append",
                "default": argparse.SUPPRESS,
                "help": "Kinds that should not be re-used from the on-push graph.",
            },
        ),
        (
            ["--comm-base-repository"],
            {
                "required": False,
                "help": "URL for 'base' comm-* repository to clone",
            },
        ),
        (
            ["--comm-head-repository"],
            {
                "required": False,
                "help": "URL for 'head' comm-* repository to fetch revision from",
            },
        ),
        (
            ["--comm-head-ref"],
            {
                "required": False,
                "help": "comm-* Reference (this is same as rev usually for hg)",
            },
        ),
        (
            ["--comm-head-rev"],
            {
                "required": False,
                "help": "Commit revision to use from head comm-* repository",
            },
        ),
    ]
    for arg in extra_args:
        parser.add_argument(*arg[0], **arg[1])

    return parser


@Command(
    "taskgraph",
    category="ci",
    description="Manipulate TaskCluster task graphs defined in-tree",
)
def taskgraph_command(command_context):
    """The taskgraph subcommands all relate to the generation of task graphs
    for Gecko continuous integration.  A task graph is a set of tasks linked
    by dependencies: for example, a binary must be built before it is tested,
    and that build may further depend on various toolchains, libraries, etc.
    """


@SubCommand(
    "taskgraph",
    "tasks",
    description="Show all tasks in the taskgraph",
    parser=partial(get_taskgraph_command_parser, "tasks"),
)
def taskgraph_tasks(command_context, **options):
    return run_show_taskgraph(command_context, **options)


@SubCommand(
    "taskgraph",
    "full",
    description="Show the full taskgraph",
    parser=partial(get_taskgraph_command_parser, "full"),
)
def taskgraph_full(command_context, **options):
    return run_show_taskgraph(command_context, **options)


@SubCommand(
    "taskgraph",
    "target",
    description="Show the target task set",
    parser=partial(get_taskgraph_command_parser, "target"),
)
def taskgraph_target(command_context, **options):
    return run_show_taskgraph(command_context, **options)


@SubCommand(
    "taskgraph",
    "target-graph",
    description="Show the target taskgraph",
    parser=partial(get_taskgraph_command_parser, "target-graph"),
)
def taskgraph_target_graph(command_context, **options):
    return run_show_taskgraph(command_context, **options)


@SubCommand(
    "taskgraph",
    "optimized",
    description="Show the optimized taskgraph",
    parser=partial(get_taskgraph_command_parser, "optimized"),
)
def taskgraph_optimized(command_context, **options):
    return run_show_taskgraph(command_context, **options)


@SubCommand(
    "taskgraph",
    "morphed",
    description="Show the morphed taskgraph",
    parser=partial(get_taskgraph_command_parser, "morphed"),
)
def taskgraph_morphed(command_context, **options):
    return run_show_taskgraph(command_context, **options)


def run_show_taskgraph(command_context, **options):
    # There are cases where we don't want to set up mach logging (e.g logs
    # are being redirected to disk). By monkeypatching the 'setup_logging'
    # function we can let 'taskgraph.main' decide whether or not to log to
    # the terminal.
    gecko_taskgraph.main.setup_logging = partial(
        setup_logging,
        command_context,
        quiet=options["quiet"],
        verbose=options["verbose"],
    )
    show_taskgraph = options.pop("func")
    return show_taskgraph(options)


@SubCommand("taskgraph", "actions", description="Write actions.json to stdout")
@CommandArgument(
    "--root", "-r", help="root of the taskgraph definition relative to topsrcdir"
)
@CommandArgument(
    "--quiet", "-q", action="store_true", help="suppress all logging output"
)
@CommandArgument(
    "--verbose",
    "-v",
    action="store_true",
    help="include debug-level logging output",
)
@CommandArgument(
    "--parameters",
    "-p",
    default="project=mozilla-central",
    help="parameters file (.yml or .json; see `taskcluster/docs/parameters.rst`)`",
)
def taskgraph_actions(command_context, **options):
    return show_actions(command_context, options)


@SubCommand(
    "taskgraph",
    "decision",
    description="Run the decision task",
    parser=get_taskgraph_decision_parser,
)
def taskgraph_decision(command_context, **options):
    """Run the decision task: generate a task graph and submit to
    TaskCluster.  This is only meant to be called within decision tasks,
    and requires a great many arguments.  Commands like `mach taskgraph
    optimized` are better suited to use on the command line, and can take
    the parameters file generated by a decision task."""
    try:
        setup_logging(command_context)
        start = time.monotonic()
        ret = taskgraph_commands["decision"].func(options)
        end = time.monotonic()
        if os.environ.get("MOZ_AUTOMATION") == "1":
            perfherder_data = {
                "framework": {"name": "build_metrics"},
                "suites": [
                    {
                        "name": "decision",
                        "value": end - start,
                        "lowerIsBetter": True,
                        "shouldAlert": True,
                        "subtests": [],
                    }
                ],
            }
            print(
                "PERFHERDER_DATA: {}".format(json.dumps(perfherder_data)),
                file=sys.stderr,
            )
        return ret
    except Exception:
        traceback.print_exc()
        sys.exit(1)


@SubCommand(
    "taskgraph",
    "cron",
    description="Provide a pointer to the new `.cron.yml` handler.",
)
def taskgraph_cron(command_context, **options):
    print(
        'Handling of ".cron.yml" files has move to '
        "https://hg.mozilla.org/ci/ci-admin/file/default/build-decision."
    )
    sys.exit(1)


@SubCommand(
    "taskgraph",
    "action-callback",
    description="Run action callback used by action tasks",
    parser=partial(get_taskgraph_command_parser, "action-callback"),
)
def action_callback(command_context, **options):
    setup_logging(command_context)
    taskgraph_commands["action-callback"].func(options)


@SubCommand(
    "taskgraph",
    "test-action-callback",
    description="Run an action callback in a testing mode",
    parser=partial(get_taskgraph_command_parser, "test-action-callback"),
)
def test_action_callback(command_context, **options):
    setup_logging(command_context)

    if not options["parameters"]:
        options["parameters"] = "project=mozilla-central"

    taskgraph_commands["test-action-callback"].func(options)


def setup_logging(command_context, quiet=False, verbose=True):
    """
    Set up Python logging for all loggers, sending results to stderr (so
    that command output can be redirected easily) and adding the typical
    mach timestamp.
    """
    # remove the old terminal handler
    old = command_context.log_manager.replace_terminal_handler(None)

    # re-add it, with level and fh set appropriately
    if not quiet:
        level = logging.DEBUG if verbose else logging.INFO
        command_context.log_manager.add_terminal_logging(
            fh=sys.stderr,
            level=level,
            write_interval=old.formatter.write_interval,
            write_times=old.formatter.write_times,
        )

    # all of the taskgraph logging is unstructured logging
    command_context.log_manager.enable_unstructured()


def show_actions(command_context, options):
    import gecko_taskgraph
    import gecko_taskgraph.actions
    import gecko_taskgraph.generator
    import gecko_taskgraph.parameters

    try:
        setup_logging(
            command_context, quiet=options["quiet"], verbose=options["verbose"]
        )
        parameters = gecko_taskgraph.parameters.parameters_loader(options["parameters"])

        tgg = gecko_taskgraph.generator.TaskGraphGenerator(
            root_dir=options.get("root"),
            parameters=parameters,
        )

        actions = gecko_taskgraph.actions.render_actions_json(
            tgg.parameters,
            tgg.graph_config,
            decision_task_id="DECISION-TASK",
        )
        print(json.dumps(actions, sort_keys=True, indent=2, separators=(",", ": ")))
    except Exception:
        traceback.print_exc()
        sys.exit(1)


@Command(
    "taskcluster-load-image",
    category="ci",
    description="Load a pre-built Docker image. Note that you need to "
    "have docker installed and running for this to work.",
    parser=partial(get_taskgraph_command_parser, "load-image"),
)
def load_image(command_context, **kwargs):
    taskgraph_commands["load-image"].func(kwargs)


@Command(
    "taskcluster-build-image",
    category="ci",
    description="Build a Docker image",
    parser=partial(get_taskgraph_command_parser, "build-image"),
)
def build_image(command_context, **kwargs):
    try:
        taskgraph_commands["build-image"].func(kwargs)
    except Exception:
        traceback.print_exc()
        sys.exit(1)


@Command(
    "taskcluster-image-digest",
    category="ci",
    description="Print the digest of the image of this name based on the "
    "current contents of the tree.",
    parser=partial(get_taskgraph_command_parser, "build-image"),
)
def image_digest(command_context, **kwargs):
    taskgraph_commands["image-digest"].func(kwargs)


@Command(
    "release-history",
    category="ci",
    description="Query balrog for release history used by enable partials generation",
)
@CommandArgument(
    "-b",
    "--branch",
    help="The gecko project branch used in balrog, such as "
    "mozilla-central, release, maple",
)
@CommandArgument(
    "--product", default="Firefox", help="The product identifier, such as 'Firefox'"
)
def generate_partials_builds(command_context, product, branch):
    from gecko_taskgraph.util.partials import populate_release_history

    try:
        import yaml

        release_history = {"release_history": populate_release_history(product, branch)}
        print(
            yaml.safe_dump(
                release_history, allow_unicode=True, default_flow_style=False
            )
        )
    except Exception:
        traceback.print_exc()
        sys.exit(1)