gecko-dev/taskcluster/taskgraph/transforms/diffoscope.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
This transform construct tasks to perform diffs between builds, as
defined in kind.yml
"""

from __future__ import absolute_import, print_function, unicode_literals

from six import text_type
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.task import task_description_schema
from taskgraph.util.schema import (
    Schema,
)
from taskgraph.util.taskcluster import get_artifact_path
from voluptuous import (
    Any,
    Optional,
    Required,
)

index_or_string = Any(
    text_type,
    {Required('index-search'): text_type},
)

diff_description_schema = Schema({
    # Name of the diff task.
    Required('name'): text_type,

    # Treeherder symbol.
    Required('symbol'): text_type,

    # relative path (from config.path) to the file the task was defined in.
    Optional('job-from'): text_type,

    # Original and new builds to compare.
    Required('original'): index_or_string,
    Required('new'): index_or_string,

    # Arguments to pass to diffoscope, used for job-defaults in
    # taskcluster/ci/diffoscope/kind.yml
    Optional('args'): text_type,

    # Extra arguments to pass to diffoscope, that can be set per job.
    Optional('extra-args'): text_type,

    # Fail the task when differences are detected.
    Optional('fail-on-diff'): bool,

    # What artifact to check the differences of. Defaults to target.tar.bz2
    # for Linux, target.dmg for Mac, target.zip for Windows, target.apk for
    # Android.
    Optional('artifact'): text_type,

    # Whether to unpack first. Diffoscope can normally work without unpacking,
    # but when one needs to --exclude some contents, that doesn't work out well
    # if said content is packed (e.g. in omni.ja).
    Optional('unpack'): bool,

    # Commands to run before performing the diff.
    Optional('pre-diff-commands'): [text_type],

    # Only run the task on a set of projects/branches.
    Optional('run-on-projects'): task_description_schema['run-on-projects'],

    Optional('optimization'): task_description_schema['optimization'],
})

transforms = TransformSequence()
transforms.add_validate(diff_description_schema)


@transforms.add
def fill_template(config, tasks):
    dummy_tasks = {}

    for task in tasks:
        name = task['name']

        deps = {}
        urls = {}
        previous_artifact = None
        artifact = task.get('artifact')
        for k in ('original', 'new'):
            value = task[k]
            if isinstance(value, text_type):
                deps[k] = value
                dep_name = k
                os_hint = value
            else:
                index = value['index-search']
                if index not in dummy_tasks:
                    dummy_tasks[index] = {
                        'label': 'index-search-' + index,
                        'description': index,
                        'worker-type': 'invalid/always-optimized',
                        'run': {
                            'using': 'always-optimized',
                        },
                        'optimization': {
                            'index-search': [index],
                        }
                    }
                    yield dummy_tasks[index]
                deps[index] = 'index-search-' + index
                dep_name = index
                os_hint = index.split('.')[-1]
            if artifact:
                pass
            elif 'linux' in os_hint:
                artifact = 'target.tar.bz2'
            elif 'macosx' in os_hint:
                artifact = 'target.dmg'
            elif 'android' in os_hint:
                artifact = 'target.apk'
            elif 'win' in os_hint:
                artifact = 'target.zip'
            else:
                raise Exception(
                    'Cannot figure out the OS for {!r}'.format(value))
            if previous_artifact is not None and previous_artifact != artifact:
                raise Exception(
                    'Cannot compare builds from different OSes')
            urls[k] = {
                'artifact-reference': '<{}/{}>'.format(
                    dep_name, get_artifact_path(task, artifact)),
            }
            previous_artifact = artifact

        taskdesc = {
            'label': 'diff-' + name,
            'description': name,
            'treeherder': {
                'symbol': task['symbol'],
                'platform': 'diff/opt',
                'kind': 'other',
                'tier': 2,
            },
            'worker-type': 'b-linux',
            'worker': {
                'docker-image': {'in-tree': 'diffoscope'},
                'artifacts': [{
                    'type': 'file',
                    'path': '/builds/worker/{}'.format(f),
                    'name': 'public/{}'.format(f),
                } for f in (
                    'diff.html',
                    'diff.txt',
                )],
                'env': {
                    'ORIG_URL': urls['original'],
                    'NEW_URL': urls['new'],
                    'DIFFOSCOPE_ARGS': ' '.join(
                        task[k] for k in ('args', 'extra-args') if k in task),
                    'PRE_DIFF': '; '.join(task.get('pre-diff-commands', [])),
                },
                'max-run-time': 1800,
            },
            'run': {
                'using': 'run-task',
                'checkout': task.get('unpack', False),
                'command': '/builds/worker/bin/get_and_diffoscope{}{}'.format(
                    ' --unpack' if task.get('unpack') else '',
                    ' --fail' if task.get('fail-on-diff') else '',
                ),
            },
            'dependencies': deps,
            'optimization': task.get('optimization'),
        }
        if 'run-on-projects' in task:
            taskdesc['run-on-projects'] = task['run-on-projects']

        if artifact.endswith('.dmg'):
            taskdesc.setdefault('fetches', {}).setdefault('toolchain', []).extend([
                'linux64-cctools-port',
                'linux64-libdmg',
            ])

        yield taskdesc