gecko-dev/tools/vcs/mach_commands.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, # You can obtain one at http://mozilla.org/MPL/2.0/.

from __future__ import absolute_import, unicode_literals

import os
import re
import subprocess
import sys

import logging

from mach.decorators import (
    CommandArgument,
    CommandProvider,
    Command,
)

from mozbuild.base import MachCommandBase

import mozpack.path as mozpath

import json

GITHUB_ROOT = 'https://github.com/'
PR_REPOSITORIES = {
    'webrender': {
        'github': 'servo/webrender',
        'path': 'gfx/wr',
        'bugzilla_product': 'Core',
        'bugzilla_component': 'Graphics: WebRender',
    },
    'webgpu': {
        'github': 'gfx-rs/wgpu',
        'path': 'gfx/wgpu',
        'bugzilla_product': 'Core',
        'bugzilla_component': 'Graphics: WebGPU',
    },
    'debugger': {
        'github': 'firefox-devtools/debugger',
        'path': 'devtools/client/debugger',
        'bugzilla_product': 'DevTools',
        'bugzilla_component': 'Debugger'
    },
}


@CommandProvider
class PullRequestImporter(MachCommandBase):
    @Command('import-pr', category='misc',
             description='Import a pull request from Github to the local repo.')
    @CommandArgument('-b', '--bug-number',
                     help='Bug number to use in the commit messages.')
    @CommandArgument('-t', '--bugzilla-token',
                     help='Bugzilla API token used to file a new bug if no bug number is '
                          'provided.')
    @CommandArgument('-r', '--reviewer',
                     help='Reviewer nick to apply to commit messages.')
    @CommandArgument('pull_request',
                     help='URL to the pull request to import (e.g. '
                          'https://github.com/servo/webrender/pull/3665).')
    def import_pr(self, pull_request, bug_number=None, bugzilla_token=None, reviewer=None):
        import requests
        pr_number = None
        repository = None
        for r in PR_REPOSITORIES.values():
            if pull_request.startswith(GITHUB_ROOT + r['github'] + '/pull/'):
                # sanitize URL, dropping anything after the PR number
                pr_number = int(re.search('/pull/([0-9]+)', pull_request).group(1))
                pull_request = GITHUB_ROOT + r['github'] + '/pull/' + str(pr_number)
                repository = r
                break

        if repository is None:
            self.log(logging.ERROR, 'unrecognized_repo', {},
                     'The pull request URL was not recognized; add it to the list of '
                     'recognized repos in PR_REPOSITORIES in %s' % __file__)
            sys.exit(1)

        self.log(logging.INFO, 'import_pr', {'pr_url': pull_request},
                 'Attempting to import {pr_url}')
        dirty = [f for f in self.repository.get_changed_files(mode='all')
                 if f.startswith(repository['path'])]
        if dirty:
            self.log(logging.ERROR, 'dirty_tree', repository,
                     'Local {path} tree is dirty; aborting!')
            sys.exit(1)
        target_dir = mozpath.join(self.topsrcdir, os.path.normpath(repository['path']))

        if bug_number is None:
            if bugzilla_token is None:
                self.log(logging.WARNING, 'no_token', {},
                         'No bug number or bugzilla API token provided; bug number will not '
                         'be added to commit messages.')
            else:
                bug_number = self._file_bug(bugzilla_token, repository, pr_number)
        elif bugzilla_token is not None:
            self.log(logging.WARNING, 'too_much_bug', {},
                     'Providing a bugzilla token is unnecessary when a bug number is provided. '
                     'Using bug number; ignoring token.')

        pr_patch = requests.get(pull_request + '.patch')
        pr_patch.raise_for_status()
        for patch in self._split_patches(pr_patch.content, bug_number, pull_request, reviewer):
            self.log(logging.INFO, 'commit_msg', patch,
                     'Processing commit [{commit_summary}] by [{author}] at [{date}]')
            patch_cmd = subprocess.Popen(['patch', '-p1', '-s'], stdin=subprocess.PIPE,
                                         cwd=target_dir)
            patch_cmd.stdin.write(patch['diff'].encode('utf-8'))
            patch_cmd.stdin.close()
            patch_cmd.wait()
            if patch_cmd.returncode != 0:
                self.log(logging.ERROR, 'commit_fail', {},
                         'Error applying diff from commit via "patch -p1 -s". Aborting...')
                sys.exit(patch_cmd.returncode)
            self.repository.commit(patch['commit_msg'], patch['author'], patch['date'],
                                   [target_dir])
            self.log(logging.INFO, 'commit_pass', {},
                     'Committed successfully.')

    def _file_bug(self, token, repo, pr_number):
        import requests
        bug = requests.post('https://bugzilla.mozilla.org/rest/bug?api_key=%s' % token,
                            json={
                                'product': repo['bugzilla_product'],
                                'component': repo['bugzilla_component'],
                                'summary': 'Land %s#%s in mozilla-central' %
                                           (repo['github'], pr_number),
                                'version': 'unspecified',
                            })
        bug.raise_for_status()
        self.log(logging.DEBUG, 'new_bug', {}, bug.content)
        bugnumber = json.loads(bug.content)['id']
        self.log(logging.INFO, 'new_bug', {'bugnumber': bugnumber},
                 'Filed bug {bugnumber}')
        return bugnumber

    def _split_patches(self, patchfile, bug_number, pull_request, reviewer):
        INITIAL = 0
        HEADERS = 1
        STAT_AND_DIFF = 2

        patch = b''
        state = INITIAL
        for line in patchfile.splitlines():
            if state == INITIAL:
                if line.startswith(b'From '):
                    state = HEADERS
            elif state == HEADERS:
                patch += line + b'\n'
                if line == b'---':
                    state = STAT_AND_DIFF
            elif state == STAT_AND_DIFF:
                if line.startswith(b'From '):
                    yield self._parse_patch(patch, bug_number, pull_request, reviewer)
                    patch = b''
                    state = HEADERS
                else:
                    patch += line + b'\n'
        if len(patch) > 0:
            yield self._parse_patch(patch, bug_number, pull_request, reviewer)
        return

    def _parse_patch(self, patch, bug_number, pull_request, reviewer):
        import email
        from email import (
            header,
            policy,
        )

        parse_policy = policy.compat32.clone(max_line_length=None)
        parsed_mail = email.message_from_bytes(patch, policy=parse_policy)

        def header_as_unicode(key):
            decoded = header.decode_header(parsed_mail[key])
            return str(header.make_header(decoded))

        author = header_as_unicode('From')
        date = header_as_unicode('Date')
        commit_summary = header_as_unicode('Subject')
        email_body = parsed_mail.get_payload(decode=True).decode('utf-8')
        (commit_body, diff) = ('\n' + email_body).rsplit('\n---\n', 1)

        bug_prefix = ''
        if bug_number is not None:
            bug_prefix = 'Bug %s - ' % bug_number
        commit_summary = re.sub(r'^\[PATCH[0-9 /]*\] ', bug_prefix, commit_summary)
        if reviewer is not None:
            commit_summary += ' r=' + reviewer

        commit_msg = commit_summary + '\n'
        if len(commit_body) > 0:
            commit_msg += commit_body + '\n'
        commit_msg += '\n[import_pr] From ' + pull_request + '\n'

        patch_obj = {
            'author': author,
            'date': date,
            'commit_summary': commit_summary,
            'commit_msg': commit_msg,
            'diff': diff,
        }
        return patch_obj