mozilla
/
moz-skia
зеркало из https://github.com/mozilla/moz-skia.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

add_codereview_message script to append messages to a CL 

Add add_codereview_message.py script.  This script takes a message and
a codereview URL as arguments calls `git cl upload` to append the
message to the given codereview issue.

Motivation: We are automating a process of testing Chromium/Skia DEPS
rolls where roll_deps.py is automatically run every night, and then at
a later time, compare_codereview.py is run to generate a report of the
trybot results.  This script can be used to append that report to the
DEPS roll Cl.

This CL also refactors functioanlity common to roll_deps and
add_codereview_message:
*   Add tools/git_utils.py module.
    -   ChangeGitBranch class was factored out of
        roll_deps.GitBranchCLUpload.  The other half of that class
        became roll_deps.git_cl_uploader function.  I make use of
        this new class in both roll_deps and upload_deps_roll.
    -   test_git_executable moved to this new module.
*   Add tools/misc_utils.py - move VerboseSubprocess, ChangeDir,
    and ReSearch classes from roll_deps module.
*   Remove generic functions from roll_deps.
*   Add git_cl_uploader function to roll_deps.  Refactor
    roll_deps.roll_deps function to make use of it

BUG=skia:
R=borenet@google.com, robertphillips@google.com

Author: halcanary@google.com

Review URL: https://codereview.chromium.org/141483011

git-svn-id: http://skia.googlecode.com/svn/trunk@13251 2bbb7eff-a529-9590-31e7-b0007b416f81
This commit is contained in:
commit-bot@chromium.org 2014-01-30 22:12:30 +00:00
Родитель 7d7e5bc296
Коммит 0ed90029e3
4 изменённых файлов: 642 добавлений и 388 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

148
tools/add_codereview_message.py Executable file
Просмотреть файл

@ -0,0 +1,148 @@
#!/usr/bin/python2
# Copyright 2014 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Add message to codereview issue.
This script takes a codereview URL or a codereview issue number as its
argument and a (possibly multi-line) message on stdin. It then calls
`git cl upload` to append the message to the given codereview issue.
Usage:
echo MESSAGE | %prog -c CHECKOUT_PATH CODEREVIEW_ISSUE
or:
cd /path/to/git/checkout
%prog CODEREVIEW_ISSUE <<EOF
MESSAGE
EOF
or:
%prog --help
"""
import optparse
import os
import sys
import git_utils
import misc_utils
DEFAULT_REVIEWERS = ','.join([
'rmistry@google.com',
'reed@google.com',
'bsalomon@google.com',
'robertphillips@google.com',
])
DEFAULT_CC_LIST = ','.join([
'skia-team@google.com',
])
def add_codereview_message(codereview_url, message, checkout_path,
skip_cl_upload, verbose, reviewers, cclist):
"""Add a message to a given codereview.
Args:
codereview_url: (string) we will extract the issue number from
this url, or this could simply be the issue number.
message: (string) will be passed to `git cl upload -m $MESSAGE`
checkout_path: (string) location of the git
repository checkout to be used.
skip_cl_upload: (boolean) if true, don't actually
add the message and keep the temporary branch around.
verbose: (boolean) print out details useful for debugging.
reviewers: (string) comma-separated list of reviewers
cclist: (string) comma-separated list of addresses to be
carbon-copied
"""
# pylint: disable=I0011,R0913
git = git_utils.git_executable()
issue = codereview_url.strip('/').split('/')[-1]
vsp = misc_utils.VerboseSubprocess(verbose)
if skip_cl_upload:
branch_name = 'issue_%s' % issue
else:
branch_name = None
upstream = 'origin/master'
with misc_utils.ChangeDir(checkout_path, verbose):
vsp.check_call([git, 'fetch', '-q', 'origin'])
with git_utils.ChangeGitBranch(branch_name, upstream, verbose):
vsp.check_call([git, 'cl', 'patch', issue])
git_upload = [
git, 'cl', 'upload', '-t', 'bot report', '-m', message]
if cclist:
git_upload.append('--cc=' + cclist)
if reviewers:
git_upload.append('--reviewers=' + reviewers)
if skip_cl_upload:
branch_name = git_utils.git_branch_name(verbose)
space = ' '
print 'You should call:'
misc_utils.print_subprocess_args(space, ['cd', os.getcwd()])
misc_utils.print_subprocess_args(
space, [git, 'checkout', branch_name])
misc_utils.print_subprocess_args(space, git_upload)
else:
vsp.check_call(git_upload)
print vsp.check_output([git, 'cl', 'issue'])
def main(argv):
"""main function; see module-level docstring and GetOptionParser help.
Args:
argv: sys.argv[1:]-type argument list.
"""
option_parser = optparse.OptionParser(usage=__doc__)
option_parser.add_option(
'-c', '--checkout_path',
default=os.curdir,
help='Path to the Git repository checkout,'
' defaults to current working directory.')
option_parser.add_option(
'', '--skip_cl_upload', action='store_true', default=False,
help='Skip the cl upload step; useful for testing.')
option_parser.add_option(
'', '--verbose', action='store_true', dest='verbose', default=False,
help='Do not suppress the output from `git cl`.',)
option_parser.add_option(
'', '--git_path', default='git',
help='Git executable, defaults to "git".',)
option_parser.add_option(
'', '--reviewers', default=DEFAULT_REVIEWERS,
help=('Comma-separated list of reviewers. Default is "%s".'
% DEFAULT_REVIEWERS))
option_parser.add_option(
'', '--cc', default=DEFAULT_CC_LIST,
help=('Comma-separated list of addresses to be carbon-copied.'
' Default is "%s".' % DEFAULT_CC_LIST))
options, arguments = option_parser.parse_args(argv)
if not options.checkout_path:
option_parser.error('Must specify checkout_path.')
if not git_utils.git_executable():
option_parser.error('Invalid git executable.')
if len(arguments) > 1:
option_parser.error('Extra arguments.')
if len(arguments) != 1:
option_parser.error('Missing Codereview URL.')
message = sys.stdin.read()
add_codereview_message(arguments[0], message, options.checkout_path,
options.skip_cl_upload, options.verbose,
options.reviewers, options.cc)
if __name__ == '__main__':
main(sys.argv[1:])

168
tools/git_utils.py Normal file
Просмотреть файл

@ -0,0 +1,168 @@
# Copyright 2014 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Module to host the ChangeGitBranch class and test_git_executable function.
"""
import os
import subprocess
import misc_utils
class ChangeGitBranch(object):
"""Class to manage git branches.
This class allows one to create a new branch in a repository based
off of a given commit, and restore the original tree state.
Assumes current working directory is a git repository.
Example:
with ChangeGitBranch():
edit_files(files)
git_add(files)
git_commit()
git_format_patch('HEAD~')
# At this point, the repository is returned to its original
# state.
Constructor Args:
branch_name: (string) if not None, the name of the branch to
use. If None, then use a temporary branch that will be
deleted. If the branch already exists, then a different
branch name will be created. Use git_branch_name() to
find the actual branch name used.
upstream_branch: (string) if not None, the name of the branch or
commit to branch from. If None, then use origin/master
verbose: (boolean) if true, makes debugging easier.
Raises:
OSError: the git executable disappeared.
subprocess.CalledProcessError: git returned unexpected status.
Exception: if the given branch name exists, or if the repository
isn't clean on exit, or git can't be found.
"""
# pylint: disable=I0011,R0903,R0902
def __init__(self,
branch_name=None,
upstream_branch=None,
verbose=False):
# pylint: disable=I0011,R0913
if branch_name:
self._branch_name = branch_name
self._delete_branch = False
else:
self._branch_name = 'ChangeGitBranchTempBranch'
self._delete_branch = True
if upstream_branch:
self._upstream_branch = upstream_branch
else:
self._upstream_branch = 'origin/master'
self._git = git_executable()
if not self._git:
raise Exception('Git can\'t be found.')
self._stash = None
self._original_branch = None
self._vsp = misc_utils.VerboseSubprocess(verbose)
def _has_git_diff(self):
"""Return true iff repository has uncommited changes."""
return bool(self._vsp.call([self._git, 'diff', '--quiet', 'HEAD']))
def _branch_exists(self, branch):
"""Return true iff branch exists."""
return 0 == self._vsp.call([self._git, 'show-ref', '--quiet', branch])
def __enter__(self):
git, vsp = self._git, self._vsp
if self._branch_exists(self._branch_name):
i, branch_name = 0, self._branch_name
while self._branch_exists(branch_name):
i += 1
branch_name = '%s_%03d' % (self._branch_name, i)
self._branch_name = branch_name
self._stash = self._has_git_diff()
if self._stash:
vsp.check_call([git, 'stash', 'save'])
self._original_branch = git_branch_name(vsp.verbose)
vsp.check_call(
[git, 'checkout', '-q', '-b',
self._branch_name, self._upstream_branch])
def __exit__(self, etype, value, traceback):
git, vsp = self._git, self._vsp
if self._has_git_diff():
status = vsp.check_output([git, 'status', '-s'])
raise Exception('git checkout not clean:\n%s' % status)
vsp.check_call([git, 'checkout', '-q', self._original_branch])
if self._stash:
vsp.check_call([git, 'stash', 'pop'])
if self._delete_branch:
assert self._original_branch != self._branch_name
vsp.check_call([git, 'branch', '-D', self._branch_name])
def git_branch_name(verbose=False):
"""Return a description of the current branch.
Args:
verbose: (boolean) makes debugging easier
Returns:
A string suitable for passing to `git checkout` later.
"""
git = git_executable()
vsp = misc_utils.VerboseSubprocess(verbose)
try:
full_branch = vsp.strip_output([git, 'symbolic-ref', 'HEAD'])
return full_branch.split('/')[-1]
except (subprocess.CalledProcessError,):
# "fatal: ref HEAD is not a symbolic ref"
return vsp.strip_output([git, 'rev-parse', 'HEAD'])
def test_git_executable(git):
"""Test the git executable.
Args:
git: git executable path.
Returns:
True if test is successful.
"""
with open(os.devnull, 'w') as devnull:
try:
subprocess.call([git, '--version'], stdout=devnull)
except (OSError,):
return False
return True
def git_executable():
"""Find the git executable.
If the GIT_EXECUTABLE environment variable is set, that will
override whatever is found in the PATH.
If no suitable executable is found, return None
Returns:
A string suiable for passing to subprocess functions, or None.
"""
env_git = os.environ.get('GIT_EXECUTABLE')
if env_git and test_git_executable(env_git):
return env_git
for git in ('git', 'git.exe', 'git.bat'):
if test_git_executable(git):
return git
return None

224
tools/misc_utils.py Normal file
Просмотреть файл

@ -0,0 +1,224 @@
# Copyright 2014 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Module to host the VerboseSubprocess, ChangeDir, and ReSearch classes.
"""
import os
import re
import subprocess
def print_subprocess_args(prefix, *args, **kwargs):
"""Print out args in a human-readable manner."""
def quote_and_escape(string):
"""Quote and escape a string if necessary."""
if ' ' in string or '\n' in string:
string = '"%s"' % string.replace('"', '\\"')
return string
if 'cwd' in kwargs:
print '%scd %s' % (prefix, kwargs['cwd'])
print prefix + ' '.join(quote_and_escape(arg) for arg in args[0])
if 'cwd' in kwargs:
print '%scd -' % prefix
class VerboseSubprocess(object):
"""Call subprocess methods, but print out command before executing.
Attributes:
verbose: (boolean) should we print out the command or not. If
not, this is the same as calling the subprocess method
quiet: (boolean) suppress stdout on check_call and call.
prefix: (string) When verbose, what to print before each command.
"""
def __init__(self, verbose):
self.verbose = verbose
self.quiet = not verbose
self.prefix = '~~$ '
def check_call(self, *args, **kwargs):
"""Wrapper for subprocess.check_call().
Args:
*args: to be passed to subprocess.check_call()
**kwargs: to be passed to subprocess.check_call()
Returns:
Whatever subprocess.check_call() returns.
Raises:
OSError or subprocess.CalledProcessError: raised by check_call.
"""
if self.verbose:
print_subprocess_args(self.prefix, *args, **kwargs)
if self.quiet:
with open(os.devnull, 'w') as devnull:
return subprocess.check_call(*args, stdout=devnull, **kwargs)
else:
return subprocess.check_call(*args, **kwargs)
def call(self, *args, **kwargs):
"""Wrapper for subprocess.check().
Args:
*args: to be passed to subprocess.check_call()
**kwargs: to be passed to subprocess.check_call()
Returns:
Whatever subprocess.call() returns.
Raises:
OSError or subprocess.CalledProcessError: raised by call.
"""
if self.verbose:
print_subprocess_args(self.prefix, *args, **kwargs)
if self.quiet:
with open(os.devnull, 'w') as devnull:
return subprocess.call(*args, stdout=devnull, **kwargs)
else:
return subprocess.call(*args, **kwargs)
def check_output(self, *args, **kwargs):
"""Wrapper for subprocess.check_output().
Args:
*args: to be passed to subprocess.check_output()
**kwargs: to be passed to subprocess.check_output()
Returns:
Whatever subprocess.check_output() returns.
Raises:
OSError or subprocess.CalledProcessError: raised by check_output.
"""
if self.verbose:
print_subprocess_args(self.prefix, *args, **kwargs)
return subprocess.check_output(*args, **kwargs)
def strip_output(self, *args, **kwargs):
"""Wrap subprocess.check_output and str.strip().
Pass the given arguments into subprocess.check_output() and return
the results, after stripping any excess whitespace.
Args:
*args: to be passed to subprocess.check_output()
**kwargs: to be passed to subprocess.check_output()
Returns:
The output of the process as a string without leading or
trailing whitespace.
Raises:
OSError or subprocess.CalledProcessError: raised by check_output.
"""
if self.verbose:
print_subprocess_args(self.prefix, *args, **kwargs)
return str(subprocess.check_output(*args, **kwargs)).strip()
def popen(self, *args, **kwargs):
"""Wrapper for subprocess.Popen().
Args:
*args: to be passed to subprocess.Popen()
**kwargs: to be passed to subprocess.Popen()
Returns:
The output of subprocess.Popen()
Raises:
OSError or subprocess.CalledProcessError: raised by Popen.
"""
if self.verbose:
print_subprocess_args(self.prefix, *args, **kwargs)
return subprocess.Popen(*args, **kwargs)
class ChangeDir(object):
"""Use with a with-statement to temporarily change directories."""
# pylint: disable=I0011,R0903
def __init__(self, directory, verbose=False):
self._directory = directory
self._verbose = verbose
def __enter__(self):
if self._directory != os.curdir:
if self._verbose:
print '~~$ cd %s' % self._directory
cwd = os.getcwd()
os.chdir(self._directory)
self._directory = cwd
def __exit__(self, etype, value, traceback):
if self._directory != os.curdir:
if self._verbose:
print '~~$ cd %s' % self._directory
os.chdir(self._directory)
class ReSearch(object):
"""A collection of static methods for regexing things."""
@staticmethod
def search_within_stream(input_stream, pattern, default=None):
"""Search for regular expression in a file-like object.
Opens a file for reading and searches line by line for a match to
the regex and returns the parenthesized group named return for the
first match. Does not search across newlines.
For example:
pattern = '^root(:[^:]*){4}:(?P<return>[^:]*)'
with open('/etc/passwd', 'r') as stream:
return search_within_file(stream, pattern)
should return root's home directory (/root on my system).
Args:
input_stream: file-like object to be read
pattern: (string) to be passed to re.compile
default: what to return if no match
Returns:
A string or whatever default is
"""
pattern_object = re.compile(pattern)
for line in input_stream:
match = pattern_object.search(line)
if match:
return match.group('return')
return default
@staticmethod
def search_within_string(input_string, pattern, default=None):
"""Search for regular expression in a string.
Args:
input_string: (string) to be searched
pattern: (string) to be passed to re.compile
default: what to return if no match
Returns:
A string or whatever default is
"""
match = re.search(pattern, input_string)
return match.group('return') if match else default
@staticmethod
def search_within_output(verbose, pattern, default, *args, **kwargs):
"""Search for regular expression in a process output.
Does not search across newlines.
Args:
verbose: (boolean) shoule we call print_subprocess_args?
pattern: (string) to be passed to re.compile
default: what to return if no match
*args: to be passed to subprocess.Popen()
**kwargs: to be passed to subprocess.Popen()
Returns:
A string or whatever default is
"""
if verbose:
print_subprocess_args('~~$ ', *args, **kwargs)
proc = subprocess.Popen(*args, stdout=subprocess.PIPE, **kwargs)
return ReSearch.search_within_stream(proc.stdout, pattern, default)

490
tools/roll_deps.py
Просмотреть файл

@ -16,6 +16,9 @@ This script:
- creates a whitespace-only commit and uploads that to to Rietveld.
- returns the Chromium tree to its previous state.
To specify the location of the git executable, set the GIT_EXECUTABLE
environment variable.
Usage:
%prog -c CHROMIUM_PATH -r REVISION [OPTIONAL_OPTIONS]
"""
@ -29,6 +32,9 @@ import subprocess
import sys
import tempfile
import git_utils
import misc_utils
DEFAULT_BOTS_LIST = [
'android_clang_dbg',
@ -74,15 +80,16 @@ class DepsRollConfig(object):
self.revision_format = (
'git-svn-id: http://skia.googlecode.com/svn/trunk@%d ')
self.git = git_utils.git_executable()
if not options:
options = DepsRollConfig.GetOptionParser()
# pylint: disable=I0011,E1103
self.verbose = options.verbose
self.vsp = VerboseSubprocess(self.verbose)
self.vsp = misc_utils.VerboseSubprocess(self.verbose)
self.save_branches = not options.delete_branches
self.search_depth = options.search_depth
self.chromium_path = options.chromium_path
self.git = options.git_path
self.skip_cl_upload = options.skip_cl_upload
# Split and remove empty strigns from the bot list.
self.cl_bot_list = [bot for bot in options.bots.split(',') if bot]
@ -94,7 +101,7 @@ class DepsRollConfig(object):
# 'bsalomon@google.com',
# 'robertphillips@google.com',
])
self.cc_list = ','.join([
self.cc_list = ','.join([
# 'skia-team@google.com',
])
@ -134,9 +141,6 @@ class DepsRollConfig(object):
option_parser.add_option(
'', '--search_depth', type='int', default=100,
help='How far back to look for the revision.')
option_parser.add_option(
'', '--git_path', help='Git executable, defaults to "git".',
default='git')
option_parser.add_option(
'', '--delete_branches', help='Delete the temporary branches',
action='store_true', dest='delete_branches', default=False)
@ -145,7 +149,7 @@ class DepsRollConfig(object):
action='store_true', dest='verbose', default=False)
option_parser.add_option(
'', '--skip_cl_upload', help='Skip the cl upload step; useful'
' for testing or with --save_branches.',
' for testing.',
action='store_true', default=False)
default_bots_help = (
@ -159,245 +163,17 @@ class DepsRollConfig(object):
return option_parser
def test_git_executable(git_executable):
"""Test the git executable.
Args:
git_executable: git executable path.
Returns:
True if test is successful.
"""
with open(os.devnull, 'w') as devnull:
try:
subprocess.call([git_executable, '--version'], stdout=devnull)
except (OSError,):
return False
return True
class DepsRollError(Exception):
"""Exceptions specific to this module."""
pass
class VerboseSubprocess(object):
"""Call subprocess methods, but print out command before executing.
Attributes:
verbose: (boolean) should we print out the command or not. If
not, this is the same as calling the subprocess method
quiet: (boolean) suppress stdout on check_call and call.
prefix: (string) When verbose, what to print before each command.
"""
def __init__(self, verbose):
self.verbose = verbose
self.quiet = not verbose
self.prefix = '~~$ '
@staticmethod
def _fix(string):
"""Quote and escape a string if necessary."""
if ' ' in string or '\n' in string:
string = '"%s"' % string.replace('\n', '\\n')
return string
@staticmethod
def print_subprocess_args(prefix, *args, **kwargs):
"""Print out args in a human-readable manner."""
if 'cwd' in kwargs:
print '%scd %s' % (prefix, kwargs['cwd'])
print prefix + ' '.join(VerboseSubprocess._fix(arg) for arg in args[0])
if 'cwd' in kwargs:
print '%scd -' % prefix
def check_call(self, *args, **kwargs):
"""Wrapper for subprocess.check_call().
Args:
*args: to be passed to subprocess.check_call()
**kwargs: to be passed to subprocess.check_call()
Returns:
Whatever subprocess.check_call() returns.
Raises:
OSError or subprocess.CalledProcessError: raised by check_call.
"""
if self.verbose:
self.print_subprocess_args(self.prefix, *args, **kwargs)
if self.quiet:
with open(os.devnull, 'w') as devnull:
return subprocess.check_call(*args, stdout=devnull, **kwargs)
else:
return subprocess.check_call(*args, **kwargs)
def call(self, *args, **kwargs):
"""Wrapper for subprocess.check().
Args:
*args: to be passed to subprocess.check_call()
**kwargs: to be passed to subprocess.check_call()
Returns:
Whatever subprocess.call() returns.
Raises:
OSError or subprocess.CalledProcessError: raised by call.
"""
if self.verbose:
self.print_subprocess_args(self.prefix, *args, **kwargs)
if self.quiet:
with open(os.devnull, 'w') as devnull:
return subprocess.call(*args, stdout=devnull, **kwargs)
else:
return subprocess.call(*args, **kwargs)
def check_output(self, *args, **kwargs):
"""Wrapper for subprocess.check_output().
Args:
*args: to be passed to subprocess.check_output()
**kwargs: to be passed to subprocess.check_output()
Returns:
Whatever subprocess.check_output() returns.
Raises:
OSError or subprocess.CalledProcessError: raised by check_output.
"""
if self.verbose:
self.print_subprocess_args(self.prefix, *args, **kwargs)
return subprocess.check_output(*args, **kwargs)
def strip_output(self, *args, **kwargs):
"""Wrap subprocess.check_output and str.strip().
Pass the given arguments into subprocess.check_output() and return
the results, after stripping any excess whitespace.
Args:
*args: to be passed to subprocess.check_output()
**kwargs: to be passed to subprocess.check_output()
Returns:
The output of the process as a string without leading or
trailing whitespace.
Raises:
OSError or subprocess.CalledProcessError: raised by check_output.
"""
if self.verbose:
self.print_subprocess_args(self.prefix, *args, **kwargs)
return str(subprocess.check_output(*args, **kwargs)).strip()
def popen(self, *args, **kwargs):
"""Wrapper for subprocess.Popen().
Args:
*args: to be passed to subprocess.Popen()
**kwargs: to be passed to subprocess.Popen()
Returns:
The output of subprocess.Popen()
Raises:
OSError or subprocess.CalledProcessError: raised by Popen.
"""
if self.verbose:
self.print_subprocess_args(self.prefix, *args, **kwargs)
return subprocess.Popen(*args, **kwargs)
class ChangeDir(object):
"""Use with a with-statement to temporarily change directories."""
# pylint: disable=I0011,R0903
def __init__(self, directory, verbose=False):
self._directory = directory
self._verbose = verbose
def __enter__(self):
if self._verbose:
print '~~$ cd %s' % self._directory
cwd = os.getcwd()
os.chdir(self._directory)
self._directory = cwd
def __exit__(self, etype, value, traceback):
if self._verbose:
print '~~$ cd %s' % self._directory
os.chdir(self._directory)
class ReSearch(object):
"""A collection of static methods for regexing things."""
@staticmethod
def search_within_stream(input_stream, pattern, default=None):
"""Search for regular expression in a file-like object.
Opens a file for reading and searches line by line for a match to
the regex and returns the parenthesized group named return for the
first match. Does not search across newlines.
For example:
pattern = '^root(:[^:]*){4}:(?P<return>[^:]*)'
with open('/etc/passwd', 'r') as stream:
return search_within_file(stream, pattern)
should return root's home directory (/root on my system).
Args:
input_stream: file-like object to be read
pattern: (string) to be passed to re.compile
default: what to return if no match
Returns:
A string or whatever default is
"""
pattern_object = re.compile(pattern)
for line in input_stream:
match = pattern_object.search(line)
if match:
return match.group('return')
return default
@staticmethod
def search_within_string(input_string, pattern, default=None):
"""Search for regular expression in a string.
Args:
input_string: (string) to be searched
pattern: (string) to be passed to re.compile
default: what to return if no match
Returns:
A string or whatever default is
"""
match = re.search(pattern, input_string)
return match.group('return') if match else default
@staticmethod
def search_within_output(verbose, pattern, default, *args, **kwargs):
"""Search for regular expression in a process output.
Does not search across newlines.
Args:
verbose: (boolean) shoule we call
VerboseSubprocess.print_subprocess_args?
pattern: (string) to be passed to re.compile
default: what to return if no match
*args: to be passed to subprocess.Popen()
**kwargs: to be passed to subprocess.Popen()
Returns:
A string or whatever default is
"""
if verbose:
VerboseSubprocess.print_subprocess_args(
'~~$ ', *args, **kwargs)
proc = subprocess.Popen(*args, stdout=subprocess.PIPE, **kwargs)
return ReSearch.search_within_stream(proc.stdout, pattern, default)
def get_svn_revision(config, commit):
"""Works in both git and git-svn. returns a string."""
svn_format = (
'(git-svn-id: [^@ ]+@|SVN changes up to revision |'
'LKGR w/ DEPS up to revision )(?P<return>[0-9]+)')
svn_revision = ReSearch.search_within_output(
svn_revision = misc_utils.ReSearch.search_within_output(
config.verbose, svn_format, None,
[config.git, 'log', '-n', '1', '--format=format:%B', commit])
if not svn_revision:
@ -423,11 +199,12 @@ class SkiaGitCheckout(object):
skia_dir = None
self._original_cwd = os.getcwd()
if config.skia_git_checkout_path:
skia_dir = config.skia_git_checkout_path
## Update origin/master if needed.
if self._config.verbose:
print '~~$', 'cd', skia_dir
os.chdir(skia_dir)
if config.skia_git_checkout_path != os.curdir:
skia_dir = config.skia_git_checkout_path
## Update origin/master if needed.
if self._config.verbose:
print '~~$', 'cd', skia_dir
os.chdir(skia_dir)
config.vsp.check_call([git, 'fetch', '-q', 'origin'])
self._use_temp = None
else:
@ -443,9 +220,10 @@ class SkiaGitCheckout(object):
raise error
def __exit__(self, etype, value, traceback):
if self._config.verbose:
print '~~$', 'cd', self._original_cwd
os.chdir(self._original_cwd)
if self._config.skia_git_checkout_path != os.curdir:
if self._config.verbose:
print '~~$', 'cd', self._original_cwd
os.chdir(self._original_cwd)
if self._use_temp:
shutil.rmtree(self._use_temp)
@ -528,129 +306,6 @@ def revision_and_hash_from_partial(config, partial_hash):
return revision, git_hash
class GitBranchCLUpload(object):
"""Class to manage git branches and git-cl-upload.
This class allows one to create a new branch in a repository based
off of origin/master, make changes to the tree inside the
with-block, upload that new branch to Rietveld, restore the original
tree state, and delete the local copy of the new branch.
See roll_deps() for an example of use.
Constructor Args:
config: (roll_deps.DepsRollConfig) object containing options.
message: (string) the commit message, can be multiline.
set_brach_name: (string or none) if not None, the name of the
branch to use. If None, then use a temporary branch that
will be deleted.
Attributes:
issue: a string describing the codereview issue, after __exit__
has been called, othrwise, None.
Raises:
OSError: failed to execute git or git-cl.
subprocess.CalledProcessError: git returned unexpected status.
"""
# pylint: disable=I0011,R0903,R0902
def __init__(self, config, message, set_branch_name):
self._message = message
self._file_list = []
self._branch_name = set_branch_name
self._stash = None
self._original_branch = None
self._config = config
self.issue = None
def stage_for_commit(self, *paths):
"""Calls `git add ...` on each argument.
Args:
*paths: (list of strings) list of filenames to pass to `git add`.
"""
self._file_list.extend(paths)
def __enter__(self):
git = self._config.git
vsp = self._config.vsp
def branch_exists(branch):
"""Return true iff branch exists."""
return 0 == vsp.call([git, 'show-ref', '--quiet', branch])
def has_diff():
"""Return true iff repository has uncommited changes."""
return bool(vsp.call([git, 'diff', '--quiet', 'HEAD']))
self._stash = has_diff()
if self._stash:
vsp.check_call([git, 'stash', 'save'])
try:
full_branch = vsp.strip_output([git, 'symbolic-ref', 'HEAD'])
self._original_branch = full_branch.split('/')[-1]
except (subprocess.CalledProcessError,):
self._original_branch = vsp.strip_output(
[git, 'rev-parse', 'HEAD'])
if not self._branch_name:
self._branch_name = self._config.default_branch_name
if branch_exists(self._branch_name):
vsp.check_call([git, 'checkout', '-q', 'master'])
vsp.check_call([git, 'branch', '-D', self._branch_name])
vsp.check_call(
[git, 'checkout', '-q', '-b', self._branch_name, 'origin/master'])
def __exit__(self, etype, value, traceback):
# pylint: disable=I0011,R0912
git = self._config.git
vsp = self._config.vsp
svn_info = str(get_svn_revision(self._config, 'HEAD'))
for filename in self._file_list:
assert os.path.exists(filename)
vsp.check_call([git, 'add', filename])
vsp.check_call([git, 'commit', '-q', '-m', self._message])
git_cl = [git, 'cl', 'upload', '-f',
'--bypass-hooks', '--bypass-watchlists']
if self._config.cc_list:
git_cl.append('--cc=%s' % self._config.cc_list)
if self._config.reviewers_list:
git_cl.append('--reviewers=%s' % self._config.reviewers_list)
git_try = [git, 'cl', 'try', '--revision', svn_info]
git_try.extend([arg for bot in self._config.cl_bot_list
for arg in ('-b', bot)])
if self._config.skip_cl_upload:
print 'You should call:'
print ' cd %s' % os.getcwd()
VerboseSubprocess.print_subprocess_args(
' ', [git, 'checkout', self._branch_name])
VerboseSubprocess.print_subprocess_args(' ', git_cl)
if self._config.cl_bot_list:
VerboseSubprocess.print_subprocess_args(' ', git_try)
print
self.issue = ''
else:
vsp.check_call(git_cl)
self.issue = vsp.strip_output([git, 'cl', 'issue'])
if self._config.cl_bot_list:
vsp.check_call(git_try)
# deal with the aftermath of failed executions of this script.
if self._config.default_branch_name == self._original_branch:
self._original_branch = 'master'
vsp.check_call([git, 'checkout', '-q', self._original_branch])
if self._config.default_branch_name == self._branch_name:
vsp.check_call([git, 'branch', '-D', self._branch_name])
if self._stash:
vsp.check_call([git, 'stash', 'pop'])
def change_skia_deps(revision, git_hash, depspath):
"""Update the DEPS file.
@ -680,6 +335,59 @@ def change_skia_deps(revision, git_hash, depspath):
shutil.move(temp_file.name, depspath)
def git_cl_uploader(config, message, file_list):
"""Create a commit in the current git branch; upload via git-cl.
Assumes that you are already on the branch you want to be on.
Args:
config: (roll_deps.DepsRollConfig) object containing options.
message: (string) the commit message, can be multiline.
file_list: (list of strings) list of filenames to pass to `git add`.
Returns:
The output of `git cl issue`, if not config.skip_cl_upload, else ''.
"""
git, vsp = config.git, config.vsp
svn_info = str(get_svn_revision(config, 'HEAD'))
for filename in file_list:
assert os.path.exists(filename)
vsp.check_call([git, 'add', filename])
vsp.check_call([git, 'commit', '-q', '-m', message])
git_cl = [git, 'cl', 'upload', '-f',
'--bypass-hooks', '--bypass-watchlists']
if config.cc_list:
git_cl.append('--cc=%s' % config.cc_list)
if config.reviewers_list:
git_cl.append('--reviewers=%s' % config.reviewers_list)
git_try = [git, 'cl', 'try', '--revision', svn_info]
git_try.extend([arg for bot in config.cl_bot_list for arg in ('-b', bot)])
branch_name = git_utils.git_branch_name(vsp.verbose)
if config.skip_cl_upload:
space = ' '
print 'You should call:'
print '%scd %s' % (space, os.getcwd())
misc_utils.print_subprocess_args(space, [git, 'checkout', branch_name])
misc_utils.print_subprocess_args(space, git_cl)
if config.cl_bot_list:
misc_utils.print_subprocess_args(space, git_try)
print
return ''
else:
vsp.check_call(git_cl)
issue = vsp.strip_output([git, 'cl', 'issue'])
if config.cl_bot_list:
vsp.check_call(git_try)
return issue
def roll_deps(config, revision, git_hash):
"""Upload changed DEPS and a whitespace change.
@ -699,10 +407,10 @@ def roll_deps(config, revision, git_hash):
"""
git = config.git
with ChangeDir(config.chromium_path, config.verbose):
with misc_utils.ChangeDir(config.chromium_path, config.verbose):
config.vsp.check_call([git, 'fetch', '-q', 'origin'])
old_revision = ReSearch.search_within_output(
old_revision = misc_utils.ReSearch.search_within_output(
config.verbose, '"skia_revision": "(?P<return>[0-9]+)",', None,
[git, 'show', 'origin/master:DEPS'])
assert old_revision
@ -714,30 +422,34 @@ def roll_deps(config, revision, git_hash):
[git, 'show-ref', 'origin/master', '--hash'])
master_revision = get_svn_revision(config, 'origin/master')
branch = None
# master_hash[8] gives each whitespace CL a unique name.
if config.save_branches:
branch = 'control_%s' % master_hash[:8]
else:
branch = None
message = ('whitespace change %s\n\n'
'Chromium base revision: %d / %s\n\n'
'This CL was created by Skia\'s roll_deps.py script.\n'
) % (master_hash[:8], master_revision, master_hash[:8])
if config.save_branches:
branch = 'control_%s' % master_hash[:8]
with git_utils.ChangeGitBranch(branch, 'origin/master',
config.verbose):
branch = git_utils.git_branch_name(config.vsp.verbose)
codereview = GitBranchCLUpload(config, message, branch)
with codereview:
with open('build/whitespace_file.txt', 'a') as output_stream:
output_stream.write('\nCONTROL\n')
codereview.stage_for_commit('build/whitespace_file.txt')
whitespace_cl = codereview.issue
if branch:
whitespace_cl = '%s\n branch: %s' % (whitespace_cl, branch)
control_url = ReSearch.search_within_string(
codereview.issue, '(?P<return>https?://[^) ]+)', '?')
whitespace_cl = git_cl_uploader(
config, message, ['build/whitespace_file.txt'])
control_url = misc_utils.ReSearch.search_within_string(
whitespace_cl, '(?P<return>https?://[^) ]+)', '?')
if config.save_branches:
whitespace_cl = '%s\n branch: %s' % (whitespace_cl, branch)
if config.save_branches:
branch = 'roll_%d_%s' % (revision, master_hash[:8])
else:
branch = None
message = (
'roll skia DEPS to %d\n\n'
'Chromium base revision: %d / %s\n'
@ -749,13 +461,14 @@ def roll_deps(config, revision, git_hash):
'NOTRY=true\n'
% (revision, master_revision, master_hash[:8],
old_revision, revision, control_url))
codereview = GitBranchCLUpload(config, message, branch)
with codereview:
with git_utils.ChangeGitBranch(branch, 'origin/master',
config.verbose):
branch = git_utils.git_branch_name(config.vsp.verbose)
change_skia_deps(revision, git_hash, 'DEPS')
codereview.stage_for_commit('DEPS')
deps_cl = codereview.issue
if branch:
deps_cl = '%s\n branch: %s' % (deps_cl, branch)
deps_cl = git_cl_uploader(config, message, ['DEPS'])
if config.save_branches:
deps_cl = '%s\n branch: %s' % (deps_cl, branch)
return deps_cl, whitespace_cl
@ -814,7 +527,8 @@ def main(args):
option_parser.error('Must specify chromium_path.')
if not os.path.isdir(options.chromium_path):
option_parser.error('chromium_path must be a directory.')
if not test_git_executable(options.git_path):
if not git_utils.git_executable():
option_parser.error('Invalid git executable.')
config = DepsRollConfig(options)