Selective build clobbering feature (landmines.py and android build scripts).
Adds the ability for devs/troopers/etc. to set 'landmines' in the tree so that the build will selectively clobber when a builder moves over a revision with such a change. This cl has an basis landmines.py, and hooks the clobber mechanism to the android build scripts. The relevant cl which implements this for compile.py is here: https://chromiumcodereview.appspot.com/11234013/ I'm planning to also implement an informational invocation for gclient to let devs know about any potential landmines so they can decide if they need to clobber. This previously attempted to land as: https://chromiumcodereview.appspot.com/11175016 R=ilevy@chromium.org,maruel@chromium.org BUG=121897 Review URL: https://chromiumcodereview.appspot.com/11377141 git-svn-id: http://src.chromium.org/svn/trunk/src/build@167595 4ff67af0-8c30-449e-8e8b-ad334ec8d88c
This commit is contained in:
iannucci@chromium.org
Родитель
64562a877b
Коммит
693c711ddb
|
|
@ -47,18 +47,6 @@ function bb_baseline_setup {
|
|||
shift
|
||||
cd $SRC_ROOT
|
||||
|
||||
if [[ $BUILDBOT_CLOBBER ]]; then
|
||||
echo "@@@BUILD_STEP Clobber@@@"
|
||||
# Sdk key expires, delete android folder.
|
||||
# crbug.com/145860
|
||||
rm -rf ~/.android
|
||||
rm -rf "${SRC_ROOT}"/out
|
||||
if [ -e "${SRC_ROOT}"/out ] ; then
|
||||
echo "Clobber appeared to fail? ${SRC_ROOT}/out still exists."
|
||||
echo "@@@STEP_WARNINGS@@@"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "@@@BUILD_STEP Environment setup@@@"
|
||||
bb_parse_args "$@"
|
||||
|
||||
|
|
@ -68,17 +56,44 @@ function bb_baseline_setup {
|
|||
fi
|
||||
export GOMA_DIR=/b/build/goma
|
||||
. build/android/envsetup.sh
|
||||
adb kill-server
|
||||
adb start-server
|
||||
}
|
||||
|
||||
function bb_compile_setup {
|
||||
local extra_gyp_defines="$(bb_get_json_prop "$FACTORY_PROPERTIES" \
|
||||
extra_gyp_defines)"
|
||||
export GYP_DEFINES+=" fastbuild=1 $extra_gyp_defines"
|
||||
if echo $extra_gyp_defines | grep -q clang; then
|
||||
unset CXX_target
|
||||
fi
|
||||
|
||||
adb kill-server
|
||||
adb start-server
|
||||
|
||||
local build_path="${SRC_ROOT}/out/${BUILDTYPE}"
|
||||
local landmines_triggered_path="$build_path/.landmines_triggered"
|
||||
python "$SRC_ROOT/build/landmines.py"
|
||||
|
||||
if [[ $BUILDBOT_CLOBBER || -f "$landmines_triggered_path" ]]; then
|
||||
echo "@@@BUILD_STEP Clobber@@@"
|
||||
|
||||
if [[ -z $BUILDBOT_CLOBBER ]]; then
|
||||
echo "Clobbering due to triggered landmines: "
|
||||
cat "$landmines_triggered_path"
|
||||
else
|
||||
# Also remove all the files under out/ on an explicit clobber
|
||||
find "${SRC_ROOT}/out" -maxdepth 1 -type f -exec rm -f {} +
|
||||
fi
|
||||
|
||||
# Sdk key expires, delete android folder.
|
||||
# crbug.com/145860
|
||||
rm -rf ~/.android
|
||||
rm -rf "$build_path"
|
||||
if [[ -e $build_path ]] ; then
|
||||
echo "Clobber appeared to fail? $build_path still exists."
|
||||
echo "@@@STEP_WARNINGS@@@"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
function bb_compile_setup {
|
||||
bb_setup_goma_internal
|
||||
# Should be called only after envsetup is done.
|
||||
gclient runhooks
|
||||
|
|
|
|||
36
gyp_chromium
36
gyp_chromium
|
|
@ -8,6 +8,7 @@
|
|||
# is invoked by Chromium beyond what can be done in the gclient hooks.
|
||||
|
||||
import glob
|
||||
import gyp_helper
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
|
|
@ -44,36 +45,6 @@ if sys.platform == 'win32':
|
|||
else:
|
||||
psyco = None
|
||||
|
||||
def apply_gyp_environment(file_path=None):
|
||||
"""
|
||||
Reads in a *.gyp_env file and applies the valid keys to os.environ.
|
||||
"""
|
||||
if not file_path or not os.path.exists(file_path):
|
||||
return
|
||||
file_contents = open(file_path).read()
|
||||
try:
|
||||
file_data = eval(file_contents, {'__builtins__': None}, None)
|
||||
except SyntaxError, e:
|
||||
e.filename = os.path.abspath(file_path)
|
||||
raise
|
||||
supported_vars = ( 'CC',
|
||||
'CHROMIUM_GYP_FILE',
|
||||
'CHROMIUM_GYP_SYNTAX_CHECK',
|
||||
'CXX',
|
||||
'GYP_DEFINES',
|
||||
'GYP_GENERATOR_FLAGS',
|
||||
'GYP_GENERATOR_OUTPUT',
|
||||
'GYP_GENERATORS', )
|
||||
for var in supported_vars:
|
||||
val = file_data.get(var)
|
||||
if val:
|
||||
if var in os.environ:
|
||||
print 'INFO: Environment value for "%s" overrides value in %s.' % (
|
||||
var, os.path.abspath(file_path)
|
||||
)
|
||||
else:
|
||||
os.environ[var] = val
|
||||
|
||||
def additional_include_files(args=[]):
|
||||
"""
|
||||
Returns a list of additional (.gypi) files to include, without
|
||||
|
|
@ -124,10 +95,7 @@ if __name__ == '__main__':
|
|||
p.communicate()
|
||||
sys.exit(p.returncode)
|
||||
|
||||
if 'SKIP_CHROMIUM_GYP_ENV' not in os.environ:
|
||||
# Update the environment based on chromium.gyp_env
|
||||
gyp_env_path = os.path.join(os.path.dirname(chrome_src), 'chromium.gyp_env')
|
||||
apply_gyp_environment(gyp_env_path)
|
||||
gyp_helper.apply_chromium_gyp_env()
|
||||
|
||||
# This could give false positives since it doesn't actually do real option
|
||||
# parsing. Oh well.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,50 @@
|
|||
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
# This file helps gyp_chromium and landmines correctly set up the gyp
|
||||
# environment from chromium.gyp_env on disk
|
||||
|
||||
import os
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
|
||||
CHROME_SRC = os.path.dirname(SCRIPT_DIR)
|
||||
|
||||
|
||||
def apply_gyp_environment_from_file(file_path):
|
||||
"""Reads in a *.gyp_env file and applies the valid keys to os.environ."""
|
||||
if not os.path.exists(file_path):
|
||||
return
|
||||
with open(file_path) as f:
|
||||
file_contents = f.read()
|
||||
try:
|
||||
file_data = eval(file_contents, {'__builtins__': None}, None)
|
||||
except SyntaxError, e:
|
||||
e.filename = os.path.abspath(file_path)
|
||||
raise
|
||||
supported_vars = (
|
||||
'CC',
|
||||
'CHROMIUM_GYP_FILE',
|
||||
'CHROMIUM_GYP_SYNTAX_CHECK',
|
||||
'CXX',
|
||||
'GYP_DEFINES',
|
||||
'GYP_GENERATOR_FLAGS',
|
||||
'GYP_GENERATOR_OUTPUT',
|
||||
'GYP_GENERATORS',
|
||||
)
|
||||
for var in supported_vars:
|
||||
file_val = file_data.get(var)
|
||||
if file_val:
|
||||
if var in os.environ:
|
||||
print 'INFO: Environment value for "%s" overrides value in %s.' % (
|
||||
var, os.path.abspath(file_path)
|
||||
)
|
||||
else:
|
||||
os.environ[var] = file_val
|
||||
|
||||
|
||||
def apply_chromium_gyp_env():
|
||||
if 'SKIP_CHROMIUM_GYP_ENV' not in os.environ:
|
||||
# Update the environment based on chromium.gyp_env
|
||||
path = os.path.join(os.path.dirname(CHROME_SRC), 'chromium.gyp_env')
|
||||
apply_gyp_environment_from_file(path)
|
||||
|
|
@ -0,0 +1,210 @@
|
|||
#!/usr/bin/env python
|
||||
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
"""
|
||||
This file holds a list of reasons why a particular build needs to be clobbered
|
||||
(or a list of 'landmines').
|
||||
|
||||
This script runs every build as a hook. If it detects that the build should
|
||||
be clobbered, it will touch the file <build_dir>/.landmine_triggered. The
|
||||
various build scripts will then check for the presence of this file and clobber
|
||||
accordingly. The script will also emit the reasons for the clobber to stdout.
|
||||
|
||||
A landmine is tripped when a builder checks out a different revision, and the
|
||||
diff between the new landmines and the old ones is non-null. At this point, the
|
||||
build is clobbered.
|
||||
"""
|
||||
|
||||
import difflib
|
||||
import functools
|
||||
import gyp_helper
|
||||
import os
|
||||
import shlex
|
||||
import sys
|
||||
import time
|
||||
|
||||
SRC_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
||||
|
||||
def memoize(default=None):
|
||||
"""This decorator caches the return value of a parameterless pure function"""
|
||||
def memoizer(func):
|
||||
val = []
|
||||
@functools.wraps(func)
|
||||
def inner():
|
||||
if not val:
|
||||
ret = func()
|
||||
val.append(ret if ret is not None else default)
|
||||
print '%s -> %r' % (func.__name__, val[0])
|
||||
return val[0]
|
||||
return inner
|
||||
return memoizer
|
||||
|
||||
|
||||
@memoize()
|
||||
def IsWindows():
|
||||
return sys.platform.startswith('win') or sys.platform == 'cygwin'
|
||||
|
||||
|
||||
@memoize()
|
||||
def IsLinux():
|
||||
return sys.platform.startswith('linux')
|
||||
|
||||
|
||||
@memoize()
|
||||
def IsMac():
|
||||
return sys.platform.startswith('darwin')
|
||||
|
||||
|
||||
@memoize()
|
||||
def gyp_defines():
|
||||
"""Parses and returns GYP_DEFINES env var as a dictionary."""
|
||||
return dict(arg.split('=', 1)
|
||||
for arg in shlex.split(os.environ.get('GYP_DEFINES', '')))
|
||||
|
||||
|
||||
@memoize()
|
||||
def distributor():
|
||||
"""
|
||||
Returns a string which is the distributed build engine in use (if any).
|
||||
Possible values: 'goma', 'ib', ''
|
||||
"""
|
||||
if 'goma' in gyp_defines():
|
||||
return 'goma'
|
||||
elif IsWindows():
|
||||
if 'CHROME_HEADLESS' in os.environ:
|
||||
return 'ib' # use (win and !goma and headless) as approximation of ib
|
||||
|
||||
|
||||
@memoize()
|
||||
def platform():
|
||||
"""
|
||||
Returns a string representing the platform this build is targetted for.
|
||||
Possible values: 'win', 'mac', 'linux', 'ios', 'android'
|
||||
"""
|
||||
if 'OS' in gyp_defines():
|
||||
if 'android' in gyp_defines()['OS']:
|
||||
return 'android'
|
||||
else:
|
||||
return gyp_defines()['OS']
|
||||
elif IsWindows():
|
||||
return 'win'
|
||||
elif IsLinux():
|
||||
return 'linux'
|
||||
else:
|
||||
return 'mac'
|
||||
|
||||
|
||||
@memoize()
|
||||
def builder():
|
||||
"""
|
||||
Returns a string representing the build engine (not compiler) to use.
|
||||
Possible values: 'make', 'ninja', 'xcode', 'msvs', 'scons'
|
||||
"""
|
||||
if 'GYP_GENERATORS' in os.environ:
|
||||
# for simplicity, only support the first explicit generator
|
||||
generator = os.environ['GYP_GENERATORS'].split(',')[0]
|
||||
if generator.endswith('-android'):
|
||||
return generator.split('-')[0]
|
||||
else:
|
||||
return generator
|
||||
else:
|
||||
if platform() == 'android':
|
||||
# Good enough for now? Do any android bots use make?
|
||||
return 'ninja'
|
||||
elif platform() == 'ios':
|
||||
return 'xcode'
|
||||
elif IsWindows():
|
||||
return 'msvs'
|
||||
elif IsLinux():
|
||||
return 'make'
|
||||
elif IsMac():
|
||||
return 'xcode'
|
||||
else:
|
||||
assert False, 'Don\'t know what builder we\'re using!'
|
||||
|
||||
|
||||
def get_landmines(target):
|
||||
"""
|
||||
ALL LANDMINES ARE DEFINED HERE.
|
||||
target is 'Release' or 'Debug'
|
||||
"""
|
||||
landmines = []
|
||||
add = lambda item: landmines.append(item + '\n')
|
||||
|
||||
if (distributor() == 'goma' and platform() == 'win32' and
|
||||
builder() == 'ninja'):
|
||||
add('Need to clobber winja goma due to backend cwd cache fix.')
|
||||
|
||||
return landmines
|
||||
|
||||
|
||||
def get_target_build_dir(build_tool, target, is_iphone=False):
|
||||
"""
|
||||
Returns output directory absolute path dependent on build and targets.
|
||||
Examples:
|
||||
r'c:\b\build\slave\win\build\src\out\Release'
|
||||
'/mnt/data/b/build/slave/linux/build/src/out/Debug'
|
||||
'/b/build/slave/ios_rel_device/build/src/xcodebuild/Release-iphoneos'
|
||||
|
||||
Keep this function in sync with tools/build/scripts/slave/compile.py
|
||||
"""
|
||||
ret = None
|
||||
if build_tool == 'xcode':
|
||||
ret = os.path.join(SRC_DIR, 'xcodebuild',
|
||||
target + ('-iphoneos' if is_iphone else ''))
|
||||
elif build_tool == 'make':
|
||||
ret = os.path.join(SRC_DIR, 'out', target)
|
||||
elif build_tool == 'ninja':
|
||||
ret = os.path.join(SRC_DIR, 'out', target)
|
||||
elif build_tool == 'msvs':
|
||||
ret = os.path.join(SRC_DIR, 'build', target)
|
||||
elif build_tool == 'scons':
|
||||
ret = os.path.join(SRC_DIR, 'sconsbuild', target)
|
||||
else:
|
||||
raise NotImplementedError()
|
||||
return os.path.abspath(ret)
|
||||
|
||||
|
||||
def main(argv):
|
||||
if len(argv) > 1:
|
||||
print('Unknown arguments %s' % argv[1:])
|
||||
return 1
|
||||
|
||||
gyp_helper.apply_chromium_gyp_env()
|
||||
|
||||
for target in ('Debug', 'Release'):
|
||||
out_dir = get_target_build_dir(builder(), target,
|
||||
platform() == 'ios')
|
||||
|
||||
landmines_path = os.path.join(out_dir, '.landmines')
|
||||
if not os.path.exists(out_dir):
|
||||
os.makedirs(out_dir)
|
||||
|
||||
new_landmines = get_landmines(target)
|
||||
|
||||
if not os.path.exists(landmines_path):
|
||||
with open(landmines_path, 'w') as f:
|
||||
f.writelines(new_landmines)
|
||||
else:
|
||||
triggered = os.path.join(out_dir, '.landmines_triggered')
|
||||
with open(landmines_path, 'r') as f:
|
||||
old_landmines = f.readlines()
|
||||
if old_landmines != new_landmines:
|
||||
old_date = time.ctime(os.stat(landmines_path).st_ctime)
|
||||
diff = difflib.unified_diff(old_landmines, new_landmines,
|
||||
fromfile='old_landmines', tofile='new_landmines',
|
||||
fromfiledate=old_date, tofiledate=time.ctime(), n=0)
|
||||
|
||||
with open(triggered, 'w') as f:
|
||||
f.writelines(diff)
|
||||
elif os.path.exists(triggered):
|
||||
# Remove false triggered landmines.
|
||||
os.remove(triggered)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main(sys.argv))
|
||||
Загрузка…
Ссылка в новой задаче