From f55feb4fa0fd64117fafec3c3c20405f928240d6 Mon Sep 17 00:00:00 2001 From: "scottmg@chromium.org" Date: Thu, 14 Aug 2014 14:03:30 +0000 Subject: [PATCH] Make landmines work on local builds too Moves (some of) gyp environment setup out of gyp_chromium into separate module, and shares that between gyp_chromium and landmines.py. landmines.py is added as the first entry in DEPS hooks so that it can clobber the entire build directory before running other hooks that extract/generate into the build dir. Reland with fix for ios, and for clean pull. R=iannucci@chromium.org BUG=400011 Committed: https://src.chromium.org/viewvc/chrome?view=rev&revision=289099 Review URL: https://codereview.chromium.org/457003004 git-svn-id: http://src.chromium.org/svn/trunk/src/build@289546 4ff67af0-8c30-449e-8e8b-ad334ec8d88c --- get_landmines.py | 15 ++---------- gyp_chromium | 33 ++++---------------------- gyp_environment.py | 33 ++++++++++++++++++++++++++ landmines.py | 59 ++++++++++++++++++++++------------------------ vs_toolchain.py | 2 ++ 5 files changed, 69 insertions(+), 73 deletions(-) create mode 100644 gyp_environment.py diff --git a/get_landmines.py b/get_landmines.py index c9e877348..3aac6e760 100755 --- a/get_landmines.py +++ b/get_landmines.py @@ -8,7 +8,6 @@ This file emits the list of reasons why a particular build needs to be clobbered (or a list of 'landmines'). """ -import optparse import sys import landmine_utils @@ -21,10 +20,9 @@ gyp_msvs_version = landmine_utils.gyp_msvs_version platform = landmine_utils.platform -def print_landmines(target): +def print_landmines(): """ ALL LANDMINES ARE EMITTED FROM HERE. - target can be one of {'Release', 'Debug', 'Debug_x64', 'Release_x64'}. """ if (distributor() == 'goma' and platform() == 'win32' and builder() == 'ninja'): @@ -60,16 +58,7 @@ def print_landmines(target): def main(): - parser = optparse.OptionParser() - parser.add_option('-t', '--target', - help=='Target for which the landmines have to be emitted') - - options, args = parser.parse_args() - - if args: - parser.error('Unknown arguments %s' % args) - - print_landmines(options.target) + print_landmines() return 0 diff --git a/gyp_chromium b/gyp_chromium index ccae75800..ff84bb315 100755 --- a/gyp_chromium +++ b/gyp_chromium @@ -8,7 +8,7 @@ # is invoked by Chromium beyond what can be done in the gclient hooks. import glob -import gyp_helper +import gyp_environment import os import re import shlex @@ -197,10 +197,6 @@ if __name__ == '__main__': args.append('-Ganalyzer_output_path=' + args.pop(0)) if int(os.environ.get('GYP_CHROMIUM_NO_ACTION', 0)): - # Check for landmines (reasons to clobber the build) in any case. - print 'Running build/landmines.py...' - subprocess.check_call( - [sys.executable, os.path.join(script_dir, 'landmines.py')]) print 'Skipping gyp_chromium due to GYP_CHROMIUM_NO_ACTION env var.' sys.exit(0) @@ -225,8 +221,6 @@ if __name__ == '__main__': p.communicate() sys.exit(p.returncode) - gyp_helper.apply_chromium_gyp_env() - # This could give false positives since it doesn't actually do real option # parsing. Oh well. gyp_file_specified = False @@ -235,6 +229,8 @@ if __name__ == '__main__': gyp_file_specified = True break + gyp_environment.SetEnvironment() + # If we didn't get a file, check an env var, and then fall back to # assuming 'all.gyp' from the same directory as the script. if not gyp_file_specified: @@ -264,21 +260,6 @@ if __name__ == '__main__': print 'Error: make gyp generator not supported (check GYP_GENERATORS).' sys.exit(1) - # Default to ninja on linux and windows, but only if no generator has - # explicitly been set. - # Also default to ninja on mac, but only when not building chrome/ios. - # . -f / --format has precedence over the env var, no need to check for it - # . set the env var only if it hasn't been set yet - # . chromium.gyp_env has been applied to os.environ at this point already - if sys.platform.startswith(('linux', 'win', 'freebsd')) and \ - not os.environ.get('GYP_GENERATORS'): - os.environ['GYP_GENERATORS'] = 'ninja' - elif sys.platform == 'darwin' and not os.environ.get('GYP_GENERATORS') and \ - not 'OS=ios' in os.environ.get('GYP_DEFINES', []): - os.environ['GYP_GENERATORS'] = 'ninja' - - vs2013_runtime_dll_dirs = vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs() - # If CHROMIUM_GYP_SYNTAX_CHECK is set to 1, it will invoke gyp with --check # to enfore syntax checking. syntax_check = os.environ.get('CHROMIUM_GYP_SYNTAX_CHECK') @@ -322,13 +303,7 @@ if __name__ == '__main__': gyp_rc = gyp.main(args) if not use_analyzer: - # Check for landmines (reasons to clobber the build). This must be run here, - # rather than a separate runhooks step so that any environment modifications - # from above are picked up. - print 'Running build/landmines.py...' - subprocess.check_call( - [sys.executable, os.path.join(script_dir, 'landmines.py')]) - + vs2013_runtime_dll_dirs = vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs() if vs2013_runtime_dll_dirs: x64_runtime, x86_runtime = vs2013_runtime_dll_dirs vs_toolchain.CopyVsRuntimeDlls( diff --git a/gyp_environment.py b/gyp_environment.py new file mode 100644 index 000000000..fb50645d5 --- /dev/null +++ b/gyp_environment.py @@ -0,0 +1,33 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Sets up various automatic gyp environment variables. These are used by +gyp_chromium and landmines.py which run at different stages of runhooks. To +make sure settings are consistent between them, all setup should happen here. +""" + +import gyp_helper +import os +import sys +import vs_toolchain + +def SetEnvironment(): + """Sets defaults for GYP_* variables.""" + gyp_helper.apply_chromium_gyp_env() + + # Default to ninja on linux and windows, but only if no generator has + # explicitly been set. + # Also default to ninja on mac, but only when not building chrome/ios. + # . -f / --format has precedence over the env var, no need to check for it + # . set the env var only if it hasn't been set yet + # . chromium.gyp_env has been applied to os.environ at this point already + if sys.platform.startswith(('linux', 'win', 'freebsd')) and \ + not os.environ.get('GYP_GENERATORS'): + os.environ['GYP_GENERATORS'] = 'ninja' + elif sys.platform == 'darwin' and not os.environ.get('GYP_GENERATORS') and \ + not 'OS=ios' in os.environ.get('GYP_DEFINES', []): + os.environ['GYP_GENERATORS'] = 'ninja' + + vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs() diff --git a/landmines.py b/landmines.py index 220b8a798..c31bac790 100755 --- a/landmines.py +++ b/landmines.py @@ -4,10 +4,8 @@ # found in the LICENSE file. """ -This script runs every build as a hook. If it detects that the build should -be clobbered, it will touch the file /.landmine_triggered. The -various build scripts will then check for the presence of this file and clobber -accordingly. The script will also emit the reasons for the clobber to stdout. +This script runs every build as the first hook (See DEPS). If it detects that +the build should be clobbered, it will remove the build directory. A landmine is tripped when a builder checks out a different revision, and the diff between the new landmines and the old ones is non-null. At this point, the @@ -16,9 +14,11 @@ build is clobbered. import difflib import errno +import gyp_environment import logging import optparse import os +import shutil import sys import subprocess import time @@ -29,35 +29,32 @@ import landmine_utils SRC_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -def get_target_build_dir(build_tool, target, is_iphone=False): +def get_build_dir(build_tool, is_iphone=False): """ Returns output directory absolute path dependent on build and targets. Examples: - r'c:\b\build\slave\win\build\src\out\Release' - '/mnt/data/b/build/slave/linux/build/src/out/Debug' - '/b/build/slave/ios_rel_device/build/src/xcodebuild/Release-iphoneos' + r'c:\b\build\slave\win\build\src\out' + '/mnt/data/b/build/slave/linux/build/src/out' + '/b/build/slave/ios_rel_device/build/src/xcodebuild' Keep this function in sync with tools/build/scripts/slave/compile.py """ ret = None if build_tool == 'xcode': - ret = os.path.join(SRC_DIR, 'xcodebuild', - target + ('-iphoneos' if is_iphone else '')) + ret = os.path.join(SRC_DIR, 'xcodebuild') elif build_tool in ['make', 'ninja', 'ninja-ios']: # TODO: Remove ninja-ios. - ret = os.path.join(SRC_DIR, 'out', target) + ret = os.path.join(SRC_DIR, 'out') elif build_tool in ['msvs', 'vs', 'ib']: - ret = os.path.join(SRC_DIR, 'build', target) + ret = os.path.join(SRC_DIR, 'build') else: raise NotImplementedError('Unexpected GYP_GENERATORS (%s)' % build_tool) return os.path.abspath(ret) -def set_up_landmines(target, new_landmines): +def clobber_if_necessary(new_landmines): """Does the work of setting, planting, and triggering landmines.""" - out_dir = get_target_build_dir(landmine_utils.builder(), target, - landmine_utils.platform() == 'ios') - - landmines_path = os.path.join(out_dir, '.landmines') + out_dir = get_build_dir(landmine_utils.builder()) + landmines_path = os.path.normpath(os.path.join(out_dir, '..', '.landmines')) try: os.makedirs(out_dir) except OSError as e: @@ -65,7 +62,6 @@ def set_up_landmines(target, new_landmines): pass if os.path.exists(landmines_path): - triggered = os.path.join(out_dir, '.landmines_triggered') with open(landmines_path, 'r') as f: old_landmines = f.readlines() if old_landmines != new_landmines: @@ -73,12 +69,13 @@ def set_up_landmines(target, new_landmines): diff = difflib.unified_diff(old_landmines, new_landmines, fromfile='old_landmines', tofile='new_landmines', fromfiledate=old_date, tofiledate=time.ctime(), n=0) + sys.stdout.write('Clobbering due to:\n') + sys.stdout.writelines(diff) - with open(triggered, 'w') as f: - f.writelines(diff) - elif os.path.exists(triggered): - # Remove false triggered landmines. - os.remove(triggered) + # Clobber. + shutil.rmtree(out_dir) + + # Save current set of landmines for next time. with open(landmines_path, 'w') as f: f.writelines(new_landmines) @@ -119,14 +116,14 @@ def main(): if landmine_utils.builder() in ('dump_dependency_json', 'eclipse'): return 0 - for target in ('Debug', 'Release', 'Debug_x64', 'Release_x64'): - landmines = [] - for s in landmine_scripts: - proc = subprocess.Popen([sys.executable, s, '-t', target], - stdout=subprocess.PIPE) - output, _ = proc.communicate() - landmines.extend([('%s\n' % l.strip()) for l in output.splitlines()]) - set_up_landmines(target, landmines) + gyp_environment.SetEnvironment() + + landmines = [] + for s in landmine_scripts: + proc = subprocess.Popen([sys.executable, s], stdout=subprocess.PIPE) + output, _ = proc.communicate() + landmines.extend([('%s\n' % l.strip()) for l in output.splitlines()]) + clobber_if_necessary(landmines) return 0 diff --git a/vs_toolchain.py b/vs_toolchain.py index b039ed01c..735827e94 100644 --- a/vs_toolchain.py +++ b/vs_toolchain.py @@ -30,6 +30,8 @@ def SetEnvironmentAndGetRuntimeDllDirs(): depot_tools_win_toolchain = \ bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', '1'))) if sys.platform in ('win32', 'cygwin') and depot_tools_win_toolchain: + if not os.path.exists(json_data_file): + Update() with open(json_data_file, 'r') as tempf: toolchain_data = json.load(tempf)