From 2cd87fa675cd084b28ea4ff713c4f7f53547f23c Mon Sep 17 00:00:00 2001 From: Mike Shal Date: Mon, 16 Jun 2014 13:17:21 -0400 Subject: [PATCH] Bug 978211 - enable MOZ_AUTOMATION_* flags in linux mozconfigs; r=glandium --- browser/config/mozconfigs/linux32/debug | 3 + browser/config/mozconfigs/linux64/debug | 3 + .../linux64/debug-static-analysis-clang | 7 +++ .../mozconfigs/linux64/nightly-nonunified | 5 ++ build/gen_mach_buildprops.py | 30 ++++++--- build/moz-automation.mk | 62 ++++++++++--------- build/mozconfig.automation | 21 +++++++ build/mozconfig.common | 2 + build/unix/mozconfig.asan | 6 ++ build/unix/mozconfig.linux | 5 ++ 10 files changed, 105 insertions(+), 39 deletions(-) create mode 100644 build/mozconfig.automation diff --git a/browser/config/mozconfigs/linux32/debug b/browser/config/mozconfigs/linux32/debug index 8ea940f70c3b..c3a640514926 100644 --- a/browser/config/mozconfigs/linux32/debug +++ b/browser/config/mozconfigs/linux32/debug @@ -4,6 +4,9 @@ ac_add_options --enable-signmar ac_add_options --disable-unified-compilation +MOZ_AUTOMATION_TALOS_SENDCHANGE=0 +MOZ_AUTOMATION_L10N_CHECK=0 + . $topsrcdir/build/unix/mozconfig.linux32 # Needed to enable breakpad in application.ini diff --git a/browser/config/mozconfigs/linux64/debug b/browser/config/mozconfigs/linux64/debug index cdbe7f7d2b41..f232b9d0186f 100644 --- a/browser/config/mozconfigs/linux64/debug +++ b/browser/config/mozconfigs/linux64/debug @@ -2,6 +2,9 @@ ac_add_options --enable-debug ac_add_options --enable-dmd ac_add_options --enable-signmar +MOZ_AUTOMATION_TALOS_SENDCHANGE=0 +MOZ_AUTOMATION_L10N_CHECK=0 + . $topsrcdir/build/unix/mozconfig.linux # Needed to enable breakpad in application.ini diff --git a/browser/config/mozconfigs/linux64/debug-static-analysis-clang b/browser/config/mozconfigs/linux64/debug-static-analysis-clang index a7a64d2fa4a3..e393eef08d5a 100644 --- a/browser/config/mozconfigs/linux64/debug-static-analysis-clang +++ b/browser/config/mozconfigs/linux64/debug-static-analysis-clang @@ -1,3 +1,10 @@ +MOZ_AUTOMATION_BUILD_SYMBOLS=0 +MOZ_AUTOMATION_PACKAGE_TESTS=0 +MOZ_AUTOMATION_UNITTEST_SENDCHANGE=0 +MOZ_AUTOMATION_TALOS_SENDCHANGE=0 +MOZ_AUTOMATION_L10N_CHECK=0 +MOZ_AUTOMATION_CHECK=0 + . "$topsrcdir/build/mozconfig.common" ac_add_options --enable-debug diff --git a/browser/config/mozconfigs/linux64/nightly-nonunified b/browser/config/mozconfigs/linux64/nightly-nonunified index 2dccd17f3ce9..7394b7cf6e73 100644 --- a/browser/config/mozconfigs/linux64/nightly-nonunified +++ b/browser/config/mozconfigs/linux64/nightly-nonunified @@ -1,3 +1,8 @@ +MOZ_AUTOMATION_UPLOAD=0 +MOZ_AUTOMATION_UNITTEST_SENDCHANGE=0 +MOZ_AUTOMATION_TALOS_SENDCHANGE=0 +MOZ_AUTOMATION_CHECK=0 + . "$topsrcdir/browser/config/mozconfigs/linux64/nightly" ac_add_options --disable-unified-compilation diff --git a/build/gen_mach_buildprops.py b/build/gen_mach_buildprops.py index 2939854ed563..30da66581b1a 100644 --- a/build/gen_mach_buildprops.py +++ b/build/gen_mach_buildprops.py @@ -4,7 +4,12 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -import sys, os, sha, json, re +import sys +import os +import sha +import json +import re +import errno from argparse import ArgumentParser def getFileHashAndSize(filename): @@ -51,15 +56,20 @@ def getUrlProperties(filename): url_re = re.compile(r'''^(https?://.*?\.(?:tar\.bz2|dmg|zip|apk|rpm|mar|tar\.gz))$''') properties = {} - with open(filename) as f: - for line in f: - m = url_re.match(line) - if m: - m = m.group(1) - for prop, condition in property_conditions: - if condition(m): - properties.update({prop: m}) - break + try: + with open(filename) as f: + for line in f: + m = url_re.match(line) + if m: + m = m.group(1) + for prop, condition in property_conditions: + if condition(m): + properties.update({prop: m}) + break + except IOError as e: + if e.errno != errno.ENOENT: + raise + properties = {prop: 'UNKNOWN' for prop, condition in property_conditions} return properties if __name__ == '__main__': diff --git a/build/moz-automation.mk b/build/moz-automation.mk index 13623d6eb422..147875e5c26c 100644 --- a/build/moz-automation.mk +++ b/build/moz-automation.mk @@ -17,43 +17,35 @@ include $(topsrcdir)/toolkit/mozapps/installer/package-name.mk # the uploaded files. AUTOMATION_UPLOAD_OUTPUT = $(DIST)/automation-upload.txt +# Helper variables to convert from MOZ_AUTOMATION_* variables to the +# corresponding the make target +tier_BUILD_SYMBOLS = buildsymbols +tier_CHECK = check +tier_L10N_CHECK = l10n-check +tier_PACKAGE = package +tier_PACKAGE_TESTS = package-tests +tier_UPDATE_PACKAGING = update-packaging +tier_UPLOAD_SYMBOLS = uploadsymbols +tier_UPLOAD = upload + # Automation build steps. Everything in MOZ_AUTOMATION_TIERS also gets used in # TIERS for mach display. As such, the MOZ_AUTOMATION_TIERS are roughly sorted # here in the order that they will be executed (since mach doesn't know of the # dependencies between them). - -MOZ_AUTOMATION_TIERS += package-tests -MOZ_AUTOMATION_TIERS += buildsymbols - -ifdef NIGHTLY_BUILD -MOZ_AUTOMATION_TIERS += uploadsymbols -endif - -ifdef MOZ_UPDATE_PACKAGING -MOZ_AUTOMATION_TIERS += update-packaging -automation/upload: automation/update-packaging -automation/update-packaging: automation/package -endif - -MOZ_AUTOMATION_TIERS += package -MOZ_AUTOMATION_TIERS += check - -ifndef MOZ_ASAN -MOZ_AUTOMATION_TIERS += l10n-check -automation/l10n-check: automation/package -endif - -MOZ_AUTOMATION_TIERS += upload - -automation/build: $(addprefix automation/,$(MOZ_AUTOMATION_TIERS)) - $(PYTHON) $(topsrcdir)/build/gen_mach_buildprops.py --complete-mar-file $(DIST)/$(COMPLETE_MAR) --upload-output $(AUTOMATION_UPLOAD_OUTPUT) +moz_automation_symbols = PACKAGE_TESTS BUILD_SYMBOLS UPLOAD_SYMBOLS PACKAGE UPDATE_PACKAGING CHECK L10N_CHECK UPLOAD +MOZ_AUTOMATION_TIERS := $(foreach sym,$(moz_automation_symbols),$(if $(filter 1,$(MOZ_AUTOMATION_$(sym))),$(tier_$(sym)))) # Dependencies between automation build steps automation/uploadsymbols: automation/buildsymbols +automation/update-packaging: automation/package + +automation/l10n-check: automation/package + automation/upload: automation/package automation/upload: automation/package-tests automation/upload: automation/buildsymbols +automation/upload: automation/update-packaging # automation/package and automation/check should depend on build (which is # implicit due to the way client.mk invokes automation/build), but buildsymbols @@ -61,6 +53,9 @@ automation/upload: automation/buildsymbols automation/package: automation/buildsymbols automation/check: automation/buildsymbols +automation/build: $(addprefix automation/,$(MOZ_AUTOMATION_TIERS)) + $(PYTHON) $(topsrcdir)/build/gen_mach_buildprops.py --complete-mar-file $(DIST)/$(COMPLETE_MAR) --upload-output $(AUTOMATION_UPLOAD_OUTPUT) + # make check runs with the keep-going flag so we can see all the failures AUTOMATION_EXTRA_CMDLINE-check = -k @@ -68,7 +63,16 @@ AUTOMATION_EXTRA_CMDLINE-check = -k # properties. AUTOMATION_EXTRA_CMDLINE-upload = 2>&1 | tee $(AUTOMATION_UPLOAD_OUTPUT) +# The commands only run if the corresponding MOZ_AUTOMATION_* variable is +# enabled. This means, for example, if we enable MOZ_AUTOMATION_UPLOAD, then +# 'buildsymbols' will only run if MOZ_AUTOMATION_BUILD_SYMBOLS is also set. +# However, the target automation/buildsymbols will still be executed in this +# case because it is a prerequisite of automation/upload. +define automation_commands +$(call BUILDSTATUS,TIER_START $1) +@$(MAKE) $1 $(AUTOMATION_EXTRA_CMDLINE-$1) +$(call BUILDSTATUS,TIER_FINISH $1) +endef + automation/%: - $(call BUILDSTATUS,TIER_START $*) - @$(MAKE) $* $(AUTOMATION_EXTRA_CMDLINE-$*) - $(call BUILDSTATUS,TIER_FINISH $*) + $(if $(filter $*,$(MOZ_AUTOMATION_TIERS)),$(call automation_commands,$*)) diff --git a/build/mozconfig.automation b/build/mozconfig.automation new file mode 100644 index 000000000000..63c54a5a06ed --- /dev/null +++ b/build/mozconfig.automation @@ -0,0 +1,21 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Common mozconfig for automation builds. +# +# We export MOZ_AUTOMATION_* variables here to trigger various steps in +# automation builds. For example, if MOZ_AUTOMATION_PACKAGE is set, then the +# package step will run. This file contains the default settings, which can be +# overridden by setting them earlier in the appropriate mozconfig. + +mk_add_options "export MOZ_AUTOMATION_BUILD_SYMBOLS=${MOZ_AUTOMATION_BUILD_SYMBOLS-1}" +mk_add_options "export MOZ_AUTOMATION_CHECK=${MOZ_AUTOMATION_CHECK-1}" +mk_add_options "export MOZ_AUTOMATION_L10N_CHECK=${MOZ_AUTOMATION_L10N_CHECK-1}" +mk_add_options "export MOZ_AUTOMATION_PACKAGE=${MOZ_AUTOMATION_PACKAGE-1}" +mk_add_options "export MOZ_AUTOMATION_PACKAGE_TESTS=${MOZ_AUTOMATION_PACKAGE_TESTS-1}" +mk_add_options "export MOZ_AUTOMATION_TALOS_SENDCHANGE=${MOZ_AUTOMATION_TALOS_SENDCHANGE-1}" +mk_add_options "export MOZ_AUTOMATION_UNITTEST_SENDCHANGE=${MOZ_AUTOMATION_UNITTEST_SENDCHANGE-1}" +mk_add_options "export MOZ_AUTOMATION_UPDATE_PACKAGING=${MOZ_AUTOMATION_UPDATE_PACKAGING-0}" +mk_add_options "export MOZ_AUTOMATION_UPLOAD=${MOZ_AUTOMATION_UPLOAD-1}" +mk_add_options "export MOZ_AUTOMATION_UPLOAD_SYMBOLS=${MOZ_AUTOMATION_UPLOAD_SYMBOLS-0}" diff --git a/build/mozconfig.common b/build/mozconfig.common index edd7c40fe30b..3b49b819ecfd 100644 --- a/build/mozconfig.common +++ b/build/mozconfig.common @@ -15,3 +15,5 @@ mk_add_options AUTOCLOBBER=1 ac_add_options --enable-crashreporter ac_add_options --enable-release + +. "$topsrcdir/build/mozconfig.automation" diff --git a/build/unix/mozconfig.asan b/build/unix/mozconfig.asan index 8bd9e2d73136..4dfde4e946a9 100644 --- a/build/unix/mozconfig.asan +++ b/build/unix/mozconfig.asan @@ -1,3 +1,9 @@ +if [ "x$IS_NIGHTLY" = "xyes" ]; then + MOZ_AUTOMATION_UPDATE_PACKAGING=1 +fi +MOZ_AUTOMATION_TALOS_SENDCHANGE=0 +MOZ_AUTOMATION_L10N_CHECK=0 + . "$topsrcdir/build/mozconfig.common" # Use Clang as specified in manifest diff --git a/build/unix/mozconfig.linux b/build/unix/mozconfig.linux index dd6883d6e79d..dc9259f0bce0 100644 --- a/build/unix/mozconfig.linux +++ b/build/unix/mozconfig.linux @@ -1,3 +1,8 @@ +if [ "x$IS_NIGHTLY" = "xyes" ]; then + MOZ_AUTOMATION_UPLOAD_SYMBOLS=1 + MOZ_AUTOMATION_UPDATE_PACKAGING=1 +fi + . "$topsrcdir/build/mozconfig.common" # some b2g desktop builds still happen on i686 machines, and the tooltool