зеркало из https://github.com/mozilla/gecko-dev.git
Bug 978211 - add an automation/build target for post-build steps; r=glandium
This commit is contained in:
Mike Shal
Родитель
c90e727466
Коммит
0bfb7d3b01
|
|
@ -132,6 +132,7 @@ install-manifests: install-tests
|
|||
install-tests: $(install_manifest_depends)
|
||||
$(call py_action,process_install_manifest,$(if $(NO_REMOVE),--no-remove )_tests _build_manifests/install/tests)
|
||||
|
||||
include $(topsrcdir)/build/moz-automation.mk
|
||||
|
||||
# _tests should be purged during cleaning. However, we don't want it purged
|
||||
# during PGO builds because it contains some auto-generated files.
|
||||
|
|
@ -168,7 +169,7 @@ endif
|
|||
endif
|
||||
|
||||
default all::
|
||||
$(call BUILDSTATUS,TIERS export $(if $(COMPILE_ENVIRONMENT),$(if $(MOZ_PSEUDO_DERECURSE),compile ))libs tools)
|
||||
$(call BUILDSTATUS,TIERS export $(if $(COMPILE_ENVIRONMENT),$(if $(MOZ_PSEUDO_DERECURSE),compile ))libs tools $(if $(MOZ_AUTOMATION),$(MOZ_AUTOMATION_TIERS)))
|
||||
|
||||
include $(topsrcdir)/config/rules.mk
|
||||
|
||||
|
|
@ -251,6 +252,9 @@ MOZ_SOURCE_STAMP := $(MOZ_SOURCE_STAMP)
|
|||
export MOZ_SOURCE_STAMP
|
||||
endif
|
||||
|
||||
update-packaging:
|
||||
$(MAKE) -C tools/update-packaging
|
||||
|
||||
#XXX: this is a hack, since we don't want to clobber for MSVC
|
||||
# PGO support, but we can't do this test in client.mk
|
||||
ifneq ($(OS_ARCH)_$(GNU_CC), WINNT_)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,79 @@
|
|||
#!/usr/bin/python
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
import sys, os, sha, json, re
|
||||
from argparse import ArgumentParser
|
||||
|
||||
def getFileHashAndSize(filename):
|
||||
sha1Hash = 'UNKNOWN'
|
||||
size = 'UNKNOWN'
|
||||
|
||||
try:
|
||||
# open in binary mode to make sure we get consistent results
|
||||
# across all platforms
|
||||
f = open(filename, "rb")
|
||||
shaObj = sha.new(f.read())
|
||||
sha1Hash = shaObj.hexdigest()
|
||||
|
||||
size = os.path.getsize(filename)
|
||||
except:
|
||||
pass
|
||||
|
||||
return (sha1Hash, size)
|
||||
|
||||
def getMarProperties(filename):
|
||||
(complete_mar_hash, complete_mar_size) = getFileHashAndSize(filename)
|
||||
return {
|
||||
'completeMarFilename': os.path.basename(filename),
|
||||
'completeMarSize': complete_mar_size,
|
||||
'completeMarHash': complete_mar_hash,
|
||||
}
|
||||
|
||||
def getUrlProperties(filename):
|
||||
# let's create a switch case using name-spaces/dict
|
||||
# rather than a long if/else with duplicate code
|
||||
property_conditions = [
|
||||
# key: property name, value: condition
|
||||
('symbolsUrl', lambda m: m.endswith('crashreporter-symbols.zip') or
|
||||
m.endswith('crashreporter-symbols-full.zip')),
|
||||
('testsUrl', lambda m: m.endswith(('tests.tar.bz2', 'tests.zip'))),
|
||||
('unsignedApkUrl', lambda m: m.endswith('apk') and
|
||||
'unsigned-unaligned' in m),
|
||||
('robocopApkUrl', lambda m: m.endswith('apk') and 'robocop' in m),
|
||||
('jsshellUrl', lambda m: 'jsshell-' in m and m.endswith('.zip')),
|
||||
('completeMarUrl', lambda m: m.endswith('.complete.mar')),
|
||||
# packageUrl must be last!
|
||||
('packageUrl', lambda m: True),
|
||||
]
|
||||
url_re = re.compile(r'''^(https?://.*?\.(?:tar\.bz2|dmg|zip|apk|rpm|mar|tar\.gz))$''')
|
||||
properties = {}
|
||||
|
||||
with open(filename) as f:
|
||||
for line in f:
|
||||
m = url_re.match(line)
|
||||
if m:
|
||||
m = m.group(1)
|
||||
for prop, condition in property_conditions:
|
||||
if condition(m):
|
||||
properties.update({prop: m})
|
||||
break
|
||||
return properties
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = ArgumentParser(description='Generate mach_build_properties.json for automation builds.')
|
||||
parser.add_argument("--complete-mar-file", required=True,
|
||||
action="store", dest="complete_mar_file",
|
||||
help="Path to the complete MAR file, relative to the objdir.")
|
||||
parser.add_argument("--upload-output", required=True,
|
||||
action="store", dest="upload_output",
|
||||
help="Path to the text output of 'make upload'")
|
||||
args = parser.parse_args()
|
||||
|
||||
json_data = getMarProperties(args.complete_mar_file)
|
||||
json_data.update(getUrlProperties(args.upload_output))
|
||||
|
||||
with open('mach_build_properties.json', 'w') as outfile:
|
||||
json.dump(json_data, outfile, indent=4)
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
include $(topsrcdir)/toolkit/mozapps/installer/package-name.mk
|
||||
|
||||
# Log file from the 'make upload' step. We need this to parse out the URLs of
|
||||
# the uploaded files.
|
||||
AUTOMATION_UPLOAD_OUTPUT = $(DIST)/automation-upload.txt
|
||||
|
||||
# Automation build steps. Everything in MOZ_AUTOMATION_TIERS also gets used in
|
||||
# TIERS for mach display. As such, the MOZ_AUTOMATION_TIERS are roughly sorted
|
||||
# here in the order that they will be executed (since mach doesn't know of the
|
||||
# dependencies between them).
|
||||
|
||||
MOZ_AUTOMATION_TIERS += package-tests
|
||||
MOZ_AUTOMATION_TIERS += buildsymbols
|
||||
|
||||
ifdef NIGHTLY_BUILD
|
||||
MOZ_AUTOMATION_TIERS += uploadsymbols
|
||||
endif
|
||||
|
||||
ifdef MOZ_UPDATE_PACKAGING
|
||||
MOZ_AUTOMATION_TIERS += update-packaging
|
||||
automation/upload: automation/update-packaging
|
||||
automation/update-packaging: automation/package
|
||||
endif
|
||||
|
||||
MOZ_AUTOMATION_TIERS += package
|
||||
MOZ_AUTOMATION_TIERS += check
|
||||
|
||||
ifndef MOZ_ASAN
|
||||
MOZ_AUTOMATION_TIERS += l10n-check
|
||||
automation/l10n-check: automation/package
|
||||
endif
|
||||
|
||||
MOZ_AUTOMATION_TIERS += upload
|
||||
|
||||
automation/build: $(addprefix automation/,$(MOZ_AUTOMATION_TIERS))
|
||||
$(PYTHON) $(topsrcdir)/build/gen_mach_buildprops.py --complete-mar-file $(DIST)/$(COMPLETE_MAR) --upload-output $(AUTOMATION_UPLOAD_OUTPUT)
|
||||
|
||||
# Dependencies between automation build steps
|
||||
automation/uploadsymbols: automation/buildsymbols
|
||||
|
||||
automation/upload: automation/package
|
||||
automation/upload: automation/package-tests
|
||||
automation/upload: automation/buildsymbols
|
||||
|
||||
# automation/package and automation/check should depend on build (which is
|
||||
# implicit due to the way client.mk invokes automation/build), but buildsymbols
|
||||
# changes the binaries/libs, and that's what we package/test.
|
||||
automation/package: automation/buildsymbols
|
||||
automation/check: automation/buildsymbols
|
||||
|
||||
# make check runs with the keep-going flag so we can see all the failures
|
||||
AUTOMATION_EXTRA_CMDLINE-check = -k
|
||||
|
||||
# We need the log from make upload to grep it for urls in order to set
|
||||
# properties.
|
||||
AUTOMATION_EXTRA_CMDLINE-upload = 2>&1 | tee $(AUTOMATION_UPLOAD_OUTPUT)
|
||||
|
||||
automation/%:
|
||||
$(call BUILDSTATUS,TIER_START $*)
|
||||
@$(MAKE) $* $(AUTOMATION_EXTRA_CMDLINE-$*)
|
||||
$(call BUILDSTATUS,TIER_FINISH $*)
|
||||
|
|
@ -176,7 +176,7 @@ CONFIGURES := $(TOPSRCDIR)/configure
|
|||
CONFIGURES += $(TOPSRCDIR)/js/src/configure
|
||||
|
||||
# Make targets that are going to be passed to the real build system
|
||||
OBJDIR_TARGETS = install export libs clean realclean distclean maybe_clobber_profiledbuild upload sdk installer package package-compare stage-package source-package l10n-check
|
||||
OBJDIR_TARGETS = install export libs clean realclean distclean maybe_clobber_profiledbuild upload sdk installer package package-compare stage-package source-package l10n-check automation/build
|
||||
|
||||
#######################################################################
|
||||
# Rules
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ class TierStatus(object):
|
|||
def __init__(self, resources):
|
||||
"""Accepts a SystemResourceMonitor to record results against."""
|
||||
self.tiers = OrderedDict()
|
||||
self.active_tier = None
|
||||
self.active_tiers = set()
|
||||
self.resources = resources
|
||||
|
||||
def set_tiers(self, tiers):
|
||||
|
|
@ -84,17 +84,18 @@ class TierStatus(object):
|
|||
# have one until Python 3.
|
||||
t['begin_time'] = time.time()
|
||||
self.resources.begin_phase(tier)
|
||||
self.active_tier = tier
|
||||
self.active_tiers.add(tier)
|
||||
|
||||
def finish_tier(self, tier):
|
||||
"""Record that execution of a tier has finished."""
|
||||
t = self.tiers[tier]
|
||||
t['finish_time'] = time.time()
|
||||
t['duration'] = self.resources.finish_phase(tier)
|
||||
self.active_tiers.remove(tier)
|
||||
|
||||
def tier_status(self):
|
||||
for tier, state in self.tiers.items():
|
||||
active = self.active_tier == tier
|
||||
active = tier in self.active_tiers
|
||||
finished = state['finish_time'] is not None
|
||||
|
||||
yield tier, active, finished
|
||||
|
|
|
|||
|
|
@ -367,6 +367,18 @@ class Build(MachCommandBase):
|
|||
allow_parallel=False, ensure_exit_code=False, num_jobs=jobs,
|
||||
silent=not verbose, force_pymake=pymake)
|
||||
|
||||
make_extra = self.mozconfig['make_extra'] or []
|
||||
make_extra = dict(m.split('=', 1) for m in make_extra)
|
||||
|
||||
moz_automation = os.getenv('MOZ_AUTOMATION') or make_extra.get('export MOZ_AUTOMATION', None)
|
||||
if moz_automation and status == 0:
|
||||
# Default to 1 job until we have make-4.0 in for output
|
||||
# buffering.
|
||||
status = self._run_make(target='automation/build',
|
||||
line_handler=output.on_line, log=False, print_directory=False,
|
||||
allow_parallel=False, ensure_exit_code=False, num_jobs=1,
|
||||
silent=not verbose, force_pymake=pymake)
|
||||
|
||||
self.log(logging.WARNING, 'warning_summary',
|
||||
{'count': len(monitor.warnings_database)},
|
||||
'{count} compiler warnings present.')
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче