From 8bf87f6f0e9ea79c9c2073d45439f1b05e45e725 Mon Sep 17 00:00:00 2001 From: Nick Alexander Date: Thu, 31 May 2018 11:12:26 -0700 Subject: [PATCH] Bug 1465836 - Make MOZ_AUTOMATION artifact builds pull host binaries (mar,mbsdiff). r=chmanchester This populates $OBJDIR/dist/host/bin as part of |mach artifact install|. Conceptually, the mar and mbsdiff utilities should be grouped (in the same way that the test-related binaries are grouped). However, it's difficult to achieve that with the current structure of the code, so this fetches mar and mbsdiff and produces $HASH-mar.processed.jar and $HASH-mbsdiff.processed.jar files. MozReview-Commit-ID: 3ks5xsUEKp5 --HG-- extra : rebase_source : 5fcf186decc95537cbaa90ffedb86774eab050d2 --- python/mozbuild/mozbuild/artifacts.py | 53 ++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 6 deletions(-) diff --git a/python/mozbuild/mozbuild/artifacts.py b/python/mozbuild/mozbuild/artifacts.py index c3e62d86767a..20f84c6d123d 100644 --- a/python/mozbuild/mozbuild/artifacts.py +++ b/python/mozbuild/mozbuild/artifacts.py @@ -146,11 +146,17 @@ class ArtifactJob(object): # be the same across platforms. _test_archive_suffix = '.common.tests.zip' - def __init__(self, package_re, tests_re, log=None, download_symbols=False, substs=None): + def __init__(self, package_re, tests_re, log=None, + download_symbols=False, + download_host_bins=False, + substs=None): self._package_re = re.compile(package_re) self._tests_re = None if tests_re: self._tests_re = re.compile(tests_re) + self._host_bins_re = None + if download_host_bins: + self._host_bins_re = re.compile(r'public/build/host/bin/(mar|mbsdiff)(.exe)?') self._log = log self._substs = substs self._symbols_archive_suffix = None @@ -168,6 +174,8 @@ class ArtifactJob(object): name = artifact['name'] if self._package_re and self._package_re.match(name): yield name + elif self._host_bins_re and self._host_bins_re.match(name): + yield name elif self._tests_re and self._tests_re.match(name): tests_artifact = name yield name @@ -186,6 +194,13 @@ class ArtifactJob(object): return self.process_tests_artifact(filename, processed_filename) if self._symbols_archive_suffix and filename.endswith(self._symbols_archive_suffix): return self.process_symbols_archive(filename, processed_filename) + if self._host_bins_re: + # Turn 'HASH-mar.exe' into 'mar.exe'. `filename` is a path on disk + # without the full path to the artifact, so we must reconstruct + # that path here. + orig_basename = os.path.basename(filename).split('-', 1)[1] + if self._host_bins_re.match('public/build/host/bin/{}'.format(orig_basename)): + return self.process_host_bin(filename, processed_filename) return self.process_package_artifact(filename, processed_filename) def process_package_artifact(self, filename, processed_filename): @@ -237,6 +252,16 @@ class ArtifactJob(object): 'Adding {destpath} to processed archive') writer.add(destpath.encode('utf-8'), reader[filename]) + def process_host_bin(self, filename, processed_filename): + with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer: + # Turn 'HASH-mar.exe' into 'mar.exe'. `filename` is a path on disk + # without any of the path parts of the artifact, so we must inject + # the desired `host/bin` prefix here. + orig_basename = os.path.basename(filename).split('-', 1)[1] + destpath = mozpath.join('host/bin', orig_basename) + writer.add(destpath.encode('utf-8'), open(filename, 'rb')) + + class AndroidArtifactJob(ArtifactJob): product = 'mobile' @@ -501,11 +526,17 @@ JOB_DETAILS = { -def get_job_details(job, log=None, download_symbols=False, substs=None): +def get_job_details(job, log=None, + download_symbols=False, + download_host_bins=False, + substs=None): cls, (package_re, tests_re) = JOB_DETAILS[job] - return cls(package_re, tests_re, log=log, download_symbols=download_symbols, + return cls(package_re, tests_re, log=log, + download_symbols=download_symbols, + download_host_bins=download_host_bins, substs=substs) + def cachedmethod(cachefunc): '''Decorator to wrap a class or instance method with a memoizing callable that saves results in a (possibly shared) cache. @@ -632,9 +663,11 @@ class TaskCache(CacheManager): CacheManager.__init__(self, cache_dir, 'artifact_url', MAX_CACHED_TASKS, log=log, skip_cache=skip_cache) @cachedmethod(operator.attrgetter('_cache')) - def artifact_urls(self, tree, job, rev, download_symbols): + def artifact_urls(self, tree, job, rev, download_symbols, download_host_bins): try: - artifact_job = get_job_details(job, log=self._log, download_symbols=download_symbols) + artifact_job = get_job_details(job, log=self._log, + download_symbols=download_symbols, + download_host_bins=download_host_bins) except KeyError: self.log(logging.INFO, 'artifact', {'job': job}, @@ -860,6 +893,11 @@ class Artifacts(object): self._substs = substs self._download_symbols = self._substs.get('MOZ_ARTIFACT_BUILD_SYMBOLS', False) + # Host binaries are not produced for macOS consumers: that is, there's + # no macOS-hosted job to produce them at this time. Therefore we + # enable this only for automation builds, which only require Linux and + # Windows host binaries. + self._download_host_bins = self._substs.get('MOZ_AUTOMATION', False) self._defines = defines self._tree = tree self._job = job or self._guess_artifact_job() @@ -873,6 +911,7 @@ class Artifacts(object): try: self._artifact_job = get_job_details(self._job, log=self._log, download_symbols=self._download_symbols, + download_host_bins=self._download_host_bins, substs=self._substs) except KeyError: self.log(logging.INFO, 'artifact', @@ -1051,7 +1090,9 @@ class Artifacts(object): def find_pushhead_artifacts(self, task_cache, job, tree, pushhead): try: - urls = task_cache.artifact_urls(tree, job, pushhead, self._download_symbols) + urls = task_cache.artifact_urls(tree, job, pushhead, + self._download_symbols, + self._download_host_bins) except ValueError: return None if urls: