Bug 1364588 - part 1 - use `mach artifact toolchain' to download clang Stylo packages; r=rillian

Using this command is more robust than our current method, and brings
several benefits, such as smart caching of the downloaded toolchain.  We
change the clang package downloaded for Windows with this change, but
bindgen has been updated to work well with LLVM 5.0, so there should be
no problems.

python/mozboot/mozboot/base.py

@@ -272,52 +272,25 @@ class BaseBootstrapper(object):
             '%s does not yet implement ensure_stylo_packages()'
             % __name__)
 
-    def install_tooltool_clang_package(self, state_dir,
-                                       package_filename, package_sha512sum):
-        TOOLTOOL_API = 'https://api.pub.build.mozilla.org/tooltool/sha512/'
+    def install_tooltool_clang_package(self, state_dir, manifest_file):
+        topsrcdir = os.path.join(os.path.dirname(__file__) '..', '..', '..')
+        abs_manifest_file = os.path.join(topsrcdir, manifest_file)
 
-        # XXX this is similar to the Android NDK download.  We should unify them.
-        download_path = os.path.join(state_dir, 'mozboot')
-        try:
-            os.makedirs(download_path)
-        except OSError as e:
-            if e.errno == errno.EEXIST and os.path.isdir(download_path):
-                pass
-            else:
-                raise
+        mach_binary = os.path.join(topsrcdir, 'mach')
+        if not os.path.exists(mach_binary):
+            raise ValueError("mach not found at %s" % mach_binary)
 
-        try:
-            package_url = TOOLTOOL_API + package_sha512sum
-            downloaded_filename = os.path.join(download_path, package_sha512sum)
-            print('Downloading clang package from', package_url)
-            self.http_download_and_save(package_url, downloaded_filename,
-                                        package_sha512sum, 'sha512')
+        # If Python can't figure out what its own executable is, there's little
+        # chance we're going to be able to execute mach on its own, particularly
+        # on Windows.
+        if not sys.executable:
+            raise ValueError("cannot determine path to Python executable")
 
-            # We don't have to handle a great variety of archive types here.
-            if package_filename.endswith('tar.gz'):
-                cmd = ['tar', 'zxf', downloaded_filename]
-            elif package_filename.endswith('.tar.bz2'):
-                cmd = ['tar', 'jxf', downloaded_filename]
-            elif package_filename.endswith('.tar.xz'):
-                cmd = ['tar', 'Jxf', downloaded_filename]
-            else:
-                raise NotImplementedError("Don't know how to unpack file: %s"
-                                          % package_filename)
+        cmd = [sys.executable, mach_binary, 'artifact', 'toolchain',
+               '--tooltool-manifest', abs_manifest_file,
+               'clang']
 
-            print('Download complete!')
-            print('Unpacking %s...' % downloaded_filename)
-
-            with open(os.devnull, 'w') as stdout:
-                subprocess.check_call(cmd, stdout=stdout, cwd=state_dir)
-
-            print('Unpacking %s...DONE' % downloaded_filename)
-
-        finally:
-            try:
-                os.remove(downloaded_filename)
-            except OSError as e:
-                if e.errno != errno.ENOENT:
-                    raise
+        subprocess.check_call(cmd, cwd=state_dir)
 
     def which(self, name):
         """Python implementation of which.

python/mozboot/mozboot/linux_common.py

@@ -11,4 +11,4 @@ class StyloInstall(object):
 
     def ensure_stylo_packages(self, state_dir):
         import stylo
-        self.install_tooltool_clang_package(state_dir, **stylo.LINUX)
+        self.install_tooltool_clang_package(state_dir, stylo.LINUX)

python/mozboot/mozboot/mozillabuild.py

@@ -91,7 +91,7 @@ class MozillaBuildBootstrapper(BaseBootstrapper):
 
     def ensure_stylo_packages(self, state_dir):
         import stylo
-        self.install_tooltool_clang_package(state_dir, **stylo.WINDOWS)
+        self.install_tooltool_clang_package(state_dir, stylo.WINDOWS)
 
     def _update_package_manager(self):
         pass

python/mozboot/mozboot/osx.py

@@ -523,7 +523,7 @@ class OSXBootstrapper(BaseBootstrapper):
 
     def ensure_stylo_packages(self, state_dir):
         import stylo
-        self.install_tooltool_clang_package(state_dir, **stylo.OSX)
+        self.install_tooltool_clang_package(state_dir, stylo.OSX)
 
     def install_homebrew(self):
         print(PACKAGE_MANAGER_INSTALL % ('Homebrew', 'Homebrew', 'Homebrew', 'brew'))

python/mozboot/mozboot/stylo.py

@@ -4,67 +4,6 @@
 
 from __future__ import print_function, unicode_literals
 
-# Tooltool packages generated from the script below.
-WINDOWS = {
-    'package_filename': 'clang.tar.bz2',
-    'package_sha512sum': 'cd3ed31acefd185f441632158dde73538c62bab7ebf2a8ec630985ab345938ec522983721ddb1bead1de22d5ac1571d50a958ae002364d739f2a78c6e7244222',
-}
-
-OSX = {
-    'package_filename': 'clang.tar.bz2',
-    'package_sha512sum': '0e1a556b65d6398fa812b9ceb5ce5e2dec3eda77d4a032a818182b34fc8ce602412f42388bb1fda6bea265d35c1dde3847a730b264fec01cd7e3dcfd39941660',
-}
-
-LINUX = {
-    'package_filename': 'clang.tar.xz',
-    'package_sha512sum': '52f3fc23f0f5c98050f8b0ac7c92a6752d067582a16f712a5a58074be98975d594f9e36249fc2be7f1cc2ca6d509c663faaf2bea66f949243cc1f41651638ba6',
-}
-
-if __name__ == '__main__':
-    '''Allow invoking the module as a utility to update tooltool downloads.'''
-    import json
-    import os
-    import sys
-
-    mod_path = os.path.dirname(__file__)
-    browser_config_dir = os.path.join('browser', 'config', 'tooltool-manifests')
-
-    os_map = {
-        'WINDOWS': ('win64', 'clang.manifest'),
-        'OSX': ('macosx64', 'releng.manifest'),
-        'LINUX': ('linux64', 'releng.manifest'),
-    }
-
-    for os_name, (os_dir, f) in os_map.iteritems():
-        manifest_file = os.path.join(browser_config_dir, os_dir, f)
-        abspath = os.path.join(mod_path, '..', '..', '..', manifest_file)
-        with open(abspath, 'r') as s:
-            manifest = json.load(s)
-            entries = filter(lambda x: x['filename'].startswith('clang'), manifest)
-            if not entries:
-                print('ERROR: could not find clang tooltool entry in %s' % manifest_file)
-                sys.exit(1)
-            if len(entries) > 1:
-                print('ERROR: too many clang entries in %s' % manifest_file)
-                sys.exit(1)
-
-            clang = entries[0]
-            if clang['algorithm'] != 'sha512':
-                print("ERROR: don't know how to handle digest %s in %s" % (clang['algorithm'],
-                                                                           manifest_file))
-                sys.exit(1)
-
-            FORMAT_STRING = """{os} = {{
-    'package_filename': '{filename}',
-    'package_sha512sum': '{digest}',
-}}
-"""
-            digest = clang['digest']
-            if os_name == 'WINDOWS':
-                # The only clang version we can retrieve from the tooltool manifest
-                # doesn't work with Stylo bindgen due to regressions in LLVM.  This
-                # is an older version that works.
-                digest = 'cd3ed31acefd185f441632158dde73538c62bab7ebf2a8ec630985ab345938ec522983721ddb1bead1de22d5ac1571d50a958ae002364d739f2a78c6e7244222'
-            print(FORMAT_STRING.format(os=os_name,
-                                       filename=clang['filename'],
-                                       digest=digest))
+WINDOWS = 'browser/config/tooltool-manifests/win64/clang.manifest'
+OSX = 'browser/config/tooltool-manifests/macosx64/releng.manifest'
+LINUX = 'browser/config/tooltool-manifests/linux64/clang.manifest'

python/mozboot/mozboot/windows.py

@@ -86,7 +86,7 @@ class WindowsBootstrapper(BaseBootstrapper):
 
     def ensure_stylo_packages(self, state_dir):
         import stylo
-        self.install_tooltool_clang_package(state_dir, **stylo.WINDOWS)
+        self.install_tooltool_clang_package(state_dir, stylo.WINDOWS)
 
     def _update_package_manager(self):
         self.pacman_update()