servo: Merge #12916 - Create `mach bootstrap` based on Mozilla's mozboot bootstrapper (from UK992:msvc-dependencies); r=larsbergstrom,wafflespeanut

Fixes https://github.com/servo/servo/issues/12914
I've made this few weeks ago, its an example how could everything looks like.
It downloads and setup all needed dependencies for MSVC.
It's has version in case if some dependencies need to be updated.
Zip files and folder in zip need to be named ``<dep>-<version>``.
Also if cmake already exist in PATH, it won't download it again.

I want opinion on that, if this is right approaches and how to improve it.

cc @vvuk

Source-Repo: https://github.com/servo/servo
Source-Revision: daa30e60f1a7af87ee88aae4fd6e47e210ee9a76

--HG--
rename : servo/python/tidy/servo_tidy_tests/shebang_license.py => servo/python/servo/bootstrapper/__init__.py

servo/appveyor.yml

@@ -2,6 +2,7 @@ version: 1.0.{build}
 
 environment:
   RUST_BACKTRACE: 1
+  HOME: '%APPVEYOR_BUILD_FOLDER%'
   # The appveyor image we use has a pretty huge set of things installed... we make the
   # initial PATH something sane so we know what to expect
   PATH: "C:\\windows\\system32;\
@@ -45,26 +46,6 @@ cache:
 install:
   - if %TARGET:*-msvc=msvc%==msvc set BUILD_ENV=msvc
   - if %TARGET:*-gnu=gnu%==gnu set BUILD_ENV=gnu
-  - ps: 'if ($env:BUILD_ENV -eq "msvc") {
-           Start-FileDownload "http://servo-rust.s3.amazonaws.com/build/openssl-and-ffmpeg.zip" -ErrorAction Stop ;
-           Start-FileDownload "http://servo-rust.s3.amazonaws.com/build/ninja.zip" -ErrorAction Stop ;
-           Start-FileDownload "http://servo-rust.s3.amazonaws.com/build/MozillaBuildSetup-2.2.0.exe" -ErrorAction Stop ;
-           .\MozillaBuildSetup-2.2.0.exe /S | Out-Null ;
-           7z x openssl-and-ffmpeg.zip | Out-Null ;
-           7z x ninja.zip | Out-Null ;
-           Copy-Item C:\mozilla-build\yasm\yasm.exe C:\mozilla-build\msys\bin ;
-           Copy-Item C:\mozilla-build\mozmake\mozmake.exe C:\mozilla-build\msys\bin ;
-           $env:MOZTOOLS_PATH="C:\mozilla-build\msys\bin" ;
-           $env:NATIVE_WIN32_PYTHON="C:/Python27/python.exe" ;
-           $env:PATH="$pwd\ninja;$env:PATH" ;
-           $env:OPENSSL_INCLUDE_DIR="$pwd\openssl-bin\openssl-1.0.1t-vs2015\include" ;
-           $env:OPENSSL_LIB_DIR="$pwd\openssl-bin\openssl-1.0.1t-vs2015\lib64" ;
-           $env:OPENSSL_LIBS="ssleay32MD:libeay32MD" ;
-           $env:FFMPEG_INCLUDE_DIR="$pwd\ffmpeg-bin\include" ;
-           $env:FFMPEG_LIB_DIR="$pwd\ffmpeg-bin\lib" ;
-           $env:FFMPEG_LIBS="avformat:avcodec:avutil" ;
-        }'
-  - if %BUILD_ENV%==msvc call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64\vcvars64.bat"
   - if %BUILD_ENV%==gnu  set PATH=C:\msys64\mingw64\bin;C:\msys64\usr\bin\;%PATH%
   - if %BUILD_ENV%==gnu  set MSYSTEM=MINGW64
   - if %BUILD_ENV%==gnu  set MSYS=winsymlinks=lnk
@@ -87,10 +68,7 @@ install:
 #  - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
 
 build_script:
-  - echo PATH %PATH%
-  - echo VSINSTALLDIR %VSINSTALLDIR%
-  - echo MOZTOOLS_PATH %MOZTOOLS_PATH%
-  - if %BUILD_ENV%==msvc cd %APPVEYOR_BUILD_FOLDER% && mach build -d -v && mach test-unit
-  - if %BUILD_ENV%==gnu  bash -lc "cd $APPVEYOR_BUILD_FOLDER; ./mach build -d -v && ./mach test-unit"
+  - if %BUILD_ENV%==msvc mach build -d -v && mach test-unit
+  - if %BUILD_ENV%==gnu  bash -lc "./mach build -d -v && ./mach test-unit"
 
 test: off

servo/python/servo/bootstrap_commands.py

@@ -19,6 +19,7 @@ import shutil
 import sys
 import StringIO
 import tarfile
+import zipfile
 import urllib2
 
 from mach.decorators import (
@@ -101,7 +102,10 @@ def download_bytes(desc, src):
 
 
 def extract(src, dst, movedir=None):
-    tarfile.open(src).extractall(dst)
+    if src.endswith(".zip"):
+        zipfile.ZipFile(src).extractall(dst)
+    else:
+        tarfile.open(src).extractall(dst)
 
     if movedir:
         for f in os.listdir(movedir):
@@ -126,6 +130,24 @@ class MachCommands(CommandBase):
         else:
             print("export LD_LIBRARY_PATH=%s" % env["LD_LIBRARY_PATH"])
 
+    @Command('bootstrap',
+             description='Install required packages for building.',
+             category='bootstrap')
+    @CommandArgument('--interactive', "-i",
+                     action='store_true',
+                     help='Need to answer any (Y/n) interactive prompts.')
+    @CommandArgument('--android',
+                     action='store_true',
+                     help='Install required packages for Android')
+    @CommandArgument('--force', '-f',
+                     action='store_true',
+                     help='Force reinstall packages')
+    def bootstrap(self, android=False, interactive=False, force=False):
+        from servo.bootstrapper.bootstrap import Bootstrapper
+
+        bootstrapper = Bootstrapper()
+        bootstrapper.bootstrap(android=android, interactive=interactive, force=force)
+
     @Command('bootstrap-rust',
              description='Download the Rust compiler',
              category='bootstrap')

servo/python/servo/bootstrapper/__init__.py

@@ -0,0 +1,3 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.

servo/python/servo/bootstrapper/base.py

@@ -0,0 +1,62 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function, unicode_literals
+
+import distutils
+import subprocess
+
+
+class BaseBootstrapper(object):
+    """Base class for system bootstrappers."""
+
+    def __init__(self, interactive=False):
+        self.package_manager_updated = False
+        self.interactive = interactive
+
+    def ensure_system_packages(self):
+        '''
+        Check for missing packages.
+        '''
+        raise NotImplementedError('%s must implement ensure_system_packages()' %
+                                  __name__)
+
+    def install_system_packages(self):
+        '''
+        Install packages required to build Servo.
+        '''
+        raise NotImplementedError('%s must implement install_system_packages()' %
+                                  __name__)
+
+    def install_mobile_android_packages(self):
+        '''
+        Install packages required to build Servo for Android.
+        '''
+        raise NotImplementedError('Cannot bootstrap Servo for Android: '
+                                  '%s does not yet implement install_mobile_android_packages()'
+                                  % __name__)
+
+    def which(self, name):
+        """Python implementation of which.
+
+        It returns the path of an executable or None if it couldn't be found.
+        """
+        return distutils.spawn.find_executable(name)
+
+    def check_output(self, *args, **kwargs):
+        """Run subprocess.check_output."""
+        return subprocess.check_output(*args, **kwargs)
+
+    def _ensure_package_manager_updated(self):
+        if self.package_manager_updated:
+            return
+
+        self._update_package_manager()
+        self.package_manager_updated = True
+
+    def _update_package_manager(self):
+        """Updates the package manager's manifests/package list.
+
+        This should be defined in child classes.
+        """

servo/python/servo/bootstrapper/bootstrap.py

@@ -0,0 +1,41 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function
+
+import sys
+
+from windows_gnu import WindowsGnuBootstrapper
+from windows_msvc import WindowsMsvcBootstrapper
+
+
+class Bootstrapper(object):
+    """Main class that performs system bootstrap."""
+
+    def __init__(self):
+        self.instance = None
+        cls = None
+        args = {}
+
+        if sys.platform.startswith('msys'):
+            cls = WindowsGnuBootstrapper
+
+        elif sys.platform.startswith('win32'):
+            cls = WindowsMsvcBootstrapper
+
+        if cls is None:
+            sys.exit('Bootstrap support is not yet available for your OS.')
+
+        self.instance = cls(**args)
+
+    def bootstrap(self, android=False, interactive=False, force=False):
+        self.instance.interactive = interactive
+        self.instance.force = force
+
+        if android:
+            self.instance.install_mobile_android_packages()
+        elif force:
+            self.instance.install_system_packages()
+        else:
+            self.instance.ensure_system_packages()

servo/python/servo/bootstrapper/packages.py

@@ -0,0 +1,28 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Listed all packages for different platforms in one file
+
+WINDOWS_GNU = [
+    "mingw-w64-x86_64-toolchain",
+    "mingw-w64-x86_64-freetype",
+    "mingw-w64-x86_64-icu",
+    "mingw-w64-x86_64-nspr",
+    "mingw-w64-x86_64-ca-certificates",
+    "mingw-w64-x86_64-expat",
+    "mingw-w64-x86_64-cmake",
+    "tar",
+    "diffutils",
+    "patch",
+    "patchutils",
+    "make",
+    "python2-setuptools",
+]
+
+WINDOWS_MSVC = [
+    "cmake-3.6.1",
+    "ninja-1.7.1",
+    "openssl-1.0.1t-vs2015",
+    "moztools-0.0.1-5",
+]

servo/python/servo/bootstrapper/windows_gnu.py

@@ -0,0 +1,75 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys
+import subprocess
+
+from base import BaseBootstrapper
+from packages import WINDOWS_GNU as deps
+
+
+class WindowsGnuBootstrapper(BaseBootstrapper):
+    '''Bootstrapper for msys2 based environments for building in Windows.'''
+
+    def __init__(self, **kwargs):
+        BaseBootstrapper.__init__(self, **kwargs)
+
+        if not self.which('pacman'):
+            raise NotImplementedError('The Windows bootstrapper only works with msys2 with pacman. Get msys2 at '
+                                      'http://msys2.github.io/')
+
+    def ensure_system_packages(self):
+        install_packages = []
+        for p in deps:
+            command = ['pacman', '-Qs', p]
+            if self.run_check(command):
+                install_packages += [p]
+        if install_packages:
+            install_packages(install_packages)
+
+    def install_system_packages(self, packages=deps):
+        self._ensure_package_manager_updated()
+        self.pacman_install(*packages)
+
+    def install_mobile_android_packages(self):
+        sys.exit('We do not support building Android on Windows. Sorry!')
+
+    def _update_package_manager(self):
+        self.pacman_update()
+
+    def run(self, command):
+        subprocess.check_call(command, stdin=sys.stdin)
+
+    def run_check(self, command):
+        return subprocess.call(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    def pacman_update(self):
+        command = ['pacman', '--sync', '--refresh']
+        self.run(command)
+
+    def pacman_upgrade(self):
+        command = ['pacman', '--sync', '--refresh', '--sysupgrade']
+        self.run(command)
+
+    def pacman_install(self, *packages):
+        command = ['pacman', '--sync']
+        if not self.force:
+            command.append('--needed')
+        if not self.interactive:
+            command.append('--noconfirm')
+        command.extend(packages)
+        self.run(command)
+
+        # downgrade GCC to 5.4.0-1
+        gcc_type = ["gcc", "gcc-ada", "gcc-fortran", "gcc-libgfortran", "gcc-libs", "gcc-objc"]
+        gcc_version = "5.4.0-1"
+        mingw_url = "http://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-{}-{}-any.pkg.tar.xz"
+        gcc_list = []
+        for gcc in gcc_type:
+            gcc_list += [mingw_url.format(gcc, gcc_version)]
+        downgrade_command = ['pacman', '-U']
+        if not self.interactive:
+            downgrade_command.append('--noconfirm')
+        downgrade_command.extend(gcc_list)
+        self.run(downgrade_command)

servo/python/servo/bootstrapper/windows_msvc.py

@@ -0,0 +1,86 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import sys
+import shutil
+from distutils import spawn
+
+from base import BaseBootstrapper
+from packages import WINDOWS_MSVC as deps
+
+
+class WindowsMsvcBootstrapper(BaseBootstrapper):
+    '''Bootstrapper for MSVC building on Windows.'''
+
+    def __init__(self, **kwargs):
+        BaseBootstrapper.__init__(self, **kwargs)
+
+    def ensure_system_packages(self):
+        self.install_system_packages()
+
+    def install_system_packages(self, packages=deps):
+        from servo.bootstrap_commands import extract, download_file
+
+        deps_dir = os.path.join(".servo", "msvc-dependencies")
+        deps_url = "https://servo-rust.s3.amazonaws.com/msvc-deps/"
+        first_run = True
+
+        if self.force:
+            if os.path.isdir(deps_dir):
+                shutil.rmtree(deps_dir)
+
+        if not os.path.isdir(deps_dir):
+            os.makedirs(deps_dir)
+
+        # Read file with installed dependencies, if exist
+        installed_deps_file = os.path.join(deps_dir, "installed-dependencies.txt")
+        if os.path.exists(installed_deps_file):
+            installed_deps = [l.strip() for l in open(installed_deps_file)]
+        else:
+            installed_deps = []
+
+        # list of dependencies that need to be updated
+        update_deps = list(set(packages) - set(installed_deps))
+
+        for dep in packages:
+            dep_name = dep.split("-")[0]
+
+            # Don't download CMake if already exists in PATH
+            if dep_name == "cmake":
+                if spawn.find_executable(dep_name):
+                    continue
+
+            dep_dir = os.path.join(deps_dir, dep_name)
+            # if not installed or need to be updated
+            if not os.path.exists(dep_dir) or dep in update_deps:
+                if first_run:
+                    print "Installing missing MSVC dependencies..."
+                    first_run = False
+
+                dep_version_dir = os.path.join(deps_dir, dep)
+
+                if os.path.exists(dep_version_dir):
+                    shutil.rmtree(dep_version_dir)
+
+                dep_zip = dep_version_dir + ".zip"
+                if not os.path.isfile(dep_zip):
+                    download_file(dep, "%s%s.zip" % (deps_url, dep), dep_zip)
+
+                print "Extracting %s..." % dep,
+                extract(dep_zip, deps_dir)
+                print "done"
+
+                # Delete directory if exist
+                if os.path.exists(dep_dir):
+                    shutil.rmtree(dep_dir)
+                os.rename(dep_version_dir, dep_dir)
+
+        # Write in installed-dependencies.txt file
+        with open(installed_deps_file, 'w') as installed_file:
+            for line in packages:
+                installed_file.write(line + "\n")
+
+    def install_mobile_android_packages(self):
+        sys.exit('We do not support building Android on Windows. Sorry!')

servo/python/servo/build_commands.py

@@ -257,16 +257,14 @@ class MachCommands(CommandBase):
                 # On windows, copy in our manifest
                 shutil.copy(path.join(self.get_top_dir(), "components", "servo", "servo.exe.manifest"),
                             servo_exe_dir)
-                if "msvc" in host_triple():
+                if "msvc" in (target or host_triple()):
+                    msvc_x64 = "64" if "x86_64" in (target or host_triple()) else ""
                     # on msvc builds, use editbin to change the subsystem to windows
                     call(["editbin", "/nologo", "/subsystem:windows", path.join(servo_exe_dir, "servo.exe")],
                          verbose=verbose)
                     # on msvc, we need to copy in some DLLs in to the servo.exe dir
                     for ssl_lib in ["ssleay32md.dll", "libeay32md.dll"]:
-                        shutil.copy(path.join(os.getenv('OPENSSL_LIB_DIR'), "../bin64", ssl_lib),
-                                    servo_exe_dir)
-                    for ffmpeg_lib in ["avutil-55.dll", "avformat-57.dll", "avcodec-57.dll", "swresample-2.dll"]:
-                        shutil.copy(path.join(os.getenv('FFMPEG_LIB_DIR'), "../bin", ffmpeg_lib),
+                        shutil.copy(path.join(env['OPENSSL_LIB_DIR'], "../bin" + msvc_x64, ssl_lib),
                                     servo_exe_dir)
 
                 elif sys.platform == "darwin":

servo/python/servo/command_base.py

@@ -372,6 +372,18 @@ class CommandBase(object):
             env['PATH'] = env['PATH'].encode('ascii', 'ignore')
         extra_path = []
         extra_lib = []
+        if "msvc" in (target or host_triple()):
+            msvc_x64 = "64" if "x86_64" in (target or host_triple()) else ""
+            msvc_deps_dir = path.join(self.context.sharedir, "msvc-dependencies")
+            extra_path += [path.join(msvc_deps_dir, "cmake", "bin")]
+            extra_path += [path.join(msvc_deps_dir, "ninja", "bin")]
+            # Link openssl
+            env["OPENSSL_INCLUDE_DIR"] = path.join(msvc_deps_dir, "openssl", "include")
+            env["OPENSSL_LIB_DIR"] = path.join(msvc_deps_dir, "openssl", "lib" + msvc_x64)
+            env["OPENSSL_LIBS"] = "ssleay32MD:libeay32MD"
+            # Link moztools
+            env["MOZTOOLS_PATH"] = path.join(msvc_deps_dir, "moztools", "bin")
+
         if not self.config["tools"]["system-rust"] \
                 or self.config["tools"]["rust-root"]:
             env["RUST_ROOT"] = self.config["tools"]["rust-root"]
@@ -493,6 +505,8 @@ class CommandBase(object):
         if self.context.bootstrapped:
             return
 
+        target_platform = target or host_triple()
+
         rust_root = self.config["tools"]["rust-root"]
         rustc_path = path.join(
             rust_root, "rustc", "bin", "rustc" + BIN_SUFFIX
@@ -501,9 +515,13 @@ class CommandBase(object):
 
         base_target_path = path.join(rust_root, "rustc", "lib", "rustlib")
 
-        target_path = path.join(base_target_path, target or host_triple())
+        target_path = path.join(base_target_path, target_platform)
         target_exists = path.exists(target_path)
 
+        # Always check if all needed MSVC dependencies are installed
+        if "msvc" in target_platform:
+            Registrar.dispatch("bootstrap", context=self.context)
+
         if not (self.config['tools']['system-rust'] or (rustc_binary_exists and target_exists)):
             print("looking for rustc at %s" % (rustc_path))
             Registrar.dispatch("bootstrap-rust", context=self.context, target=filter(None, [target]),