bug 1401647 - use a 64-bit Rust toolchain for win32 builds. r=rillian

We currently use a 32-bit Rust toolchain for win32 builds, but this can lead
to OOM situations. This patch makes win32 builds use a 64-bit Rust toolchain,
which requires a little bit of extra configuration because rustc needs to
be able to find a link.exe that produces 64-bit binaries for building
things like build scripts, which are host binaries.

We will now generate a batch file that sets LIB to the paths to 64-bit
libraries and invokes the x64-targeting link.exe, and add a section to the
.cargo/config file to instruct cargo to use that batch file as the linker
when producing 64-bit binaries.

MozReview-Commit-ID: 9vKBbm7Gvra

--HG--
extra : rebase_source : 366dd966cafe4f07b8e59fc170d2db2dada32627

.cargo/config.in

@@ -9,3 +9,5 @@ replace-with = "vendored-sources"
 
 [source.vendored-sources]
 directory = '@top_srcdir@/third_party/rust'
+
+@WIN64_CARGO_LINKER_CONFIG@

build/moz.build

@@ -12,6 +12,8 @@ SPHINX_TREES['buildsystem'] = 'docs'
 
 if CONFIG['OS_ARCH'] == 'WINNT':
     DIRS += ['win32']
+    if CONFIG['WIN64_CARGO_LINKER']:
+        CONFIGURE_SUBST_FILES += ['win64/cargo-linker.bat']
 else:
     DIRS += ['unix']
 

build/moz.configure/rust.configure

@@ -26,6 +26,7 @@ def rustc_info(rustc):
     return namespace(
         version=Version(info.get('release', '0')),
         commit=info.get('commit-hash', 'unknown'),
+        host=info['host'],
     )
 
 
@@ -263,3 +264,39 @@ option('--enable-rust-tests',
        help='Enable building of Rust tests, and build-time execution of them')
 
 set_config('MOZ_RUST_TESTS', depends('--enable-rust-tests')(lambda v: bool(v)))
+
+option(env='WIN64_LINK', nargs=1, help='Path to link.exe that targets win64')
+option(env='WIN64_LIB', nargs=1, help='Paths to libraries for the win64 linker')
+
+set_config('WIN64_LINK', depends('WIN64_LINK')(lambda x: x))
+set_config('WIN64_LIB', depends('WIN64_LIB')(lambda x: x))
+
+
+@depends(target, rustc_info, c_compiler, 'WIN64_LINK', 'WIN64_LIB')
+def win64_cargo_linker(target, rustc_info, compiler_info, link, lib):
+    # When we're building a 32-bit Windows build with a 64-bit rustc, we
+    # need to configure the linker it will use for host binaries (build scripts)
+    # specially because the compiler configuration we use for the build is for
+    # MSVC targeting 32-bit binaries.
+    if target.kernel == 'WINNT' and target.cpu == 'x86' and \
+       compiler_info.type in ('msvc', 'clang-cl') and \
+       rustc_info.host == 'x86_64-pc-windows-msvc' and link and lib:
+        return True
+
+
+set_config('WIN64_CARGO_LINKER', win64_cargo_linker)
+
+
+@depends(win64_cargo_linker, check_build_environment)
+@imports(_from='textwrap', _import='dedent')
+def win64_cargo_linker_config(linker, env):
+    if linker:
+        return dedent('''\
+        [target.x86_64-pc-windows-msvc]
+        linker = "{objdir}/build/win64/cargo-linker.bat"
+        '''.format(objdir=env.topobjdir))
+    # We want an empty string here so we don't leave the @ variable in the config file.
+    return ''
+
+
+set_config('WIN64_CARGO_LINKER_CONFIG', win64_cargo_linker_config)

build/win32/mozconfig.vs2017

@@ -15,6 +15,9 @@ if [ -d "${VSPATH}" ]; then
 
     export INCLUDE="${VSPATH}/VC/include:${VSPATH}/VC/atlmfc/include:${VSPATH}/SDK/Include/10.0.15063.0/ucrt:${VSPATH}/SDK/Include/10.0.15063.0/shared:${VSPATH}/SDK/Include/10.0.15063.0/um:${VSPATH}/SDK/Include/10.0.15063.0/winrt:${VSPATH}/DIA SDK/include"
     export LIB="${VSPATH}/VC/lib/x86:${VSPATH}/VC/atlmfc/lib/x86:${VSPATH}/SDK/Lib/10.0.15063.0/ucrt/x86:${VSPATH}/SDK/Lib/10.0.15063.0/um/x86:${VSPATH}/DIA SDK/lib"
+
+    export WIN64_LINK="${VSPATH}/VC/bin/Hostx64/x64/link.exe"
+    export WIN64_LIB="${VSPATH}/VC/lib/x64:${VSPATH}/VC/atlmfc/lib/x64:${VSPATH}/SDK/Lib/10.0.15063.0/ucrt/x64:${VSPATH}/SDK/Lib/10.0.15063.0/um/x64:${VSPATH}/DIA SDK/lib/amd64"
 fi
 
 . $topsrcdir/build/mozconfig.vs-common

build/win64/cargo-linker.bat.in

@@ -0,0 +1,3 @@
+set LIB=@WIN64_LIB@
+
+@WIN64_LINK@ %*

python/mozbuild/mozbuild/test/configure/test_toolchain_configure.py

@@ -1420,7 +1420,7 @@ class RustTest(BaseConfigureTest):
 
     def invoke_rustc(self, stdin, args):
         if args == ('--version', '--verbose'):
-            return 0, 'rustc 2.0\nrelease: 2.0', ''
+            return 0, 'rustc 2.0\nrelease: 2.0\nhost: x86_64-unknown-linux-gnu', ''
         if args == ('--print', 'target-list'):
             # Raw list returned by rustc version 1.19, + ios, which somehow
             # don't appear in the default list.

taskcluster/ci/build/windows.yml

@@ -22,8 +22,8 @@ win32/debug:
             - builds/taskcluster_base_win32.py
             - builds/taskcluster_sub_win32/debug.py
     toolchains:
-        - win32-clang-cl
-        - win32-rust
+        - win64-clang-cl
+        - win64-rust
         - win64-sccache
 
 win32/opt:
@@ -50,8 +50,8 @@ win32/opt:
             - builds/taskcluster_base_win32.py
             - builds/taskcluster_sub_win32/opt.py
     toolchains:
-        - win32-clang-cl
-        - win32-rust
+        - win64-clang-cl
+        - win64-rust
         - win64-sccache
 
 win32-dmd/opt:
@@ -81,8 +81,8 @@ win32-dmd/opt:
             mozconfig_variant: 'opt-dmd'
     run-on-projects: []
     toolchains:
-        - win32-clang-cl
-        - win32-rust
+        - win64-clang-cl
+        - win64-rust
         - win64-sccache
 
 win32/pgo:
@@ -109,8 +109,8 @@ win32/pgo:
             - builds/taskcluster_base_win32.py
             - builds/taskcluster_sub_win32/opt.py
     toolchains:
-        - win32-clang-cl
-        - win32-rust
+        - win64-clang-cl
+        - win64-rust
         - win64-sccache
 
 win64/debug:
@@ -232,8 +232,8 @@ win32-nightly/opt:
             - disable_signing.py
             - taskcluster_nightly.py
     toolchains:
-        - win32-clang-cl
-        - win32-rust
+        - win64-clang-cl
+        - win64-rust
         - win64-sccache
 
 win64-nightly/opt:
@@ -326,8 +326,8 @@ win32-add-on-devel/opt:
             - balrog/production.py
     run-on-projects: ['mozilla-beta', 'mozilla-release', 'mozilla-esr45']
     toolchains:
-        - win32-clang-cl
-        - win32-rust
+        - win64-clang-cl
+        - win64-rust
         - win64-sccache
 
 win64-add-on-devel/opt:
@@ -414,8 +414,8 @@ win32-noopt/debug:
             - builds/taskcluster_sub_win32/noopt_debug.py
     run-on-projects: ['trunk', 'try']
     toolchains:
-        - win32-clang-cl
-        - win32-rust
+        - win64-clang-cl
+        - win64-rust
         - win64-sccache
 
 win32-rusttests/opt:
@@ -444,8 +444,8 @@ win32-rusttests/opt:
             - builds/taskcluster_sub_win32/rusttests_opt.py
     run-on-projects: ['trunk', 'try']
     toolchains:
-        - win32-clang-cl
-        - win32-rust
+        - win64-clang-cl
+        - win64-rust
         - win64-sccache
 
 win64-rusttests/opt:
@@ -600,8 +600,8 @@ win32-devedition-nightly/opt:
         custom-build-variant-cfg: devedition
     run-on-projects: ['mozilla-beta']
     toolchains:
-        - win32-clang-cl
-        - win32-rust
+        - win64-clang-cl
+        - win64-rust
         - win64-sccache
 
 win64-devedition-nightly/opt: