gn win: Various toolchain fixes to get compile working

This fixes
"fatal error C1902: Program database manager mismatch; please check your installation"
and
"cannot find msvcr120.dll"
during gn compilation.

There were many things wrong. The primary ones were that:
1) the compiler's setup script wasn't called so we didn't get
   PATH/LIB/INCLUDE for the correct cl.
2) the VS runtime dirs were not getting added to environment block.

I also removed the Express handling, as that's been removed from the
gyp path now.

This does not yet support a system-installed Visual Studio. Code
needs to be added to query the registry to find the install location.
(see _SetupScript).

R=dpranke@chromium.org
TBR=brettw@chromium.org
BUG=432375

Review URL: https://codereview.chromium.org/736073002

Cr-Original-Commit-Position: refs/heads/master@{#304864}
Cr-Mirrored-From: https://chromium.googlesource.com/chromium/src
Cr-Mirrored-Commit: 9bf8fb3b74de68a85122cb45bb477d604b8c3898

config/win/BUILD.gn

@@ -16,19 +16,10 @@ config("sdk") {
     "NTDDI_VERSION=0x06030000",
     "PSAPI_VERSION=1",
     "WIN32",
+    "_SECURE_ATL",
   ]
 
   include_dirs = system_include_dirs
-
-  if (is_visual_studio_express) {
-    # https://code.google.com/p/chromium/issues/detail?id=372451#c20
-    # Warning 4702 ("Unreachable code") should be re-enabled once Express users
-    # are updated to VS2013 Update 2.
-    cflags = [ "/wd4702" ]
-  } else {
-    # Only supported on non-Express versions.
-    defines += [ "_SECURE_ATL" ]
-  }
 }
 
 # Sets the default Windows build version. This is separated because some
@@ -49,9 +40,6 @@ config("sdk_link") {
       "$visual_studio_path\VC\lib\amd64",
       "$visual_studio_path\VC\atlmfc\lib\amd64",
     ]
-    if (is_visual_studio_express) {
-      lib_dirs += [ "$wdk_path/lib/ATL/amd64" ]
-    }
   } else {
     ldflags = [
       "/MACHINE:X86",
@@ -62,26 +50,10 @@ config("sdk_link") {
       "$visual_studio_path\VC\lib",
       "$visual_studio_path\VC\atlmfc\lib",
     ]
-    if (is_visual_studio_express) {
-      lib_dirs += [ "$wdk_path/lib/ATL/i386" ]
-    }
     if (!is_asan) {
       ldflags += [ "/largeaddressaware" ]
     }
   }
-
-  if (is_visual_studio_express) {
-    # Explicitly required when using the ATL with express.
-    libs = [ "atlthunk.lib" ]
-
-    # ATL 8.0 included in WDK 7.1 makes the linker to generate almost eight
-    # hundred LNK4254 and LNK4078 warnings:
-    #   - warning LNK4254: section 'ATL' (50000040) merged into '.rdata'
-    #     (40000040) with different attributes
-    #   - warning LNK4078: multiple 'ATL' sections found with different
-    #     attributes
-    ldflags += [ "/ignore:4254", "/ignore:4078" ]
-  }
 }
 
 # This default linker setup is provided separately from the SDK setup so

config/win/visual_studio_version.gni

@@ -19,7 +19,7 @@ declare_args() {
   # Full path to the Windows SDK, not including a backslash at the end.
   # This value is the default location, override if you have a different
   # installation location.
-  windows_sdk_path = "C:\Program Files (x86)\Windows Kits\8.0"
+  windows_sdk_path = "C:\Program Files (x86)\Windows Kits\8.1"
 
   # The list of include directories that are treated as "system" include
   # directories. TODO(scottmg): These are incorrectly put on the command line
@@ -34,17 +34,15 @@ if (visual_studio_path == "") {
   windows_sdk_path = toolchain_data.sdk_path
   visual_studio_version = toolchain_data.vs_version
   wdk_path = toolchain_data.wdk_dir
+  visual_studio_runtime_dirs = toolchain_data.runtime_dirs
 } else {
   assert(visual_studio_version != "",
          "You must set the visual_studio_version if you set the path")
   assert(wdk_path != "",
          "You must set the wdk_path if you set the visual studio path")
+  visual_studio_runtime_dirs = []
 }
 
-# Set when using the "Express" version of a Visual Studio version we support.
-is_visual_studio_express = (visual_studio_version == "2013e")
-
-
 # The Windows SDK include directories must be first. They both have a sal.h,
 # and the SDK one is newer and the SDK uses some newer features from it not
 # present in the Visual Studio one.
@@ -55,10 +53,3 @@ system_include_dirs = [
   "$visual_studio_path\VC\include",
   "$visual_studio_path\VC\atlmfc\include",
 ]
-
-if (is_visual_studio_express) {
-  system_include_dirs += [
-    "$wdk_path/inc/atl71",
-    "$wdk_path/inc/mfc42",
-  ]
-}

toolchain/win/BUILD.gn

@@ -16,7 +16,12 @@ assert(is_win)
 gyp_win_tool_path = rebase_path("//tools/gyp/pylib/gyp/win_tool.py",
                                 root_build_dir)
 exec_script("setup_toolchain.py",
-            [ visual_studio_path, gyp_win_tool_path, windows_sdk_path ])
+            [
+              visual_studio_path,
+              gyp_win_tool_path,
+              windows_sdk_path,
+              visual_studio_runtime_dirs
+            ])
 
 # This value will be inherited in the toolchain below.
 concurrent_links = exec_script("../get_concurrent_links.py", [], "value")

toolchain/win/setup_toolchain.py

@@ -19,39 +19,59 @@ and the files will be written to the current directory.
 """
 
 
-def ExtractImportantEnvironment():
-  """Extracts environment variables required for the toolchain from the
-  current environment."""
+def _ExtractImportantEnvironment(output_of_set):
+  """Extracts environment variables required for the toolchain to run from
+  a textual dump output by the cmd.exe 'set' command."""
   envvars_to_save = (
-      'goma_.*',  # TODO(scottmg): This is ugly, but needed for goma.
-      'include',  # Needed by midl compiler.
+      'goma_.*', # TODO(scottmg): This is ugly, but needed for goma.
+      'include',
+      'lib',
+      'libpath',
       'path',
       'pathext',
       'systemroot',
       'temp',
       'tmp',
       )
-  result = {}
-  for envvar in envvars_to_save:
-    if envvar in os.environ:
-      envvar = envvar.lower()
-      if envvar == 'path':
-        # Our own rules (for running gyp-win-tool) and other actions in
-        # Chromium rely on python being in the path. Add the path to this
-        # python here so that if it's not in the path when ninja is run
-        # later, python will still be found.
-        result[envvar.upper()] = os.path.dirname(sys.executable) + \
-            os.pathsep + os.environ[envvar]
-      else:
-        result[envvar.upper()] = os.environ[envvar]
+  env = {}
+  for line in output_of_set.splitlines():
+    for envvar in envvars_to_save:
+      if re.match(envvar + '=', line.lower()):
+        var, setting = line.split('=', 1)
+        if envvar == 'path':
+          # Our own rules (for running gyp-win-tool) and other actions in
+          # Chromium rely on python being in the path. Add the path to this
+          # python here so that if it's not in the path when ninja is run
+          # later, python will still be found.
+          setting = os.path.dirname(sys.executable) + os.pathsep + setting
+        env[var.upper()] = setting
+        break
   for required in ('SYSTEMROOT', 'TEMP', 'TMP'):
-    if required not in result:
+    if required not in env:
       raise Exception('Environment variable "%s" '
                       'required to be set to valid path' % required)
-  return result
+  return env
 
 
-def FormatAsEnvironmentBlock(envvar_dict):
+def _SetupScript(target_arch, sdk_dir):
+  """Returns a command (with arguments) to be used to set up the
+  environment."""
+  # Check if we are running in the SDK command line environment and use
+  # the setup script from the SDK if so. |target_arch| should be either
+  # 'x86' or 'x64'.
+  assert target_arch in ('x86', 'x64')
+  if bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', 1))) and sdk_dir:
+    return [os.path.normpath(os.path.join(sdk_dir, 'Bin/SetEnv.Cmd')),
+            '/' + target_arch]
+  else:
+    # We only support x64-hosted tools.
+    # TODO(scottmg|dpranke): Non-depot_tools toolchain: need to get Visual
+    # Studio install location from registry.
+    return [os.path.normpath(os.path.join(FIND_VS_IN_REG, 'VC/vcvarsall.bat')),
+            'amd64_x86' if target_arch == 'x86' else 'amd64']
+
+
+def _FormatAsEnvironmentBlock(envvar_dict):
   """Format as an 'environment block' directly suitable for CreateProcess.
   Briefly this is a list of key=value\0, terminated by an additional \0. See
   CreateProcess documentation for more details."""
@@ -63,7 +83,7 @@ def FormatAsEnvironmentBlock(envvar_dict):
   return block
 
 
-def CopyTool(source_path):
+def _CopyTool(source_path):
   """Copies the given tool to the current directory, including a warning not
   to edit it."""
   with open(source_path) as source_file:
@@ -76,33 +96,39 @@ def CopyTool(source_path):
                              '# Generated by setup_toolchain.py do not edit.\n']
                             + tool_source[1:]))
 
-if len(sys.argv) != 4:
-  print('Usage setup_toolchain.py '
-        '<visual studio path> <win tool path> <win sdk path>')
-  sys.exit(2)
-vs_path = sys.argv[1]
-tool_source = sys.argv[2]
-win_sdk_path = sys.argv[3]
 
-CopyTool(tool_source)
+def main():
+  if len(sys.argv) != 5:
+    print('Usage setup_toolchain.py '
+          '<visual studio path> <win tool path> <win sdk path> <runtime dirs>')
+    sys.exit(2)
+  vs_path = sys.argv[1]
+  tool_source = sys.argv[2]
+  win_sdk_path = sys.argv[3]
+  runtime_dirs = sys.argv[4]
 
-important_env_vars = ExtractImportantEnvironment()
-path = important_env_vars["PATH"].split(";")
+  _CopyTool(tool_source)
 
-# Add 32-bit compiler path to the beginning and write the block.
-path32 = [os.path.join(vs_path, "VC\\BIN")] + \
-         [os.path.join(win_sdk_path, "bin\\x86")] + \
-         path
-important_env_vars["PATH"] = ";".join(path32)
-environ = FormatAsEnvironmentBlock(important_env_vars)
-with open('environment.x86', 'wb') as env_file:
-  env_file.write(environ)
+  archs = ('x86', 'x64')
+  # TODO(scottmg|goma): Do we need an equivalent of
+  # ninja_use_custom_environment_files?
+  for arch in archs:
+    # Extract environment variables for subprocesses.
+    args = _SetupScript(arch, win_sdk_path)
+    args.extend(('&&', 'set'))
+    popen = subprocess.Popen(
+        args, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    variables, _ = popen.communicate()
+    env = _ExtractImportantEnvironment(variables)
+    env['PATH'] = runtime_dirs + ';' + env['PATH']
 
-# Add 64-bit compiler path to the beginning and write the block.
-path64 = [os.path.join(vs_path, "VC\\BIN\\amd64")] + \
-         [os.path.join(win_sdk_path, "bin\\x64")] + \
-         path
-important_env_vars["PATH"] = ";".join(path64)
-environ = FormatAsEnvironmentBlock(important_env_vars)
-with open('environment.x64', 'wb') as env_file:
-  env_file.write(environ)
+    # TODO(scottmg|thakis|dpranke): Is there an equivalent to
+    # msvs_system_include_dirs that we need to inject into INCLUDE here?
+
+    env_block = _FormatAsEnvironmentBlock(env)
+    with open('environment.' + arch, 'wb') as f:
+      f.write(env_block)
+
+
+if __name__ == '__main__':
+  main()

vs_toolchain.py

@@ -184,11 +184,11 @@ def Update():
 def GetToolchainDir():
   """Gets location information about the current toolchain (must have been
   previously updated by 'update'). This is used for the GN build."""
-  SetEnvironmentAndGetRuntimeDllDirs()
+  runtime_dll_dirs = SetEnvironmentAndGetRuntimeDllDirs()
 
   # If WINDOWSSDKDIR is not set, search the default SDK path and set it.
   if not 'WINDOWSSDKDIR' in os.environ:
-    default_sdk_path = 'C:\\Program Files (x86)\\Windows Kits\\8.0'
+    default_sdk_path = 'C:\\Program Files (x86)\\Windows Kits\\8.1'
     if os.path.isdir(default_sdk_path):
       os.environ['WINDOWSSDKDIR'] = default_sdk_path
 
@@ -196,11 +196,13 @@ def GetToolchainDir():
 sdk_path = "%s"
 vs_version = "%s"
 wdk_dir = "%s"
+runtime_dirs = "%s"
 ''' % (
       os.environ['GYP_MSVS_OVERRIDE_PATH'],
       os.environ['WINDOWSSDKDIR'],
       os.environ['GYP_MSVS_VERSION'],
-      os.environ.get('WDK_DIR', ''))
+      os.environ.get('WDK_DIR', ''),
+      ';'.join(runtime_dll_dirs))
 
 
 def main():