From e83738f7d834ae7ff8b966076bfc36b70e7dc9e0 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Tue, 25 Jan 2022 22:43:56 +0000 Subject: [PATCH] Bug 1629184 - Preprocess MIDL inputs manually. r=firefox-build-system-reviewers,nalexander This avoids needing clang-cl.exe on cross compilations. We could keep Windows builds on having MIDL do the preprocessing, but that would be a difference between native and cross builds, and it's better to avoid that. Differential Revision: https://phabricator.services.mozilla.com/D136831 --- build/midl.py | 123 +++++++++++++++++++++++++++++++++++++----- toolkit/moz.configure | 21 +------- 2 files changed, 112 insertions(+), 32 deletions(-) diff --git a/build/midl.py b/build/midl.py index add17006d61c..3480e8df3ad1 100644 --- a/build/midl.py +++ b/build/midl.py @@ -3,6 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. import buildconfig +import shutil import subprocess import os import sys @@ -23,22 +24,118 @@ def relativize(path, base=None): return os.path.relpath(path, base) +def search_path(paths, path): + for p in paths: + f = os.path.join(p, path) + if os.path.isfile(f): + return f + raise RuntimeError(f"Cannot find {path}") + + +# Preprocess all the direct and indirect inputs of midl, and put all the +# preprocessed inputs in the given `base` directory. Returns a tuple containing +# the path of the main preprocessed input, and the modified flags to use instead +# of the flags given as argument. +def preprocess(base, input, flags): + import argparse + import re + from collections import deque + + IMPORT_RE = re.compile('import\s*"([^"]+)";') + + parser = argparse.ArgumentParser() + parser.add_argument("-I", action="append") + parser.add_argument("-D", action="append") + parser.add_argument("-acf") + args, remainder = parser.parse_known_args(flags) + preprocessor = ( + [buildconfig.substs["_CXX"]] + # Ideally we'd use the real midl version, but querying it adds a + # significant overhead to configure. In practice, the version number + # doesn't make a difference at the moment. + + ["-E", "-D__midl=801"] + + [f"-D{d}" for d in args.D or ()] + + [f"-I{i}" for i in args.I or ()] + ) + includes = ["."] + buildconfig.substs["INCLUDE"].split(";") + (args.I or []) + seen = set() + queue = deque([input]) + if args.acf: + queue.append(args.acf) + output = os.path.join(base, os.path.basename(input)) + while True: + try: + input = queue.popleft() + except IndexError: + break + if os.path.basename(input) in seen: + continue + seen.add(os.path.basename(input)) + input = search_path(includes, input) + # If there is a .acf file corresponding to the .idl we're processing, + # we also want to preprocess that file because midl might look for it too. + if input.endswith(".idl") and os.path.exists(input[:-4] + ".acf"): + queue.append(input[:-4] + ".acf") + command = preprocessor + [input] + preprocessed = os.path.join(base, os.path.basename(input)) + subprocess.run(command, stdout=open(preprocessed, "wb"), check=True) + # Read the resulting file, and search for imports, that we'll want to + # preprocess as well. + with open(preprocessed, "r") as fh: + for line in fh: + if not line.startswith("import"): + continue + m = IMPORT_RE.match(line) + if not m: + continue + imp = m.group(1) + queue.append(imp) + flags = [] + # Add -I first in the flags, so that midl resolves imports to the + # preprocessed files we created. + for i in [base] + (args.I or []): + flags.extend(["-I", i]) + # Add the preprocessed acf file if one was given on the command line. + if args.acf: + flags.extend(["-acf", os.path.join(base, os.path.basename(args.acf))]) + flags.extend(remainder) + return output, flags + + def midl(out, input, *flags): out.avoid_writing_to_file() - midl = buildconfig.substs["MIDL"] - wine = buildconfig.substs.get("WINE") + midl_flags = buildconfig.substs["MIDL_FLAGS"] base = os.path.dirname(out.name) or "." - if midl.lower().endswith(".exe") and wine: - command = [wine, midl] - else: - command = [midl] - command.extend(buildconfig.substs["MIDL_FLAGS"]) - command.extend([relativize(f, base) for f in flags]) - command.append("-Oicf") - command.append(relativize(input, base)) - print("Executing:", " ".join(command)) - result = subprocess.run(command, cwd=base) - return result.returncode + tmpdir = None + try: + # If the build system is asking to not use the preprocessor to midl, + # we need to do the preprocessing ourselves. + if "-no_cpp" in midl_flags: + # Normally, we'd use tempfile.TemporaryDirectory, but in this specific + # case, we actually want a deterministic directory name, because it's + # recorded in the code midl generates. + tmpdir = os.path.join(base, os.path.basename(input) + ".tmp") + os.makedirs(tmpdir, exist_ok=True) + try: + input, flags = preprocess(tmpdir, input, flags) + except subprocess.CalledProcessError as e: + return e.returncode + midl = buildconfig.substs["MIDL"] + wine = buildconfig.substs.get("WINE") + if midl.lower().endswith(".exe") and wine: + command = [wine, midl] + else: + command = [midl] + command.extend(midl_flags) + command.extend([relativize(f, base) for f in flags]) + command.append("-Oicf") + command.append(relativize(input, base)) + print("Executing:", " ".join(command)) + result = subprocess.run(command, cwd=base) + return result.returncode + finally: + if tmpdir: + shutil.rmtree(tmpdir) # midl outputs dlldata to a single dlldata.c file by default. This prevents running diff --git a/toolkit/moz.configure b/toolkit/moz.configure index 2b6675312d4b..60df0ace9df5 100644 --- a/toolkit/moz.configure +++ b/toolkit/moz.configure @@ -1358,13 +1358,11 @@ option(env="MIDL_FLAGS", nargs=1, help="Extra flags to pass to MIDL") @depends( "MIDL_FLAGS", - c_compiler, target, - host, midl, when=depends(midl, target)(lambda m, t: m and t.kernel == "WINNT"), ) -def midl_flags(flags, c_compiler, target, host, midl): +def midl_flags(flags, target, midl): if flags: flags = flags[0].split() else: @@ -1376,22 +1374,7 @@ def midl_flags(flags, c_compiler, target, host, midl): "x86_64": "x64", "aarch64": "arm64", }[target.cpu] - flags += ["-env", env] - - if host.os == "WINNT": - return flags + ["-cpp_cmd", c_compiler.compiler] - - # If cross-compiling and using midl instead of widl, for now, we'll - # assume we can find the Windows version of clang-cl in the PATH. - # It is required because while Wine is able to spawn Linux - # processes from Windows programs(!), the calling program doesn't - # have access to the process output and can't wait for it to - # finish. Midl runs clang-cl as a preprocessor and expects to read - # its output... - clang_cl_exe = find_program("clang-cl.exe") - if not clang_cl_exe: - die("Cannot find clang-cl.exe") - return flags + ["-cpp_cmd", clang_cl_exe] + return flags + ["-nologo", "-no_cpp", "-env", env] # widl return flags + {