Bug 1751948 - Part 3: Build IPDL sources within the directory they were declared, r=glandium

This change allows IPDL sources to respect FINAL_LIBRARY when building, which
is important for allowing us to build gtest-only IPDL_SOURCES files.

Differential Revision: https://phabricator.services.mozilla.com/D137167

layout/moz.build

@@ -27,6 +27,10 @@ IPDL_SOURCES = [
     "printing/ipc/PRemotePrintJob.ipdl",
 ]
 
+include("/ipc/chromium/chromium-config.mozbuild")
+
+FINAL_LIBRARY = "xul"
+
 if CONFIG["NS_PRINTING"]:
     DIRS += ["printing"]
 

python/mozbuild/mozbuild/backend/common.py

@@ -137,18 +137,11 @@ class CommonBackend(BuildBackend):
             self._handle_webidl_collection(obj)
 
         elif isinstance(obj, IPDLCollection):
-            self._handle_generated_sources(
-                mozpath.join(obj.objdir, f) for f in obj.all_generated_sources()
-            )
-            self._write_unified_files(
-                obj.unified_source_mapping, obj.objdir, poison_windows_h=False
-            )
             self._handle_ipdl_sources(
                 obj.objdir,
                 list(sorted(obj.all_sources())),
                 list(sorted(obj.all_preprocessed_sources())),
                 list(sorted(obj.all_regular_sources())),
-                obj.unified_source_mapping,
             )
 
         elif isinstance(obj, XPCOMComponentManifests):

python/mozbuild/mozbuild/backend/recursivemake.py

@@ -1763,7 +1763,6 @@ class RecursiveMakeBackend(MakeBackend):
         sorted_ipdl_sources,
         sorted_nonstatic_ipdl_sources,
         sorted_static_ipdl_sources,
-        unified_ipdl_cppsrcs_mapping,
     ):
         # Write out a master list of all IPDL source files.
         mk = Makefile()
@@ -1790,10 +1789,6 @@ class RecursiveMakeBackend(MakeBackend):
             )
         )
 
-        self._add_unified_build_rules(
-            mk, unified_ipdl_cppsrcs_mapping, unified_files_makefile_variable="CPPSRCS"
-        )
-
         # Preprocessed ipdl files are generated in ipdl_dir.
         mk.add_statement(
             "IPDLDIRS := %s %s"

python/mozbuild/mozbuild/compilation/database.py

@@ -184,10 +184,8 @@ class CompileDBBackend(CommonBackend):
         sorted_ipdl_sources,
         sorted_nonstatic_ipdl_sources,
         sorted_static_ipdl_sources,
-        unified_ipdl_cppsrcs_mapping,
     ):
-        for f in unified_ipdl_cppsrcs_mapping:
-            self._build_db_line(ipdl_dir, None, self.environment, f[0], ".cpp")
+        pass
 
     def _handle_webidl_build(
         self,

python/mozbuild/mozbuild/frontend/data.py

@@ -24,7 +24,6 @@ import mozpack.path as mozpath
 from .context import FinalTargetValue
 
 from collections import defaultdict, OrderedDict
-import itertools
 import six
 
 from ..util import group_unified_files
@@ -351,35 +350,11 @@ class IPDLCollection(ContextDerived):
     def all_preprocessed_sources(self):
         return self.preprocessed_sources
 
-    def all_generated_sources(self):
-        sorted_ipdl_sources = list(sorted(self.all_sources()))
-
-        def files_from(ipdl):
-            base = mozpath.basename(ipdl)
-            root, ext = mozpath.splitext(base)
-
-            # Both .ipdl and .ipdlh become .cpp files
-            files = ["%s.cpp" % root]
-            if ext == ".ipdl":
-                # .ipdl also becomes Child/Parent.cpp files
-                files.extend(["%sChild.cpp" % root, "%sParent.cpp" % root])
-            return files
-
-        return list(itertools.chain(*[files_from(p) for p in sorted_ipdl_sources]))
-
-    @property
-    def unified_source_mapping(self):
-        return list(
-            group_unified_files(
-                self.all_generated_sources(),
-                unified_prefix="UnifiedProtocols",
-                unified_suffix="cpp",
-                files_per_unified_file=16,
-            )
-        )
-
     def all_source_files(self):
-        return sorted(set(p for p, _ in self.unified_source_mapping))
+        # Source files generated by IPDL are built as generated UnifiedSources
+        # from the context which included the IPDL file, rather than the context
+        # which builds the IPDLCollection, so we report no files here.
+        return []
 
 
 class XPCOMComponentManifests(ContextDerived):

python/mozbuild/mozbuild/frontend/emitter.py

@@ -1050,6 +1050,27 @@ class TreeMetadataEmitter(LoggingMixin):
                         context,
                     )
 
+        # Process the .cpp files generated by IPDL as generated sources within
+        # the context which declared the IPDL_SOURCES attribute.
+        ipdl_root = self.config.substs.get("IPDL_ROOT")
+        for symbol in ("IPDL_SOURCES", "PREPROCESSED_IPDL_SOURCES"):
+            context_srcs = context.get(symbol, [])
+            for f in context_srcs:
+                root, ext = mozpath.splitext(mozpath.basename(f))
+
+                suffix_map = {
+                    ".ipdlh": [".cpp"],
+                    ".ipdl": [".cpp", "Child.cpp", "Parent.cpp"],
+                }
+                if ext not in suffix_map:
+                    raise SandboxValidationError(
+                        "Unexpected extension for IPDL source %s" % ext
+                    )
+
+                gen_sources["UNIFIED_SOURCES"].extend(
+                    mozpath.join(ipdl_root, root + suffix) for suffix in suffix_map[ext]
+                )
+
         no_pgo = context.get("NO_PGO")
         no_pgo_sources = [f for f, flags in six.iteritems(all_flags) if flags.no_pgo]
         if no_pgo:

python/mozbuild/mozbuild/test/backend/data/ipdl_sources/bar/moz.build

@@ -12,3 +12,5 @@ IPDL_SOURCES += [
     "bar.ipdl",
     "bar2.ipdlh",
 ]
+
+FINAL_LIBRARY = "dummy"

python/mozbuild/mozbuild/test/backend/data/ipdl_sources/foo/moz.build

@@ -12,3 +12,5 @@ IPDL_SOURCES += [
     "foo.ipdl",
     "foo2.ipdlh",
 ]
+
+FINAL_LIBRARY = "dummy"

python/mozbuild/mozbuild/test/backend/data/ipdl_sources/ipdl/moz.build

@@ -0,0 +1,9 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This file just exists to establish a directory as the IPDL root directory.
+
+FINAL_LIBRARY = "dummy"

python/mozbuild/mozbuild/test/backend/data/ipdl_sources/moz.build

@@ -4,7 +4,16 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+
+@template
+def Library(name):
+    LIBRARY_NAME = name
+
+
+Library("dummy")
+
 DIRS += [
     "bar",
     "foo",
+    "ipdl",
 ]

python/mozbuild/mozbuild/test/backend/test_recursivemake.py

@@ -793,33 +793,81 @@ class TestRecursiveMakeBackend(BackendTester):
         ipdlsrcs.mk correctly."""
         env = self._get_environment("ipdl_sources")
 
+        # Use the ipdl directory as the IPDL root for testing.
+        ipdl_root = mozpath.join(env.topobjdir, "ipdl")
+
         # Make substs writable so we can set the value of IPDL_ROOT to reflect
         # the correct objdir.
         env.substs = dict(env.substs)
-        env.substs["IPDL_ROOT"] = env.topobjdir
+        env.substs["IPDL_ROOT"] = ipdl_root
 
         self._consume("ipdl_sources", RecursiveMakeBackend, env)
 
-        manifest_path = mozpath.join(env.topobjdir, "ipdlsrcs.mk")
+        manifest_path = mozpath.join(ipdl_root, "ipdlsrcs.mk")
         lines = [l.strip() for l in open(manifest_path, "rt").readlines()]
 
         # Handle Windows paths correctly
-        topsrcdir = env.topsrcdir.replace(os.sep, "/")
+        topsrcdir = mozpath.normsep(env.topsrcdir)
 
         expected = [
             "ALL_IPDLSRCS := bar1.ipdl foo1.ipdl %s/bar/bar.ipdl %s/bar/bar2.ipdlh %s/foo/foo.ipdl %s/foo/foo2.ipdlh"  # noqa
             % tuple([topsrcdir] * 4),
-            "CPPSRCS := UnifiedProtocols0.cpp",
-            "IPDLDIRS := %s %s/bar %s/foo" % (env.topobjdir, topsrcdir, topsrcdir),
+            "IPDLDIRS := %s %s/bar %s/foo" % (ipdl_root, topsrcdir, topsrcdir),
         ]
 
-        found = [
-            str
-            for str in lines
-            if str.startswith(("ALL_IPDLSRCS", "CPPSRCS", "IPDLDIRS"))
-        ]
+        found = [str for str in lines if str.startswith(("ALL_IPDLSRCS", "IPDLDIRS"))]
         self.assertEqual(found, expected)
 
+        # Check that each directory declares the generated relevant .cpp files
+        # to be built in CPPSRCS.
+        # ENABLE_UNIFIED_BUILD defaults to False without mozilla-central's
+        # moz.configure so we don't see unified sources here.
+        for dir, expected in (
+            (".", []),
+            ("ipdl", []),
+            (
+                "bar",
+                [
+                    "CPPSRCS += "
+                    + " ".join(
+                        f"{ipdl_root}/{f}"
+                        for f in [
+                            "bar.cpp",
+                            "bar1.cpp",
+                            "bar1Child.cpp",
+                            "bar1Parent.cpp",
+                            "bar2.cpp",
+                            "barChild.cpp",
+                            "barParent.cpp",
+                        ]
+                    )
+                ],
+            ),
+            (
+                "foo",
+                [
+                    "CPPSRCS += "
+                    + " ".join(
+                        f"{ipdl_root}/{f}"
+                        for f in [
+                            "foo.cpp",
+                            "foo1.cpp",
+                            "foo1Child.cpp",
+                            "foo1Parent.cpp",
+                            "foo2.cpp",
+                            "fooChild.cpp",
+                            "fooParent.cpp",
+                        ]
+                    )
+                ],
+            ),
+        ):
+            backend_path = mozpath.join(env.topobjdir, dir, "backend.mk")
+            lines = [l.strip() for l in open(backend_path, "rt").readlines()]
+
+            found = [str for str in lines if str.startswith("CPPSRCS")]
+            self.assertEqual(found, expected)
+
     def test_defines(self):
         """Test that DEFINES are written to backend.mk correctly."""
         env = self._consume("defines", RecursiveMakeBackend)

python/mozbuild/mozbuild/test/frontend/test_emitter.py

@@ -1058,27 +1058,6 @@ class TestEmitterBasic(unittest.TestCase):
         expected = set(["bar/bar1.ipdl", "foo/foo1.ipdl"])
         self.assertEqual(pp_ipdls, expected)
 
-        generated_sources = set(ipdl_collection.all_generated_sources())
-        expected = set(
-            [
-                "bar.cpp",
-                "barChild.cpp",
-                "barParent.cpp",
-                "bar1.cpp",
-                "bar1Child.cpp",
-                "bar1Parent.cpp",
-                "bar2.cpp",
-                "foo.cpp",
-                "fooChild.cpp",
-                "fooParent.cpp",
-                "foo1.cpp",
-                "foo1Child.cpp",
-                "foo1Parent.cpp",
-                "foo2.cpp",
-            ]
-        )
-        self.assertEqual(generated_sources, expected)
-
     def test_local_includes(self):
         """Test that LOCAL_INCLUDES is emitted correctly."""
         reader = self.reader("local_includes")