Backed out 4 changesets (bug 1372381) for artifact build bustages

Backed out changeset 56c7ffeaa9fd (bug 1372381)
Backed out changeset 4e0f7fe818d0 (bug 1372381)
Backed out changeset 92a96d6b599b (bug 1372381)
Backed out changeset 620bdfafd42f (bug 1372381)

--HG--
extra : rebase_source : f3f465077a9a4198c88caf8977a02b384ccbef01

build/automation-build.mk

@@ -0,0 +1,67 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+include $(MOZILLA_DIR)/build/binary-location.mk
+
+browser_path := '"$(browser_path)"'
+
+_PROFILE_DIR = $(TARGET_DEPTH)/_profile/pgo
+
+ABSOLUTE_TOPSRCDIR = $(abspath $(MOZILLA_DIR))
+_CERTS_SRC_DIR = $(ABSOLUTE_TOPSRCDIR)/build/pgo/certs
+
+AUTOMATION_PPARGS = 	\
+			-DBROWSER_PATH=$(browser_path) \
+			-DXPC_BIN_PATH='"$(DIST)/bin"' \
+			-DBIN_SUFFIX='"$(BIN_SUFFIX)"' \
+			-DPROFILE_DIR='"$(_PROFILE_DIR)"' \
+			-DCERTS_SRC_DIR='"$(_CERTS_SRC_DIR)"' \
+			-DPERL='"$(PERL)"' \
+			$(NULL)
+
+ifeq ($(OS_ARCH),Darwin)
+AUTOMATION_PPARGS += -DIS_MAC=1
+else
+AUTOMATION_PPARGS += -DIS_MAC=0
+endif
+
+ifeq ($(OS_ARCH),Linux)
+AUTOMATION_PPARGS += -DIS_LINUX=1
+else
+AUTOMATION_PPARGS += -DIS_LINUX=0
+endif
+
+ifeq ($(host_os), cygwin)
+AUTOMATION_PPARGS += -DIS_CYGWIN=1
+endif
+
+ifeq ($(ENABLE_TESTS), 1)
+AUTOMATION_PPARGS += -DIS_TEST_BUILD=1
+else
+AUTOMATION_PPARGS += -DIS_TEST_BUILD=0
+endif
+
+ifeq ($(MOZ_DEBUG), 1)
+AUTOMATION_PPARGS += -DIS_DEBUG_BUILD=1
+else
+AUTOMATION_PPARGS += -DIS_DEBUG_BUILD=0
+endif
+
+ifdef MOZ_CRASHREPORTER
+AUTOMATION_PPARGS += -DCRASHREPORTER=1
+else
+AUTOMATION_PPARGS += -DCRASHREPORTER=0
+endif
+
+ifdef MOZ_ASAN
+AUTOMATION_PPARGS += -DIS_ASAN=1
+else
+AUTOMATION_PPARGS += -DIS_ASAN=0
+endif
+
+automation.py: $(MOZILLA_DIR)/build/automation.py.in $(MOZILLA_DIR)/build/automation-build.mk
+	$(call py_action,preprocessor, \
+	$(AUTOMATION_PPARGS) $(DEFINES) $(ACDEFINES) $< -o $@)
+
+GARBAGE += automation.py automation.pyc

build/automation.py.in

@@ -53,6 +53,7 @@ _IS_CYGWIN = False
 #endif
 #expand _BIN_SUFFIX = __BIN_SUFFIX__
 
+#expand _DEFAULT_APP = "./" + __BROWSER_PATH__
 #expand _CERTS_SRC_DIR = __CERTS_SRC_DIR__
 #expand _IS_TEST_BUILD = __IS_TEST_BUILD__
 #expand _IS_DEBUG_BUILD = __IS_DEBUG_BUILD__
@@ -98,6 +99,7 @@ class Automation(object):
 
   UNIXISH = not IS_WIN32 and not IS_MAC
 
+  DEFAULT_APP = _DEFAULT_APP
   CERTS_SRC_DIR = _CERTS_SRC_DIR
   IS_TEST_BUILD = _IS_TEST_BUILD
   IS_DEBUG_BUILD = _IS_DEBUG_BUILD
@@ -136,6 +138,7 @@ class Automation(object):
            "runApp",
            "Process",
            "DIST_BIN",
+           "DEFAULT_APP",
            "CERTS_SRC_DIR",
            "environment",
            "IS_TEST_BUILD",

build/gen_automation.py

@@ -1,48 +0,0 @@
-# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
-# vim: set filetype=python:
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distibuted with this
-# file, You can obtain one at http://mozilla.og/MPL/2.0/.
-
-import sys
-import buildconfig
-from mozbuild.preprocessor import Preprocessor
-
-
-def main(output, input_file):
-    pp = Preprocessor()
-    pp.context.update(buildconfig.defines['ALLDEFINES'])
-
-    substs = buildconfig.substs
-
-    # Substs taken verbatim.
-    substs_vars = (
-        'BIN_SUFFIX',
-    )
-    for var in substs_vars:
-        pp.context[var] = '"%s"' % substs[var]
-
-    # Derived values.
-    for key, condition in (
-            ('IS_MAC', substs['OS_ARCH'] == 'Darwin'),
-            ('IS_LINUX', substs['OS_ARCH'] == 'Linux'),
-            ('IS_TEST_BUILD', substs.get('ENABLE_TESTS') == '1'),
-            ('IS_DEBUG_BUILD', substs.get('MOZ_DEBUG') == '1'),
-            ('CRASHREPORTER', substs.get('MOZ_CRASHREPORTER')),
-            ('IS_ASAN', substs.get('MOZ_ASAN'))):
-        if condition:
-            pp.context[key] = '1'
-        else:
-            pp.context[key] = '0'
-
-    pp.context.update({
-        'XPC_BIN_PATH': '"%s/dist/bin"' % buildconfig.topobjdir,
-        'CERTS_SRC_DIR': '"%s/build/pgo/certs"' % buildconfig.topsrcdir,
-    })
-
-    pp.out = output
-    pp.do_include(input_file)
-
-
-if __name__ == '__main__':
-    main(*sys.agv[1:])

build/moz.build

@@ -96,12 +96,6 @@ if CONFIG['MOZ_APP_BASENAME']:
     appini.script = 'appini_header.py'
     appini.inputs = ['!application.ini']
 
-if CONFIG['ENABLE_TESTS']:
-    GENERATED_FILES += ['automation.py']
-    auto = GENERATED_FILES['automation.py']
-    auto.script = 'gen_automation.py'
-    auto.inputs = ['automation.py.in']
-
 # NOTE: Keep .gdbinit in the topsrcdir for people who run gdb from the topsrcdir.
 OBJDIR_FILES += ['/.gdbinit']
 

layout/tools/reftest/Makefile.in

@@ -7,6 +7,10 @@ _DEST_DIR = $(DEPTH)/_tests/reftest
 
 include $(topsrcdir)/config/rules.mk
 
+# We're installing to _tests/reftest
+TARGET_DEPTH = ../..
+include $(topsrcdir)/build/automation-build.mk
+
 # copy harness and the reftest extension bits to $(_DEST_DIR)
 # This needs to happen after jar.mn handling from rules.mk included above.
 # The order of the :: rules ensures that.

layout/tools/reftest/moz.build

@@ -14,8 +14,9 @@ JAR_MANIFESTS += ['jar.mn']
 FINAL_TARGET_PP_FILES += ['install.rdf']
 FINAL_TARGET_FILES += ['bootstrap.js']
 
+GENERATED_FILES += ['automation.py']
 TEST_HARNESS_FILES.reftest += [
-    '!/build/automation.py',
+    '!automation.py',
     '/build/mobile/remoteautomation.py',
     '/build/pgo/server-locations.txt',
     '/testing/mochitest/server.js',

python/mozbuild/mozbuild/backend/tup.py

@@ -40,10 +40,7 @@ from ..frontend.data import (
     ObjdirFiles,
     PerSourceFlag,
     Program,
-    SimpleProgram,
-    HostLibrary,
     HostProgram,
-    HostSimpleProgram,
     SharedLibrary,
     Sources,
     StaticLibrary,
@@ -56,7 +53,6 @@ from ..util import (
 from ..frontend.context import (
     AbsolutePath,
     ObjDirPath,
-    RenamedSourcePath,
 )
 
 
@@ -83,19 +79,9 @@ class BackendTupfile(object):
         self.variables = {}
         self.static_lib = None
         self.shared_lib = None
-        self.programs = []
-        self.host_programs = []
-        self.host_library = None
+        self.program = None
         self.exports = set()
 
-        # These files are special, ignore anything that generates them or
-        # depends on them.
-        self._skip_files = [
-            'signmar',
-            'libxul.so',
-            'libtestcrasher.so',
-        ]
-
         self.fh = FileAvoidWrite(self.name, capture_diff=True, dry_run=dry_run)
         self.fh.write('# THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT EDIT.\n')
         self.fh.write('\n')
@@ -112,11 +98,6 @@ class BackendTupfile(object):
              extra_inputs=None, extra_outputs=None, check_unchanged=False):
         inputs = inputs or []
         outputs = outputs or []
-
-        for f in inputs + outputs:
-            if any(f.endswith(skip_file) for skip_file in self._skip_files):
-                return
-
         display = display or ""
         self.include_rules()
         flags = ""
@@ -225,17 +206,11 @@ class TupBackend(CommonBackend):
         self._backend_files = {}
         self._cmd = MozbuildObject.from_environment()
         self._manifest_entries = OrderedDefaultDict(set)
-
-        # These are a hack to approximate things that are needed for the
-        # compile phase.
-        self._compile_env_files = (
-            '*.api',
+        self._compile_env_gen_files = (
             '*.c',
-            '*.cfg',
             '*.cpp',
             '*.h',
             '*.inc',
-            '*.msg',
             '*.py',
             '*.rs',
         )
@@ -308,6 +283,9 @@ class TupBackend(CommonBackend):
 
     def _gen_shared_library(self, backend_file):
         shlib = backend_file.shared_lib
+        if shlib.name == 'libxul.so':
+            # This will fail to link currently due to missing rust symbols.
+            return
 
         if shlib.cxx_link:
             mkshlib = (
@@ -335,6 +313,9 @@ class TupBackend(CommonBackend):
         list_file = self._make_list_file(backend_file.objdir, objs, list_file_name)
 
         inputs = objs + static_libs + shared_libs
+        if any(i.endswith('libxul.so') for i in inputs):
+            # Don't attempt to link anything that depends on libxul.
+            return
 
         symbols_file = []
         if shlib.symbols_file:
@@ -364,27 +345,23 @@ class TupBackend(CommonBackend):
                                                       shlib.install_target,
                                                       shlib.lib_name))
 
-    def _gen_programs(self, backend_file):
-        for p in backend_file.programs:
-            self._gen_program(backend_file, p)
 
-    def _gen_program(self, backend_file, prog):
-        cc_or_cxx = 'CXX' if prog.cxx_link else 'CC'
-        objs, _, shared_libs, os_libs, static_libs = self._expand_libs(prog)
+    def _gen_program(self, backend_file):
+        cc_or_cxx = 'CXX' if backend_file.program.cxx_link else 'CC'
+        objs, _, shared_libs, os_libs, static_libs = self._expand_libs(backend_file.program)
         static_libs = self._lib_paths(backend_file.objdir, static_libs)
         shared_libs = self._lib_paths(backend_file.objdir, shared_libs)
 
         inputs = objs + static_libs + shared_libs
+        if any(i.endswith('libxul.so') for i in inputs):
+            # Don't attempt to link anything that depends on libxul.
+            return
 
-        list_file_name = '%s.list' % prog.name.replace('.', '_')
+        list_file_name = '%s.list' % backend_file.program.name.replace('.', '_')
         list_file = self._make_list_file(backend_file.objdir, objs, list_file_name)
 
-        if isinstance(prog, SimpleProgram):
-            outputs = [prog.name]
-        else:
-            outputs = [mozpath.relpath(prog.output_path.full_path,
-                                       backend_file.objdir)]
-
+        outputs = [mozpath.relpath(backend_file.program.output_path.full_path,
+                                   backend_file.objdir)]
         cmd = (
             [backend_file.environment.substs[cc_or_cxx], '-o', '%o'] +
             backend_file.local_flags['CXX_LDFLAGS'] +
@@ -404,57 +381,6 @@ class TupBackend(CommonBackend):
         )
 
 
-    def _gen_host_library(self, backend_file):
-        objs = backend_file.host_library.objs
-        inputs = objs
-        outputs = [backend_file.host_library.name]
-        cmd = (
-            [backend_file.environment.substs['HOST_AR']] +
-            [backend_file.environment.substs['HOST_AR_FLAGS'].replace('$@', '%o')] +
-            objs
-        )
-        backend_file.rule(
-            cmd=cmd,
-            inputs=inputs,
-            outputs=outputs,
-            display='AR %o'
-        )
-
-
-    def _gen_host_programs(self, backend_file):
-        for p in backend_file.host_programs:
-            self._gen_host_program(backend_file, p)
-
-
-    def _gen_host_program(self, backend_file, prog):
-        _, _, _, extra_libs, _ = self._expand_libs(prog)
-        objs = prog.objs
-        outputs = [prog.program]
-        host_libs = []
-        for lib in prog.linked_libraries:
-            if isinstance(lib, HostLibrary):
-                host_libs.append(lib)
-        host_libs = self._lib_paths(backend_file.objdir, host_libs)
-
-        inputs = objs + host_libs
-        use_cxx = any(f.endswith(('.cc', '.cpp')) for f in prog.source_files())
-        cc_or_cxx = 'HOST_CXX' if use_cxx else 'HOST_CC'
-        cmd = (
-            [backend_file.environment.substs[cc_or_cxx], '-o', '%o'] +
-            backend_file.local_flags['HOST_CXX_LDFLAGS'] +
-            backend_file.local_flags['HOST_LDFLAGS'] +
-            objs +
-            host_libs +
-            extra_libs
-        )
-        backend_file.rule(
-            cmd=cmd,
-            inputs=inputs,
-            outputs=outputs,
-            display='LINK %o'
-        )
-
-
     def _gen_static_library(self, backend_file):
         ar = [
             backend_file.environment.substs['AR'],
@@ -496,7 +422,7 @@ class TupBackend(CommonBackend):
             skip_files = []
 
             if self.environment.is_artifact_build:
-                skip_files = self._compile_env_gen
+                skip_files = self._compile_env_gen_files
 
             for f in obj.outputs:
                 if any(mozpath.match(f, p) for p in skip_files):
@@ -538,12 +464,10 @@ class TupBackend(CommonBackend):
             backend_file.static_lib = obj
         elif isinstance(obj, SharedLibrary):
             backend_file.shared_lib = obj
-        elif isinstance(obj, (HostProgram, HostSimpleProgram)):
-            backend_file.host_programs.append(obj)
-        elif isinstance(obj, HostLibrary):
-            backend_file.host_library = obj
-        elif isinstance(obj, (Program, SimpleProgram)):
-            backend_file.programs.append(obj)
+        elif isinstance(obj, HostProgram):
+            pass
+        elif isinstance(obj, Program):
+            backend_file.program = obj
         elif isinstance(obj, DirectoryTraversal):
             pass
 
@@ -566,13 +490,11 @@ class TupBackend(CommonBackend):
 
         for objdir, backend_file in sorted(self._backend_files.items()):
             backend_file.gen_sources_rules([self._installed_files])
-            for var, gen_method in ((backend_file.shared_lib, self._gen_shared_library),
-                                    (backend_file.static_lib and backend_file.static_lib.no_expand_lib,
-                                     self._gen_static_library),
-                                    (backend_file.programs, self._gen_programs),
-                                    (backend_file.host_programs, self._gen_host_programs),
-                                    (backend_file.host_library, self._gen_host_library)):
-                if var:
+            for condition, gen_method in ((backend_file.shared_lib, self._gen_shared_library),
+                                          (backend_file.static_lib and backend_file.static_lib.no_expand_lib,
+                                           self._gen_static_library),
+                                          (backend_file.program, self._gen_program)):
+                if condition:
                     backend_file.export_shell()
                     gen_method(backend_file)
             for obj in backend_file.delayed_generated_files:
@@ -617,6 +539,7 @@ class TupBackend(CommonBackend):
         # TODO: These are directories that don't work in the tup backend
         # yet, because things they depend on aren't built yet.
         skip_directories = (
+            'layout/style/test', # HostSimplePrograms
             'toolkit/library', # libxul.so
         )
         if obj.script and obj.method and obj.relobjdir not in skip_directories:
@@ -646,8 +569,7 @@ class TupBackend(CommonBackend):
                 if exports:
                     backend_file.export(exports)
 
-            if any(f.endswith(('automation.py', 'source-repo.h', 'buildid.h'))
-                   for f in obj.outputs):
+            if any(f in obj.outputs for f in ('source-repo.h', 'buildid.h')):
                 extra_outputs = [self._early_generated_files]
             else:
                 extra_outputs = [self._installed_files] if obj.required_for_compile else []
@@ -688,14 +610,14 @@ class TupBackend(CommonBackend):
             if not path:
                 raise Exception("Cannot install to " + target)
 
+        if target.startswith('_tests'):
+            # TODO: TEST_HARNESS_FILES present a few challenges for the tup
+            # backend (bug 1372381).
+            return
+
         for path, files in obj.files.walk():
             self._add_features(target, path)
             for f in files:
-                output_group = None
-                if any(mozpath.match(mozpath.basename(f), p)
-                       for p in self._compile_env_files):
-                    output_group = self._installed_files
-
                 if not isinstance(f, ObjDirPath):
                     backend_file = self._get_backend_file(mozpath.join(target, path))
                     if '*' in f:
@@ -715,27 +637,13 @@ class TupBackend(CommonBackend):
                                         yield p + '/'
                             prefix = ''.join(_prefix(f.full_path))
                             self.backend_input_files.add(prefix)
-
-                            output_dir = ''
-                            # If we have a RenamedSourcePath here, the common backend
-                            # has generated this object from a jar manifest, and we
-                            # can rely on 'path' to be our destination path relative
-                            # to any wildcard match. Otherwise, the output file may
-                            # contribute to our destination directory.
-                            if not isinstance(f, RenamedSourcePath):
-                                output_dir = ''.join(_prefix(mozpath.dirname(f)))
-
                             finder = FileFinder(prefix)
                             for p, _ in finder.find(f.full_path[len(prefix):]):
-                                install_dir = prefix[len(obj.srcdir) + 1:]
-                                output = p
-                                if f.target_basename and '*' not in f.target_basename:
-                                    output = mozpath.join(f.target_basename, output)
                                 backend_file.symlink_rule(mozpath.join(prefix, p),
-                                                          output=mozpath.join(output_dir, output),
-                                                          output_group=output_group)
+                                                          output=mozpath.join(f.target_basename, p),
+                                                          output_group=self._installed_files)
                     else:
-                        backend_file.symlink_rule(f.full_path, output=f.target_basename, output_group=output_group)
+                        backend_file.symlink_rule(f.full_path, output=f.target_basename, output_group=self._installed_files)
                 else:
                     if (self.environment.is_artifact_build and
                         any(mozpath.match(f.target_basename, p) for p in self._compile_env_gen_files)):
@@ -745,16 +653,18 @@ class TupBackend(CommonBackend):
 
                     # We're not generating files in these directories yet, so
                     # don't attempt to install files generated from them.
-                    if f.context.relobjdir not in ('toolkit/library',
+                    if f.context.relobjdir not in ('layout/style/test',
+                                                   'toolkit/library',
                                                    'js/src/shell'):
                         output = mozpath.join('$(MOZ_OBJ_ROOT)', target, path,
                                               f.target_basename)
                         gen_backend_file = self._get_backend_file(f.context.relobjdir)
                         if gen_backend_file.requires_delay([f]):
+                            output_group = self._installed_files if f.target_basename.endswith('.h') else None
                             gen_backend_file.delayed_installed_files.append((f.full_path, output, output_group))
                         else:
                             gen_backend_file.symlink_rule(f.full_path, output=output,
-                                                          output_group=output_group)
+                                                          output_group=self._installed_files)
 
 
     def _process_final_target_pp_files(self, obj, backend_file):

python/mozbuild/mozbuild/frontend/data.py

@@ -532,14 +532,6 @@ class HostSimpleProgram(HostMixin, BaseProgram):
     SUFFIX_VAR = 'HOST_BIN_SUFFIX'
     KIND = 'host'
 
-    def source_files(self):
-        for srcs in self.sources.values():
-            for f in srcs:
-                if ('host_%s' % mozpath.basename(mozpath.splitext(f)[0]) ==
-                    mozpath.splitext(self.program)[0]):
-                    return [f]
-        return []
-
 
 def cargo_output_directory(context, target_var):
     # cargo creates several directories and places its build artifacts

testing/mochitest/Makefile.in

@@ -7,6 +7,10 @@
 _DEST_DIR = $(DEPTH)/_tests/$(relativesrcdir)
 
 include $(topsrcdir)/config/rules.mk
+# We're installing to _tests/testing/mochitest, so this is the depth
+# necessary for relative objdir paths.
+TARGET_DEPTH = ../../..
+include $(topsrcdir)/build/automation-build.mk
 
 libs:: 
 	(cd $(DIST)/xpi-stage && tar $(TAR_CREATE_FLAGS) - mochijar) | (cd $(_DEST_DIR) && tar -xf -)

testing/mochitest/moz.build

@@ -26,8 +26,12 @@ MOCHITEST_MANIFESTS += [
 ]
 MOCHITEST_CHROME_MANIFESTS += ['chrome/chrome.ini']
 
+GENERATED_FILES += [
+    'automation.py',
+]
+
 TEST_HARNESS_FILES.testing.mochitest += [
-    '!/build/automation.py',
+    '!automation.py',
     '/build/mobile/remoteautomation.py',
     '/build/pgo/server-locations.txt',
     '/build/sanitizers/lsan_suppressions.txt',