Bug 1575471 - Move `make check` from mozharness to moz-automation.mk. r=froydnj

Currently, we have no real visibility on the time spent after the build
finished, despite the fact that a large chunk is actually happening via
make check (although thankfully more and more of it is moving out to
separate tasks).

Also, the mozharness machinery for make check dates from when we were
running in buildbot and takes care of turning builds orange instead of
red in case of failure, etc. which doesn't do anything useful anymore.

Differential Revision: https://phabricator.services.mozilla.com/D42806

--HG--
extra : moz-landing-system : lando

build/moz-automation.mk

@@ -8,6 +8,22 @@ ifndef ENABLE_TESTS
 MOZ_AUTOMATION_PACKAGE_TESTS = 0
 endif
 
+ifdef CROSS_COMPILE
+# Narrow the definition of cross compilation to not include win32 builds
+# on win64 and linux32 builds on linux64.
+ifeq ($(HOST_OS_ARCH),$(OS_TARGET))
+FUZZY_CROSS_COMPILE =
+else
+FUZZY_CROSS_COMPILE = 1
+endif
+endif
+
+# Don't run make check when cross compiling, when doing artifact builds
+# or when building instrumented builds for PGO.
+ifneq (,$(USE_ARTIFACT)$(FUZZY_CROSS_COMPILE)$(MOZ_PROFILE_GENERATE))
+MOZ_AUTOMATION_CHECK := 0
+endif
+
 ifneq (,$(filter automation/%,$(MAKECMDGOALS)))
 ifeq (4.0,$(firstword $(sort 4.0 $(MAKE_VERSION))))
 MAKEFLAGS += --output-sync=target
@@ -33,6 +49,7 @@ tier_MOZ_AUTOMATION_PACKAGE_TESTS = package-tests
 tier_MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES = package-generated-sources
 tier_MOZ_AUTOMATION_UPLOAD_SYMBOLS = uploadsymbols
 tier_MOZ_AUTOMATION_UPLOAD = upload
+tier_MOZ_AUTOMATION_CHECK = check
 
 # Automation build steps. Everything in MOZ_AUTOMATION_TIERS also gets used in
 # TIERS for mach display. As such, the MOZ_AUTOMATION_TIERS are roughly sorted
@@ -46,6 +63,7 @@ moz_automation_symbols = \
   MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES \
   MOZ_AUTOMATION_L10N_CHECK \
   MOZ_AUTOMATION_UPLOAD \
+  MOZ_AUTOMATION_CHECK \
   $(NULL)
 MOZ_AUTOMATION_TIERS := $(foreach sym,$(moz_automation_symbols),$(if $(filter 1,$($(sym))),$(tier_$(sym))))
 
@@ -59,12 +77,18 @@ automation-start/upload: automation/package-tests
 automation-start/upload: automation/buildsymbols
 automation-start/upload: automation/package-generated-sources
 
+# Run the check tier after everything else.
+automation-start/check: $(addprefix automation/,$(filter-out check,$(MOZ_AUTOMATION_TIERS)))
+
 automation/build: $(addprefix automation/,$(MOZ_AUTOMATION_TIERS))
 	@echo Automation steps completed.
 
 # Note: We have to force -j1 here, at least until bug 1036563 is fixed.
 AUTOMATION_EXTRA_CMDLINE-l10n-check = -j1
 
+# Run as many tests as possible, even in case of one of them failing.
+AUTOMATION_EXTRA_CMDLINE-check = --keep-going
+
 # The commands only run if the corresponding MOZ_AUTOMATION_* variable is
 # enabled. This means, for example, if we enable MOZ_AUTOMATION_UPLOAD, then
 # 'buildsymbols' will only run if MOZ_AUTOMATION_BUILD_SYMBOLS is also set.

build/mozconfig.automation

@@ -15,5 +15,6 @@ mk_add_options "export MOZ_AUTOMATION_PACKAGE=${MOZ_AUTOMATION_PACKAGE-1}"
 mk_add_options "export MOZ_AUTOMATION_PACKAGE_TESTS=${MOZ_AUTOMATION_PACKAGE_TESTS-1}"
 mk_add_options "export MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES=${MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES-1}"
 mk_add_options "export MOZ_AUTOMATION_UPLOAD=${MOZ_AUTOMATION_UPLOAD-1}"
+mk_add_options "export MOZ_AUTOMATION_CHECK=${MOZ_AUTOMATION_CHECK-1}"
 
 export MOZ_AUTOMATION_MOZCONFIG=1

taskcluster/ci/build/linux.yml

@@ -17,7 +17,7 @@ linux64/opt:
         max-run-time: 7200
     run:
         using: mozharness
-        actions: [get-secrets, build, check-test]
+        actions: [get-secrets, build]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_64_builds.py
@@ -89,7 +89,7 @@ linux64-fuzzing/debug:
             FORCE_GCC: '1'
     run:
         using: mozharness
-        actions: [get-secrets, build, check-test]
+        actions: [get-secrets, build]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_64_builds.py
@@ -125,7 +125,7 @@ linux64/debug:
         max-run-time: 5400
     run:
         using: mozharness
-        actions: [get-secrets, build, check-test]
+        actions: [get-secrets, build]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_64_builds.py
@@ -202,7 +202,7 @@ linux64-devedition-nightly/opt:
         max-run-time: 7200
     run:
         using: mozharness
-        actions: [get-secrets, build, check-test]
+        actions: [get-secrets, build]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_64_builds.py
@@ -242,7 +242,7 @@ linux64-base-toolchains/opt:
             FORCE_GCC: '1'
     run:
         using: mozharness
-        actions: [get-secrets, build, check-test]
+        actions: [get-secrets, build]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_64_builds.py
@@ -277,7 +277,7 @@ linux64-base-toolchains/debug:
             FORCE_GCC: '1'
     run:
         using: mozharness
-        actions: [get-secrets, build, check-test]
+        actions: [get-secrets, build]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_64_builds.py
@@ -313,7 +313,7 @@ linux64-base-toolchains-clang/opt:
             PERFHERDER_EXTRA_OPTIONS: base-toolchains-clang
     run:
         using: mozharness
-        actions: [get-secrets, build, check-test]
+        actions: [get-secrets, build]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_64_builds.py
@@ -346,7 +346,7 @@ linux64-base-toolchains-clang/debug:
             PERFHERDER_EXTRA_OPTIONS: base-toolchains-clang
     run:
         using: mozharness
-        actions: [get-secrets, build, check-test]
+        actions: [get-secrets, build]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_64_builds.py
@@ -381,7 +381,7 @@ linux/opt:
         max-run-time: 7200
     run:
         using: mozharness
-        actions: [get-secrets, build, check-test]
+        actions: [get-secrets, build]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_32_builds.py
@@ -417,7 +417,7 @@ linux/debug:
         max-run-time: 5400
     run:
         using: mozharness
-        actions: [get-secrets, build, check-test]
+        actions: [get-secrets, build]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_32_builds.py
@@ -537,7 +537,7 @@ linux-devedition-nightly/opt:
         max-run-time: 7200
     run:
         using: mozharness
-        actions: [get-secrets, build, check-test]
+        actions: [get-secrets, build]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_32_builds.py
@@ -581,7 +581,7 @@ linux-shippable/opt:
         max-run-time: 9000
     run:
         using: mozharness
-        actions: [get-secrets, build, check-test]
+        actions: [get-secrets, build]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_32_builds.py
@@ -616,7 +616,7 @@ linux64-asan/opt:
         max-run-time: 5400
     run:
         using: mozharness
-        actions: [get-secrets, build, check-test]
+        actions: [get-secrets, build]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_64_builds.py
@@ -653,7 +653,7 @@ linux64-asan-fuzzing/opt:
         max-run-time: 5400
     run:
         using: mozharness
-        actions: [get-secrets, build, check-test]
+        actions: [get-secrets, build]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_64_builds.py
@@ -691,7 +691,7 @@ linux64-asan-fuzzing-ccov/opt:
         max-run-time: 5400
     run:
         using: mozharness
-        actions: [get-secrets, build, check-test]
+        actions: [get-secrets, build]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_64_builds.py
@@ -729,7 +729,7 @@ linux64-fuzzing-ccov/opt:
             ASAN_OPTIONS: "detect_leaks=0"
     run:
         using: mozharness
-        actions: [get-secrets, build, check-test]
+        actions: [get-secrets, build]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_64_builds.py
@@ -773,7 +773,7 @@ linux64-asan-reporter-nightly/opt:
         max-run-time: 3600
     run:
         using: mozharness
-        actions: [get-secrets, build, check-test]
+        actions: [get-secrets, build]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_64_builds.py
@@ -812,7 +812,7 @@ linux64-asan/debug:
         max-run-time: 5400
     run:
         using: mozharness
-        actions: [get-secrets, build, check-test]
+        actions: [get-secrets, build]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_64_builds.py
@@ -853,7 +853,7 @@ linux64-shippable/opt:
         max-run-time: 9000
     run:
         using: mozharness
-        actions: [get-secrets, build, check-test]
+        actions: [get-secrets, build]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_64_builds.py
@@ -888,7 +888,7 @@ linux64-noopt/debug:
         max-run-time: 5400
     run:
         using: mozharness
-        actions: [get-secrets, build, check-test]
+        actions: [get-secrets, build]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_64_builds.py
@@ -1046,7 +1046,7 @@ linux64-ccov/debug:
             FORCE_GCC: '1'
     run:
         using: mozharness
-        actions: [get-secrets, build, check-test]
+        actions: [get-secrets, build]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_64_builds.py
@@ -1084,7 +1084,7 @@ linux64-ccov/opt:
             FORCE_GCC: '1'
     run:
         using: mozharness
-        actions: [get-secrets, build, check-test]
+        actions: [get-secrets, build]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_64_builds.py
@@ -1118,7 +1118,7 @@ linux64-add-on-devel/opt:
         max-run-time: 3600
     run:
         using: mozharness
-        actions: [get-secrets, build, check-test]
+        actions: [get-secrets, build]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_64_builds.py
@@ -1197,7 +1197,7 @@ linux-gcp/debug:
             PERFHERDER_EXTRA_OPTIONS: gcp
     run:
         using: mozharness
-        actions: [get-secrets, build, check-test]
+        actions: [get-secrets, build]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_32_builds.py
@@ -1237,7 +1237,7 @@ linux-gcp/opt:
             PERFHERDER_EXTRA_OPTIONS: gcp
     run:
         using: mozharness
-        actions: [get-secrets, build, check-test]
+        actions: [get-secrets, build]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_32_builds.py
@@ -1275,7 +1275,7 @@ linux64-gcp/debug:
             PERFHERDER_EXTRA_OPTIONS: gcp
     run:
         using: mozharness
-        actions: [get-secrets, build, check-test]
+        actions: [get-secrets, build]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_64_builds.py
@@ -1314,7 +1314,7 @@ linux64-gcp/opt:
             PERFHERDER_EXTRA_OPTIONS: gcp
     run:
         using: mozharness
-        actions: [get-secrets, build, check-test]
+        actions: [get-secrets, build]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_64_builds.py

taskcluster/ci/build/windows.yml

@@ -170,7 +170,7 @@ win64-plain/debug:
             PERFHERDER_EXTRA_OPTIONS: plain
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/releng.manifest"
     run:
-        actions: [build, check-test]
+        actions: [build]
         options: [append-env-variables-from-configs]
         script: mozharness/scripts/fx_desktop_build.py
         config:

testing/mozharness/configs/builds/releng_base_linux_32_builds.py

@@ -12,7 +12,6 @@ config = {
     'default_actions': [
         'clobber',
         'build',
-        'check-test',
     ],
     'app_ini_path': '%(obj_dir)s/dist/bin/application.ini',
     'secret_files': [

testing/mozharness/configs/builds/releng_base_linux_64_builds.py

@@ -11,7 +11,6 @@ config = {
     'default_actions': [
         'clobber',
         'build',
-        'check-test',
     ],
     'app_ini_path': '%(obj_dir)s/dist/bin/application.ini',
     'secret_files': [

testing/mozharness/configs/builds/releng_sub_linux_configs/64_asan.py

@@ -4,7 +4,6 @@ config = {
     'default_actions': [
         'clobber',
         'build',
-        'check-test',
     ],
     'stage_platform': 'linux64-asan',
     #### 64 bit build specific #####

testing/mozharness/configs/builds/releng_sub_linux_configs/64_asan_and_debug.py

@@ -4,7 +4,6 @@ config = {
     'default_actions': [
         'clobber',
         'build',
-        'check-test',
     ],
     'stage_platform': 'linux64-asan-debug',
     'debug_build': True,

testing/mozharness/configs/builds/releng_sub_linux_configs/64_valgrind.py

@@ -4,7 +4,6 @@ config = {
     'default_actions': [
         'clobber',
         'build',
-        'check-test',
         'valgrind-test',
     ],
     'stage_platform': 'linux64-valgrind',

testing/mozharness/configs/builds/releng_sub_mac_configs/64_debug.py

@@ -4,7 +4,6 @@ config = {
     'default_actions': [
         'clobber',
         'build',
-        'check-test',
     ],
     'stage_platform': 'macosx64-debug',
     'debug_build': True,

testing/mozharness/configs/builds/releng_sub_windows_configs/32_add-on-devel.py

@@ -4,7 +4,6 @@ config = {
     'default_actions': [
         'clobber',
         'build',
-        'check-test',
     ],
     'stage_platform': 'win32-add-on-devel',
     #### 32 bit build specific #####

testing/mozharness/configs/builds/releng_sub_windows_configs/32_debug.py

@@ -4,7 +4,6 @@ config = {
     'default_actions': [
         'clobber',
         'build',
-        'check-test',
     ],
     'stage_platform': 'win32-debug',
     'debug_build': True,

testing/mozharness/configs/builds/releng_sub_windows_configs/64_add-on-devel.py

@@ -4,7 +4,6 @@ config = {
     'default_actions': [
         'clobber',
         'build',
-        'check-test',
     ],
     'stage_platform': 'win64-add-on-devel',
     #### 64 bit build specific #####

testing/mozharness/configs/builds/releng_sub_windows_configs/64_debug.py

@@ -4,7 +4,6 @@ config = {
     'default_actions': [
         'clobber',
         'build',
-        'check-test',
     ],
     'stage_platform': 'win64-debug',
     'debug_build': True,

testing/mozharness/configs/builds/taskcluster_base_windows.py

@@ -4,7 +4,6 @@ config = {
     'default_actions': [
         'get-secrets',
         'build',
-        'check-test',
     ],
     'app_ini_path': '%(obj_dir)s/dist/bin/application.ini',
     'vcs_share_base': os.path.join('y:', os.sep, 'hg-shared'),

testing/mozharness/mozharness/mozilla/building/buildbase.py

@@ -40,8 +40,6 @@ from mozharness.mozilla.automation import (
     TBPL_WORST_LEVEL_TUPLE,
 )
 from mozharness.mozilla.secrets import SecretsMixin
-from mozharness.mozilla.testing.errors import TinderBoxPrintRe
-from mozharness.mozilla.testing.unittest import tbox_print_summary
 from mozharness.base.python import (
     PerfherderResourceOptionsMixin,
     VirtualenvMixin,
@@ -102,69 +100,6 @@ class MakeUploadOutputParser(OutputParser):
             self.info(line)
 
 
-class CheckTestCompleteParser(OutputParser):
-    tbpl_error_list = TBPL_UPLOAD_ERRORS
-
-    def __init__(self, **kwargs):
-        self.matches = {}
-        super(CheckTestCompleteParser, self).__init__(**kwargs)
-        self.pass_count = 0
-        self.fail_count = 0
-        self.leaked = False
-        self.harness_err_re = TinderBoxPrintRe['harness_error']['full_regex']
-        self.tbpl_status = TBPL_SUCCESS
-
-    def parse_single_line(self, line):
-        # Counts and flags.
-        # Regular expression for crash and leak detections.
-        if "TEST-PASS" in line:
-            self.pass_count += 1
-            return self.info(line)
-        if "TEST-UNEXPECTED-" in line:
-            # Set the error flags.
-            # Or set the failure count.
-            m = self.harness_err_re.match(line)
-            if m:
-                r = m.group(1)
-                if r == "missing output line for total leaks!":
-                    self.leaked = None
-                else:
-                    self.leaked = True
-            self.fail_count += 1
-            return self.warning(line)
-        self.info(line)  # else
-
-    def evaluate_parser(self, return_code,  success_codes=None):
-        success_codes = success_codes or [0]
-
-        if self.num_errors:  # ran into a script error
-            self.tbpl_status = self.worst_level(TBPL_FAILURE, self.tbpl_status,
-                                                levels=TBPL_WORST_LEVEL_TUPLE)
-
-        if self.fail_count > 0:
-            self.tbpl_status = self.worst_level(TBPL_WARNING, self.tbpl_status,
-                                                levels=TBPL_WORST_LEVEL_TUPLE)
-
-        # Account for the possibility that no test summary was output.
-        if (self.pass_count == 0 and self.fail_count == 0 and
-           os.environ.get('TRY_SELECTOR') != 'coverage'):
-            self.error('No tests run or test summary not found')
-            self.tbpl_status = self.worst_level(TBPL_WARNING, self.tbpl_status,
-                                                levels=TBPL_WORST_LEVEL_TUPLE)
-
-        if return_code not in success_codes:
-            self.tbpl_status = self.worst_level(TBPL_FAILURE, self.tbpl_status,
-                                                levels=TBPL_WORST_LEVEL_TUPLE)
-
-        # Print the summary.
-        summary = tbox_print_summary(self.pass_count,
-                                     self.fail_count,
-                                     self.leaked)
-        self.info("TinderboxPrint: check<br/>%s\n" % summary)
-
-        return self.tbpl_status
-
-
 class MozconfigPathError(Exception):
     """
     There was an error getting a mozconfig path from a mozharness config.
@@ -809,20 +744,6 @@ or run without that action (ie: --no-{action})"
                 return True
         return False
 
-    def query_check_test_env(self):
-        c = self.config
-        dirs = self.query_abs_dirs()
-        check_test_env = {}
-        if c.get('check_test_env'):
-            for env_var, env_value in c['check_test_env'].iteritems():
-                check_test_env[env_var] = env_value % dirs
-        # Check tests don't upload anything, however our mozconfigs depend on
-        # UPLOAD_PATH, so we prevent configure from re-running by keeping the
-        # environments consistent.
-        if c.get('upload_env'):
-            check_test_env.update(c['upload_env'])
-        return check_test_env
-
     def _rm_old_package(self):
         """rm the old package."""
         c = self.config
@@ -1196,41 +1117,6 @@ or run without that action (ie: --no-{action})"
             env=env, output_timeout=60*45, halt_on_failure=True,
         )
 
-    def check_test(self):
-        if os.environ.get('USE_ARTIFACT'):
-            self.info('Skipping due to forced artifact build.')
-            return
-        c = self.config
-        dirs = self.query_abs_dirs()
-
-        env = self.query_build_env()
-        env.update(self.query_check_test_env())
-
-        cmd = self._query_mach() + [
-            '--log-no-times',
-            'build',
-            '-v',
-            '--keep-going',
-            'check',
-        ]
-
-        parser = CheckTestCompleteParser(config=c,
-                                         log_obj=self.log_obj)
-        return_code = self.run_command(command=cmd,
-                                       cwd=dirs['abs_src_dir'],
-                                       env=env,
-                                       output_parser=parser)
-        tbpl_status = parser.evaluate_parser(return_code)
-        return_code = EXIT_STATUS_DICT[tbpl_status]
-
-        if return_code:
-            self.return_code = self.worst_level(
-                return_code,  self.return_code,
-                AUTOMATION_EXIT_CODES[::-1]
-            )
-            self.error("'mach build check' did not run successfully. Please "
-                       "check log for errors.")
-
     def _is_configuration_shipped(self):
         """Determine if the current build configuration is shipped to users.
 

testing/mozharness/scripts/fx_desktop_build.py

@@ -35,7 +35,6 @@ class FxDesktopBuild(BuildScript, TryToolsMixin, object):
                 'clobber',
                 'build',
                 'static-analysis-autotest',
-                'check-test',
                 'valgrind-test',
                 'multi-l10n',
                 'package-source',