Bug 1618216 - When doing test packaging test for the nightly attribute as well, and validate against it. r=mshal

Future work should hopefully find a way to not assume tests are packaged with beetmover, and to have a solution for w64's emefree design.

Differential Revision: https://phabricator.services.mozilla.com/D64429

--HG--
extra : moz-landing-system : lando

taskcluster/ci/build/android.yml

@@ -112,6 +112,8 @@ android-x86-nightly/opt:
     attributes:
         enable-full-crashsymbols: true
         nightly: true
+        # Android nightlies don't actually need tests
+        skip-verify-test-packaging: true
     shipping-phase: build
     shipping-product: fennec
     index:
@@ -186,6 +188,8 @@ android-api-16-nightly/opt:
     attributes:
         enable-full-crashsymbols: true
         nightly: true
+        # Android nightlies don't actually need tests
+        skip-verify-test-packaging: true
     shipping-phase: build
     shipping-product: fennec
     index:
@@ -283,6 +287,8 @@ android-aarch64-nightly/opt:
     attributes:
         enable-full-crashsymbols: true
         nightly: true
+        # Android nightlies don't actually need tests
+        skip-verify-test-packaging: true
     shipping-phase: build
     shipping-product: fennec
     index:
@@ -333,6 +339,8 @@ android-x86_64-nightly/opt:
     attributes:
         enable-full-crashsymbols: true
         nightly: true
+        # Android nightlies don't actually need tests
+        skip-verify-test-packaging: true
     shipping-phase: build
     shipping-product: fennec
     index:

taskcluster/ci/build/linux.yml

@@ -577,6 +577,8 @@ linux-devedition/opt:
     worker:
         docker-image: {in-tree: debian7-i386-build}
         max-run-time: 7200
+        env:
+            MOZ_AUTOMATION_PACKAGE_TESTS: "1"
     run:
         using: mozharness
         actions: [get-secrets, build]
@@ -950,6 +952,7 @@ linux64-asan-reporter-nightly/opt:
         env:
             PERFHERDER_EXTRA_OPTIONS: asan-reporter
             ASAN_OPTIONS: "detect_leaks=0"
+            MOZ_AUTOMATION_PACKAGE_TESTS: "1"
         max-run-time: 3600
     run:
         using: mozharness

taskcluster/ci/build/windows.yml

@@ -799,6 +799,7 @@ win64-asan-reporter-nightly/opt:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/releng.manifest"
             PERFHERDER_EXTRA_OPTIONS: "asan-reporter"
+            MOZ_AUTOMATION_PACKAGE_TESTS: "1"
     run:
         options: [append-env-variables-from-configs]
         script: mozharness/scripts/fx_desktop_build.py
@@ -1243,6 +1244,7 @@ win64-aarch64-devedition/opt:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/aarch64.manifest"
             PERFHERDER_EXTRA_OPTIONS: aarch64-devedition
+            MOZ_AUTOMATION_PACKAGE_TESTS: "1"
     run:
         actions: [get-secrets, build]
         options: [append-env-variables-from-configs]

taskcluster/taskgraph/util/verify.py

@@ -266,35 +266,45 @@ def verify_nightly_no_sccache(task, taskgraph, scratch_pad, graph_config):
 @verifications.add('full_task_graph')
 def verify_test_packaging(task, taskgraph, scratch_pad, graph_config):
     if task is None:
+        exceptions = []
         for task in taskgraph.tasks.itervalues():
             if task.kind == 'build' and not task.attributes.get('skip-verify-test-packaging'):
                 build_env = task.task.get('payload', {}).get('env', {})
                 package_tests = build_env.get('MOZ_AUTOMATION_PACKAGE_TESTS')
                 shippable = task.attributes.get('shippable', False)
+                nightly = task.attributes.get('nightly', False)
                 build_has_tests = scratch_pad.get(task.label)
 
                 if package_tests != '1':
                     # Shippable builds should always package tests.
                     if shippable:
-                        raise Exception('Build job {} is shippable and does not specify '
-                                        'MOZ_AUTOMATION_PACKAGE_TESTS=1 in the '
-                                        'environment.'.format(task.label))
+                        exceptions.append('Build job {} is shippable and does not specify '
+                                          'MOZ_AUTOMATION_PACKAGE_TESTS=1 in the '
+                                          'environment.'.format(task.label))
+                    if nightly:
+                        exceptions.append('Build job {} is nightly and does not specify '
+                                          'MOZ_AUTOMATION_PACKAGE_TESTS=1 in the '
+                                          'environment.'.format(task.label))
 
                     # Build tasks in the scratch pad have tests dependent on
                     # them, so we need to package tests during build.
                     if build_has_tests:
-                        raise Exception(
+                        exceptions.append(
                             'Build job {} has tests dependent on it and does not specify '
                             'MOZ_AUTOMATION_PACKAGE_TESTS=1 in the environment'.format(task.label))
                 else:
                     # Build tasks that aren't in the scratch pad have no
                     # dependent tests, so we shouldn't package tests.
-                    if not build_has_tests:
-                        raise Exception(
+                    # With the caveat that we expect shippable and nightly jobs to always
+                    # produce tests.
+                    if not build_has_tests and not any([shippable, nightly]):
+                        exceptions.append(
                             'Build job {} has no tests, but specifies '
                             'MOZ_AUTOMATION_PACKAGE_TESTS={} in the environment. '
                             'Unset MOZ_AUTOMATION_PACKAGE_TESTS in the task definition '
                             'to fix.'.format(task.label, package_tests))
+        if exceptions:
+            raise Exception("\n".join(exceptions))
         return
     if task.kind == 'test':
         build_task = taskgraph[task.dependencies['build']]