Bug 1383880: add support for SCHEDULES in moz.build; r=gps

MozReview-Commit-ID: 2pfLr0VTy2J

--HG--
extra : rebase_source : dab39fe2634ac3899ef235f6e237d918af5ddccd
extra : histedit_source : 3503bc9f35cf9034820568fb68a65a88e9fe1234

python/mozbuild/mozbuild/frontend/context.py

@@ -24,6 +24,7 @@ from collections import (
 )
 from mozbuild.util import (
     HierarchicalStringList,
+    ImmutableStrictOrderingOnAppendList,
     KeyedDefaultDict,
     List,
     ListWithAction,
@@ -37,6 +38,8 @@ from mozbuild.util import (
     TypedNamedTuple,
 )
 
+from .. import schedules
+
 from ..testing import (
     all_test_flavors,
     read_manifestparser_manifest,
@@ -559,6 +562,54 @@ def ContextDerivedTypedRecord(*fields):
     return _TypedRecord
 
 
+class Schedules(object):
+    """Similar to a ContextDerivedTypedRecord, but with different behavior
+    for the properties:
+
+     * VAR.inclusive can only be appended to (+=), and can only contain values
+       from mozbuild.schedules.INCLUSIVE_COMPONENTS
+
+     * VAR.exclusive can only be assigned to (no +=), and can only contain
+       values from mozbuild.schedules.ALL_COMPONENTS
+    """
+    __slots__ = ('_exclusive', '_inclusive')
+
+    def __init__(self):
+        self._inclusive = TypedList(Enum(*schedules.INCLUSIVE_COMPONENTS))()
+        self._exclusive = ImmutableStrictOrderingOnAppendList(schedules.EXCLUSIVE_COMPONENTS)
+
+    # inclusive is mutable cannot be assigned to (+= only)
+    @property
+    def inclusive(self):
+        return self._inclusive
+
+    @inclusive.setter
+    def inclusive(self, value):
+        if value is not self._inclusive:
+            raise AttributeError("Cannot assign to this value - use += instead")
+        unexpected = [v for v in value if v not in schedules.INCLUSIVE_COMPONENTS]
+        if unexpected:
+            raise Exception("unexpected exclusive component(s) " + ', '.join(unexpected))
+
+    # exclusive is immuntable but can be set (= only)
+    @property
+    def exclusive(self):
+        return self._exclusive
+
+    @exclusive.setter
+    def exclusive(self, value):
+        if not isinstance(value, (tuple, list)):
+            raise Exception("expected a tuple or list")
+        unexpected = [v for v in value if v not in schedules.ALL_COMPONENTS]
+        if unexpected:
+            raise Exception("unexpected exclusive component(s) " + ', '.join(unexpected))
+        self._exclusive = ImmutableStrictOrderingOnAppendList(sorted(value))
+
+    # components provides a synthetic summary of all components
+    @property
+    def components(self):
+        return list(sorted(set(self._inclusive) | set(self._exclusive)))
+
 @memoize
 def ContextDerivedTypedHierarchicalStringList(type):
     """Specialized HierarchicalStringList for use with ContextDerivedValue
@@ -629,6 +680,9 @@ DependentTestsEntry = ContextDerivedTypedRecord(('files', OrderedSourceList),
                                                 ('flavors', OrderedTestFlavorList))
 BugzillaComponent = TypedNamedTuple('BugzillaComponent',
                         [('product', unicode), ('component', unicode)])
+SchedulingComponents = ContextDerivedTypedRecord(
+        ('inclusive', TypedList(unicode, StrictOrderingOnAppendList)),
+        ('exclusive', TypedList(unicode, StrictOrderingOnAppendList)))
 
 
 class Files(SubContext):
@@ -747,6 +801,35 @@ class Files(SubContext):
             Would suggest that nsGlobalWindow.cpp is potentially relevant to
             any plain mochitest.
             """),
+        'SCHEDULES': (Schedules, list,
+            """Maps source files to the CI tasks that should be scheduled when
+            they change.  The tasks are grouped by named components, and those
+            names appear again in the taskgraph configuration
+            `($topsrcdir/taskgraph/).
+
+            Some components are "inclusive", meaning that changes to most files
+            do not schedule them, aside from those described in a Files
+            subcontext.  For example, py-lint tasks need not be scheduled for
+            most changes, but should be scheduled when any Python file changes.
+            Such components are named by appending to `SCHEDULES.inclusive`:
+
+            with Files('**.py'):
+                SCHEDULES.inclusive += ['py-lint']
+
+            Other components are 'exclusive', meaning that changes to most
+            files schedule them, but some files affect only one or two
+            components. For example, most files schedule builds and tests of
+            Firefox for Android, OS X, Windows, and Linux, but files under
+            `mobile/android/` affect Android builds and tests exclusively, so
+            builds for other operating systems are not needed.  Test suites
+            provide another example: most files schedule reftests, but changes
+            to reftest scripts need only schedule reftests and no other suites.
+
+            Exclusive components are named by setting `SCHEDULES.exclusive`:
+
+            with Files('mobile/android/**'):
+                SCHEDULES.exclusive = ['android']
+            """),
     }
 
     def __init__(self, parent, pattern=None):

python/mozbuild/mozbuild/frontend/mach_commands.py

@@ -17,6 +17,7 @@ from mach.decorators import (
 from mozbuild.base import MachCommandBase
 import mozpack.path as mozpath
 
+TOPSRCDIR = os.path.abspath(os.path.join(__file__, '../../../../../'))
 
 class InvalidPathException(Exception):
     """Represents an error due to an invalid path."""
@@ -221,3 +222,23 @@ class MozbuildFileCommands(MachCommandBase):
 
         reader = self._get_reader(finder=reader_finder)
         return reader.files_info(allpaths)
+
+
+    @SubCommand('file-info', 'schedules',
+                'Show the combined SCHEDULES for the files listed.')
+    @CommandArgument('paths', nargs='+',
+                     help='Paths whose data to query')
+    def file_info_schedules(self, paths):
+        """Show what is scheduled by the given files.
+
+        Given a requested set of files (which can be specified using
+        wildcards), print the total set of scheduled components.
+        """
+        from mozbuild.frontend.reader import EmptyConfig, BuildReader
+        config = EmptyConfig(TOPSRCDIR)
+        reader = BuildReader(config)
+        schedules = set()
+        for p, m in reader.files_info(paths).items():
+            schedules |= set(m['SCHEDULES'].components)
+
+        print(", ".join(schedules))

python/mozbuild/mozbuild/schedules.py

@@ -0,0 +1,26 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Constants for SCHEDULES configuration in moz.build files and for
+skip-unless-schedules optimizations in task-graph generation.
+"""
+
+from __future__ import absolute_import, unicode_literals, print_function
+
+# TODO: ideally these lists could be specified in moz.build itself
+
+INCLUSIVE_COMPONENTS = [
+    'py-lint',
+    'js-lint',
+    'yaml-lint',
+]
+EXCLUSIVE_COMPONENTS = [
+    # os families
+    'android',
+    'linux',
+    'macosx',
+    'windows',
+]
+ALL_COMPONENTS = INCLUSIVE_COMPONENTS + EXCLUSIVE_COMPONENTS

python/mozbuild/mozbuild/test/frontend/data/schedules/moz.build

@@ -0,0 +1,11 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+with Files('*.win'):
+    SCHEDULES.exclusive = ['windows']
+
+with Files('*.osx'):
+    SCHEDULES.exclusive = ['macosx']
+
+with Files('subd/**.py'):
+    SCHEDULES.inclusive += ['py-lint']

python/mozbuild/mozbuild/test/frontend/data/schedules/subd/moz.build

@@ -0,0 +1,2 @@
+with Files('yaml.py'):
+    SCHEDULES.inclusive += ['yaml-lint']

python/mozbuild/mozbuild/test/frontend/test_reader.py

@@ -480,6 +480,27 @@ class TestBuildReader(unittest.TestCase):
         with self.assertRaises(BuildReaderError):
             reader.files_info(['foo.js'])
 
+    def test_schedules(self):
+        reader = self.reader('schedules')
+        info = reader.files_info(['somefile', 'foo.win', 'foo.osx', 'subd/aa.py', 'subd/yaml.py'])
+        # default: all exclusive, no inclusive
+        self.assertEqual(info['somefile']['SCHEDULES'].inclusive, [])
+        self.assertEqual(info['somefile']['SCHEDULES'].exclusive, ['android', 'linux', 'macosx', 'windows'])
+        # windows-only
+        self.assertEqual(info['foo.win']['SCHEDULES'].inclusive, [])
+        self.assertEqual(info['foo.win']['SCHEDULES'].exclusive, ['windows'])
+        # osx-only
+        self.assertEqual(info['foo.osx']['SCHEDULES'].inclusive, [])
+        self.assertEqual(info['foo.osx']['SCHEDULES'].exclusive, ['macosx'])
+        # top-level moz.build specifies subd/**.py with an inclusive option
+        self.assertEqual(info['subd/aa.py']['SCHEDULES'].inclusive, ['py-lint'])
+        self.assertEqual(info['subd/aa.py']['SCHEDULES'].exclusive, ['android', 'linux', 'macosx', 'windows'])
+        # Files('yaml.py') in subd/moz.build *overrides* Files('subdir/**.py')
+        self.assertEqual(info['subd/yaml.py']['SCHEDULES'].inclusive, ['yaml-lint'])
+        self.assertEqual(info['subd/yaml.py']['SCHEDULES'].exclusive, ['android', 'linux', 'macosx', 'windows'])
+
+        self.assertEqual(info['subd/yaml.py']['SCHEDULES'].components,
+                ['android', 'linux', 'macosx', 'windows', 'yaml-lint'])
 
 if __name__ == '__main__':
     main()

python/mozbuild/mozbuild/util.py

@@ -485,6 +485,25 @@ class StrictOrderingOnAppendList(ListMixin, StrictOrderingOnAppendListMixin,
     elements be ordered. This enforces cleaner style in moz.build files.
     """
 
+class ImmutableStrictOrderingOnAppendList(StrictOrderingOnAppendList):
+    """Like StrictOrderingOnAppendList, but not allowing mutations of the value.
+    """
+    def append(self, elt):
+        raise Exception("cannot use append on this type")
+
+    def extend(self, iterable):
+        raise Exception("cannot use extend on this type")
+
+    def __setslice__(self, i, j, iterable):
+        raise Exception("cannot assign to slices on this type")
+
+    def __setitem__(self, i, elt):
+        raise Exception("cannot assign to indexes on this type")
+
+    def __iadd__(self, other):
+        raise Exception("cannot use += on this type")
+
+
 class ListWithActionMixin(object):
     """Mixin to create lists with pre-processing. See ListWithAction."""
     def __init__(self, iterable=None, action=None):