Bug 883954 - part 2 - add support for defining generating scripts for GENERATED_FILES; r=gps

Now that we have proper moz.build objects for GENERATED_FILES, we can
add 'script' flags and 'args' flags in moz.build for select
GENERATED_FILES.  We restrict 'args' to being filenames for ease of
implementing checks for file existence, and many (all?) of the examples
of file generation throughout the tree don't need arbitrary strings or
Python data.

python/mozbuild/mozbuild/action/file_generate.py

@@ -0,0 +1,50 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Given a Python script and arguments describing the output file, and
+# the arguments that can be used to generate the output file, call the
+# script's |main| method with appropriate arguments.
+
+from __future__ import print_function
+import argparse
+import imp
+import os
+import sys
+import traceback
+
+from mozbuild.util import FileAvoidWrite
+
+def main(argv):
+    parser = argparse.ArgumentParser('Generate a file from a Python script',
+                                     add_help=False)
+    parser.add_argument('python_script', metavar='python-script', type=str,
+                        help='The Python script to run')
+    parser.add_argument('output_file', metavar='output-file', type=str,
+                        help='The file to generate')
+    parser.add_argument('additional_arguments', metavar='arg', nargs='*',
+                        help="Additional arguments to the script's main() method")
+
+    args = parser.parse_args(argv)
+
+    script = args.python_script
+    with open(script, 'r') as fh:
+        module = imp.load_module('script', fh, script,
+                                 ('.py', 'r', imp.PY_SOURCE))
+    if not hasattr(module, 'main'):
+        print('Error: script "{0}" is missing a main method'.format(script),
+              file=sys.stderr)
+        return 1
+
+    ret = 1
+    try:
+        with FileAvoidWrite(args.output_file) as output:
+            ret = module.main(output, *args.additional_arguments)
+    except IOError as e:
+        print('Error opening file "{0}"'.format(e.filename), file=sys.stderr)
+        traceback.print_exc()
+        return 1
+    return ret
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

python/mozbuild/mozbuild/backend/recursivemake.py

@@ -406,7 +406,14 @@ class RecursiveMakeBackend(CommonBackend):
             self._process_exports(obj, obj.exports, backend_file)
 
         elif isinstance(obj, GeneratedFile):
-            backend_file.write('GENERATED_FILES += %s\n' % obj.filename)
+            backend_file.write('GENERATED_FILES += %s\n' % obj.output)
+            if obj.script:
+                backend_file.write("""{output}: {script}{inputs}
+\t$(call py_action,file_generate,{script} {output}{inputs})
+
+""".format(output=obj.output,
+           inputs=' ' + ' '.join(obj.inputs) if obj.inputs else '',
+           script=obj.script))
 
         elif isinstance(obj, TestHarnessFiles):
             self._process_test_harness_files(obj, backend_file)

python/mozbuild/mozbuild/frontend/context.py

@@ -444,12 +444,37 @@ VARIABLES = {
         and reduce the debug info size.
         """, None),
 
-    'GENERATED_FILES': (StrictOrderingOnAppendList, list,
+    'GENERATED_FILES': (StrictOrderingOnAppendListWithFlagsFactory({
+                'script': unicode,
+                'inputs': list }), list,
         """Generic generated files.
 
-        This variable contains a list of generate files for the build system
-        to generate at export time. The rules for those files still live in
-        Makefile.in.
+        This variable contains a list of files for the build system to
+        generate at export time. The generation method may be declared
+        with optional ``script`` and ``inputs`` flags on individual entries.
+        If the optional ``script`` flag is not present on an entry, it
+        is assumed that rules for generating the file are present in
+        the associated Makefile.in.
+
+        Example::
+
+           GENERATED_FILES += ['bar.c', 'baz.c', 'foo.c']
+           bar = GENERATED_FILES['bar.c']
+           bar.script = 'generate.py'
+           bar.inputs = ['datafile-for-bar']
+           foo = GENERATED_FILES['foo.c']
+           foo.script = 'generate.py'
+           foo.inputs = ['datafile-for-foo']
+
+        This definition will generate bar.c by calling the main method of
+        generate.py with a open (for writing) file object for bar.c, and
+        the string ``datafile-for-bar``. In a similar fashion, the main
+        method of generate.py will also be called with an open
+        (for writing) file object for foo.c and the string
+        ``datafile-for-foo``. Please note that only string arguments are
+        supported for passing to scripts, and that all arguments provided
+        to the script should be filenames relative to the directory in which
+        the moz.build file is located.
         """, 'export'),
 
     'DEFINES': (OrderedDict, dict,

python/mozbuild/mozbuild/frontend/data.py

@@ -857,12 +857,16 @@ class GeneratedFile(ContextDerived):
     """Represents a generated file."""
 
     __slots__ = (
-        'filename',
+        'script',
+        'output',
+        'inputs',
     )
 
-    def __init__(self, context, filename):
+    def __init__(self, context, script, output, inputs):
         ContextDerived.__init__(self, context)
-        self.filename = filename
+        self.script = script
+        self.output = output
+        self.inputs = inputs
 
 
 class ClassPathEntry(object):

python/mozbuild/mozbuild/frontend/emitter.py

@@ -521,7 +521,29 @@ class TreeMetadataEmitter(LoggingMixin):
         generated_files = context.get('GENERATED_FILES')
         if generated_files:
             for f in generated_files:
-                yield GeneratedFile(context, f)
+                flags = generated_files[f]
+                output = f
+                if flags.script:
+                    script = mozpath.join(context.srcdir, flags.script)
+                    inputs = [mozpath.join(context.srcdir, i) for i in flags.inputs]
+
+                    if not os.path.exists(script):
+                        raise SandboxValidationError(
+                            'Script for generating %s does not exist: %s'
+                            % (f, script), context)
+                    if os.path.splitext(script)[1] != '.py':
+                        raise SandboxValidationError(
+                            'Script for generating %s does not end in .py: %s'
+                            % (f, script), context)
+                    for i in inputs:
+                        if not os.path.exists(i):
+                            raise SandboxValidationError(
+                                'Input for generating %s does not exist: %s'
+                                % (f, i), context)
+                else:
+                    script = None
+                    inputs = []
+                yield GeneratedFile(context, script, output, inputs)
 
         test_harness_files = context.get('TEST_HARNESS_FILES')
         if test_harness_files:

python/mozbuild/mozbuild/test/backend/data/generated-files/moz.build

@@ -2,4 +2,11 @@
 # Any copyright is dedicated to the Public Domain.
 # http://creativecommons.org/publicdomain/zero/1.0/
 
-GENERATED_FILES += [ 'bar.c', 'foo.c' ]
+GENERATED_FILES += [ 'bar.c', 'foo.c', 'quux.c' ]
+
+bar = GENERATED_FILES['bar.c']
+bar.script = 'generate-bar.py'
+
+foo = GENERATED_FILES['foo.c']
+foo.script = 'generate-foo.py'
+foo.inputs = ['foo-data']

python/mozbuild/mozbuild/test/backend/test_recursivemake.py

@@ -378,9 +378,17 @@ class TestRecursiveMakeBackend(BackendTester):
 
         expected = [
             'GENERATED_FILES += bar.c',
+            'bar.c: %s/generate-bar.py' % env.topsrcdir,
+            '$(call py_action,file_generate,%s/generate-bar.py bar.c)' % env.topsrcdir,
+            '',
             'GENERATED_FILES += foo.c',
+            'foo.c: %s/generate-foo.py %s/foo-data' % (env.topsrcdir, env.topsrcdir),
+            '$(call py_action,file_generate,%s/generate-foo.py foo.c %s/foo-data)' % (env.topsrcdir, env.topsrcdir),
+            '',
+            'GENERATED_FILES += quux.c',
         ]
 
+        self.maxDiff = None
         self.assertEqual(lines, expected)
 
     def test_resources(self):

python/mozbuild/mozbuild/test/frontend/data/generated-files-no-inputs/moz.build

@@ -0,0 +1,9 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+GENERATED_FILES += ['bar.c', 'foo.c']
+
+foo = GENERATED_FILES['foo.c']
+foo.script = 'script.py'
+foo.inputs = ['datafile']

python/mozbuild/mozbuild/test/frontend/data/generated-files-no-python-script/moz.build

@@ -0,0 +1,8 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+GENERATED_FILES += ['bar.c', 'foo.c']
+
+bar = GENERATED_FILES['bar.c']
+bar.script = 'script.rb'

python/mozbuild/mozbuild/test/frontend/data/generated-files-no-script/moz.build

@@ -0,0 +1,8 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+GENERATED_FILES += [ 'bar.c', 'foo.c' ]
+
+bar = GENERATED_FILES['bar.c']
+bar.script = 'nonexistent-script.py'

python/mozbuild/mozbuild/test/frontend/test_emitter.py

@@ -193,7 +193,25 @@ class TestEmitterBasic(unittest.TestCase):
 
         expected = ['bar.c', 'foo.c']
         for o, expected_filename in zip(objs, expected):
-            self.assertEqual(o.filename, expected_filename)
+            self.assertEqual(o.output, expected_filename)
+
+    def test_generated_files_no_script(self):
+        reader = self.reader('generated-files-no-script')
+        with self.assertRaisesRegexp(SandboxValidationError,
+            'Script for generating bar.c does not exist'):
+            objs = self.read_topsrcdir(reader)
+
+    def test_generated_files_no_inputs(self):
+        reader = self.reader('generated-files-no-inputs')
+        with self.assertRaisesRegexp(SandboxValidationError,
+            'Input for generating foo.c does not exist'):
+            objs = self.read_topsrcdir(reader)
+
+    def test_generated_files_no_python_script(self):
+        reader = self.reader('generated-files-no-python-script')
+        with self.assertRaisesRegexp(SandboxValidationError,
+            'Script for generating bar.c does not end in .py'):
+            objs = self.read_topsrcdir(reader)
 
     def test_exports(self):
         reader = self.reader('exports')