From 93c505b07a7736cc9bf7d4fea4195d23c6c040c4 Mon Sep 17 00:00:00 2001 From: Nick Alexander Date: Thu, 8 Mar 2018 16:04:06 -0800 Subject: [PATCH] Bug 1443208 - Pre: Add force flag to GENERATED_FILES. r=ted.mielczarek The forces on the system are such that I really need to be able to FORCE a few RecursiveMake targets in order to make Android and Gradle use GENERATED_FILES and LOCALIZED_GENERATED_FILES. Over time, I hope to avoid FORCE, but that time is not now. MozReview-Commit-ID: 453FpnihSRK --HG-- rename : python/mozbuild/mozbuild/test/backend/data/generated-files/foo-data => python/mozbuild/mozbuild/test/backend/data/generated-files-force/foo-data rename : python/mozbuild/mozbuild/test/backend/data/generated-files/generate-bar.py => python/mozbuild/mozbuild/test/backend/data/generated-files-force/generate-bar.py rename : python/mozbuild/mozbuild/test/backend/data/generated-files/generate-foo.py => python/mozbuild/mozbuild/test/backend/data/generated-files-force/generate-foo.py rename : python/mozbuild/mozbuild/test/backend/data/generated-files/moz.build => python/mozbuild/mozbuild/test/backend/data/generated-files-force/moz.build rename : python/mozbuild/mozbuild/test/backend/data/localized-generated-files/en-US/localized-input => python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/en-US/localized-input rename : python/mozbuild/mozbuild/test/backend/data/localized-generated-files/foo-data => python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/foo-data rename : python/mozbuild/mozbuild/test/backend/data/localized-generated-files/generate-foo.py => python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/generate-foo.py rename : python/mozbuild/mozbuild/test/backend/data/localized-generated-files/moz.build => python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/moz.build rename : python/mozbuild/mozbuild/test/backend/data/localized-generated-files/non-localized-input => python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/non-localized-input rename : python/mozbuild/mozbuild/test/frontend/data/generated-files/moz.build => python/mozbuild/mozbuild/test/frontend/data/generated-files-force/moz.build rename : python/mozbuild/mozbuild/test/frontend/data/localized-generated-files/moz.build => python/mozbuild/mozbuild/test/frontend/data/localized-generated-files-force/moz.build extra : rebase_source : 6820ac4b377ac596cafbdf065112738376cc473e --- .../mozbuild/backend/recursivemake.py | 11 +++- python/mozbuild/mozbuild/frontend/context.py | 10 +++- python/mozbuild/mozbuild/frontend/data.py | 5 +- python/mozbuild/mozbuild/frontend/emitter.py | 2 +- .../data/generated-files-force/foo-data | 0 .../generated-files-force/generate-bar.py | 0 .../generated-files-force/generate-foo.py | 0 .../data/generated-files-force/moz.build | 14 +++++ .../en-US/localized-input | 0 .../localized-generated-files-force/foo-data | 0 .../generate-foo.py | 0 .../localized-generated-files-force/moz.build | 22 +++++++ .../non-localized-input | 0 .../test/backend/test_recursivemake.py | 57 +++++++++++++++++++ .../data/generated-files-force/moz.build | 7 +++ .../localized-generated-files-force/moz.build | 6 ++ .../mozbuild/test/frontend/test_emitter.py | 21 ++++++- 17 files changed, 148 insertions(+), 7 deletions(-) create mode 100644 python/mozbuild/mozbuild/test/backend/data/generated-files-force/foo-data create mode 100644 python/mozbuild/mozbuild/test/backend/data/generated-files-force/generate-bar.py create mode 100644 python/mozbuild/mozbuild/test/backend/data/generated-files-force/generate-foo.py create mode 100644 python/mozbuild/mozbuild/test/backend/data/generated-files-force/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/en-US/localized-input create mode 100644 python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/foo-data create mode 100644 python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/generate-foo.py create mode 100644 python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/non-localized-input create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-files-force/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/localized-generated-files-force/moz.build diff --git a/python/mozbuild/mozbuild/backend/recursivemake.py b/python/mozbuild/mozbuild/backend/recursivemake.py index ae91cff202ab..70d9e5cd22b8 100644 --- a/python/mozbuild/mozbuild/backend/recursivemake.py +++ b/python/mozbuild/mozbuild/backend/recursivemake.py @@ -578,8 +578,15 @@ class RecursiveMakeBackend(CommonBackend): backend_file.write('%s: %s ;\n' % (output, first_output)) backend_file.write('GARBAGE += %s\n' % output) backend_file.write('EXTRA_MDDEPEND_FILES += %s\n' % dep_file) + + force = '' + if obj.force: + force = ' FORCE' + elif obj.localized: + force = ' $(if $(IS_LANGUAGE_REPACK),FORCE)' + if obj.script: - backend_file.write("""{output}: {script}{inputs}{backend}{repack_force} + backend_file.write("""{output}: {script}{inputs}{backend}{force} \t$(REPORT_BUILD) \t$(call py_action,file_generate,{locale}{script} {method} {output} $(MDDEPDIR)/{dep_file}{inputs}{flags}) @@ -592,7 +599,7 @@ class RecursiveMakeBackend(CommonBackend): # so standard mtime dependencies won't work properly when the build is re-run # with a different locale as input. IS_LANGUAGE_REPACK will reliably be set # in this situation, so simply force the generation to run in that case. - repack_force=' $(if $(IS_LANGUAGE_REPACK),FORCE)' if obj.localized else '', + force=force, locale='--locale=$(AB_CD) ' if obj.localized else '', script=obj.script, method=obj.method)) diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozbuild/frontend/context.py index 95798f2f003f..fd5ca8aef2f1 100644 --- a/python/mozbuild/mozbuild/frontend/context.py +++ b/python/mozbuild/mozbuild/frontend/context.py @@ -931,6 +931,7 @@ SchedulingComponents = ContextDerivedTypedRecord( GeneratedFilesList = StrictOrderingOnAppendListWithFlagsFactory({ 'script': unicode, 'inputs': list, + 'force': bool, 'flags': list, }) @@ -1276,8 +1277,8 @@ VARIABLES = { This variable contains a list of files for the build system to generate at export time. The generation method may be declared - with optional ``script``, ``inputs`` and ``flags`` attributes on - individual entries. + with optional ``script``, ``inputs``, ``flags``, and ``force`` + attributes on individual entries. If the optional ``script`` attribute is not present on an entry, it is assumed that rules for generating the file are present in the associated Makefile.in. @@ -1315,6 +1316,11 @@ VARIABLES = { When the ``flags`` attribute is present, the given list of flags is passed as extra arguments following the inputs. + + When the ``force`` attribute is present, the file is generated every + build, regardless of whether it is stale. This is special to the + RecursiveMake backend and intended for special situations only (e.g., + localization). Please consult a build peer before using ``force``. """), 'DEFINES': (InitializedDefines, dict, diff --git a/python/mozbuild/mozbuild/frontend/data.py b/python/mozbuild/mozbuild/frontend/data.py index 1cbd32271e5b..81a3e37ddfca 100644 --- a/python/mozbuild/mozbuild/frontend/data.py +++ b/python/mozbuild/mozbuild/frontend/data.py @@ -1122,9 +1122,11 @@ class GeneratedFile(ContextDerived): 'flags', 'required_for_compile', 'localized', + 'force', ) - def __init__(self, context, script, method, outputs, inputs, flags=(), localized=False): + def __init__(self, context, script, method, outputs, inputs, + flags=(), localized=False, force=False): ContextDerived.__init__(self, context) self.script = script self.method = method @@ -1132,6 +1134,7 @@ class GeneratedFile(ContextDerived): self.inputs = inputs self.flags = flags self.localized = localized + self.force = force suffixes = ( '.c', diff --git a/python/mozbuild/mozbuild/frontend/emitter.py b/python/mozbuild/mozbuild/frontend/emitter.py index 83ad431ae6cd..4b72b6335ea8 100644 --- a/python/mozbuild/mozbuild/frontend/emitter.py +++ b/python/mozbuild/mozbuild/frontend/emitter.py @@ -1391,7 +1391,7 @@ class TreeMetadataEmitter(LoggingMixin): inputs.append(p) yield GeneratedFile(context, script, method, outputs, inputs, - flags.flags, localized=localized) + flags.flags, localized=localized, force=flags.force) def _process_test_manifests(self, context): for prefix, info in TEST_MANIFESTS.items(): diff --git a/python/mozbuild/mozbuild/test/backend/data/generated-files-force/foo-data b/python/mozbuild/mozbuild/test/backend/data/generated-files-force/foo-data new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/test/backend/data/generated-files-force/generate-bar.py b/python/mozbuild/mozbuild/test/backend/data/generated-files-force/generate-bar.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/test/backend/data/generated-files-force/generate-foo.py b/python/mozbuild/mozbuild/test/backend/data/generated-files-force/generate-foo.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/test/backend/data/generated-files-force/moz.build b/python/mozbuild/mozbuild/test/backend/data/generated-files-force/moz.build new file mode 100644 index 000000000000..6af74db18d85 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/generated-files-force/moz.build @@ -0,0 +1,14 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +GENERATED_FILES += [ 'bar.c', 'foo.c', 'quux.c' ] + +bar = GENERATED_FILES['bar.c'] +bar.script = 'generate-bar.py:baz' +bar.force = True + +foo = GENERATED_FILES['foo.c'] +foo.script = 'generate-foo.py' +foo.inputs = ['foo-data'] +foo.force = False diff --git a/python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/en-US/localized-input b/python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/en-US/localized-input new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/foo-data b/python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/foo-data new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/generate-foo.py b/python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/generate-foo.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/moz.build b/python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/moz.build new file mode 100644 index 000000000000..7181809f4a35 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/moz.build @@ -0,0 +1,22 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +LOCALIZED_GENERATED_FILES += [ 'foo.xyz' ] + +foo = LOCALIZED_GENERATED_FILES['foo.xyz'] +foo.script = 'generate-foo.py' +foo.inputs = [ + 'en-US/localized-input', + 'non-localized-input', +] + +LOCALIZED_GENERATED_FILES += [ 'abc.xyz' ] + +abc = LOCALIZED_GENERATED_FILES['abc.xyz'] +abc.script = 'generate-foo.py' +abc.inputs = [ + 'en-US/localized-input', + 'non-localized-input', +] +abc.force = True diff --git a/python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/non-localized-input b/python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/non-localized-input new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py index 802f2a20e9c4..63ba4ca02555 100644 --- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py +++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py @@ -418,6 +418,36 @@ class TestRecursiveMakeBackend(BackendTester): self.maxDiff = None self.assertEqual(lines, expected) + def test_generated_files_force(self): + """Ensure GENERATED_FILES with .force is handled properly.""" + env = self._consume('generated-files-force', RecursiveMakeBackend) + + backend_path = mozpath.join(env.topobjdir, 'backend.mk') + lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]] + + expected = [ + 'export:: bar.c', + 'GARBAGE += bar.c', + 'EXTRA_MDDEPEND_FILES += bar.c.pp', + 'bar.c: %s/generate-bar.py FORCE' % env.topsrcdir, + '$(REPORT_BUILD)', + '$(call py_action,file_generate,%s/generate-bar.py baz bar.c $(MDDEPDIR)/bar.c.pp)' % env.topsrcdir, + '', + 'export:: foo.c', + 'GARBAGE += foo.c', + 'EXTRA_MDDEPEND_FILES += foo.c.pp', + 'foo.c: %s/generate-foo.py $(srcdir)/foo-data' % (env.topsrcdir), + '$(REPORT_BUILD)', + '$(call py_action,file_generate,%s/generate-foo.py main foo.c $(MDDEPDIR)/foo.c.pp $(srcdir)/foo-data)' % (env.topsrcdir), + '', + 'export:: quux.c', + 'GARBAGE += quux.c', + 'EXTRA_MDDEPEND_FILES += quux.c.pp', + ] + + self.maxDiff = None + self.assertEqual(lines, expected) + def test_localized_generated_files(self): """Ensure LOCALIZED_GENERATED_FILES is handled properly.""" env = self._consume('localized-generated-files', RecursiveMakeBackend) @@ -442,6 +472,33 @@ class TestRecursiveMakeBackend(BackendTester): self.maxDiff = None self.assertEqual(lines, expected) + def test_localized_generated_files_force(self): + """Ensure LOCALIZED_GENERATED_FILES with .force is handled properly.""" + env = self._consume('localized-generated-files-force', RecursiveMakeBackend) + + backend_path = mozpath.join(env.topobjdir, 'backend.mk') + lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]] + + expected = [ + 'libs:: foo.xyz', + 'GARBAGE += foo.xyz', + 'EXTRA_MDDEPEND_FILES += foo.xyz.pp', + 'foo.xyz: %s/generate-foo.py $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input $(if $(IS_LANGUAGE_REPACK),FORCE)' % env.topsrcdir, + '$(REPORT_BUILD)', + '$(call py_action,file_generate,--locale=$(AB_CD) %s/generate-foo.py main foo.xyz $(MDDEPDIR)/foo.xyz.pp $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input)' % env.topsrcdir, + '', + 'libs:: abc.xyz', + 'GARBAGE += abc.xyz', + 'EXTRA_MDDEPEND_FILES += abc.xyz.pp', + 'abc.xyz: %s/generate-foo.py $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input FORCE' % env.topsrcdir, + '$(REPORT_BUILD)', + '$(call py_action,file_generate,--locale=$(AB_CD) %s/generate-foo.py main abc.xyz $(MDDEPDIR)/abc.xyz.pp $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input)' % env.topsrcdir, + '', + ] + + self.maxDiff = None + self.assertEqual(lines, expected) + def test_localized_generated_files_AB_CD(self): """Ensure LOCALIZED_GENERATED_FILES is handled properly when {AB_CD} and {AB_rCD} are used.""" diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-files-force/moz.build b/python/mozbuild/mozbuild/test/frontend/data/generated-files-force/moz.build new file mode 100644 index 000000000000..781d0aafeddc --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/generated-files-force/moz.build @@ -0,0 +1,7 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +GENERATED_FILES += [ 'bar.c', 'foo.c', ('xpidllex.py', 'xpidlyacc.py'), ] +GENERATED_FILES['bar.c'].force = True +GENERATED_FILES['foo.c'].force = False diff --git a/python/mozbuild/mozbuild/test/frontend/data/localized-generated-files-force/moz.build b/python/mozbuild/mozbuild/test/frontend/data/localized-generated-files-force/moz.build new file mode 100644 index 000000000000..c3184c48c803 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/localized-generated-files-force/moz.build @@ -0,0 +1,6 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +LOCALIZED_GENERATED_FILES += [ 'abc.ini', ('bar', 'baz') ] +LOCALIZED_GENERATED_FILES['abc.ini'].force = True diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/mozbuild/mozbuild/test/frontend/test_emitter.py index 64d4ae4aece4..48913c2856b5 100644 --- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py +++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py @@ -452,7 +452,6 @@ class TestEmitterBasic(unittest.TestCase): 'AS_DASH_C_FLAG': ''}) self.maxDiff = maxDiff - def test_generated_files(self): reader = self.reader('generated-files') objs = self.read_topsrcdir(reader) @@ -461,6 +460,7 @@ class TestEmitterBasic(unittest.TestCase): for o in objs: self.assertIsInstance(o, GeneratedFile) self.assertFalse(o.localized) + self.assertFalse(o.force) expected = ['bar.c', 'foo.c', ('xpidllex.py', 'xpidlyacc.py'), ] for o, f in zip(objs, expected): @@ -470,6 +470,15 @@ class TestEmitterBasic(unittest.TestCase): self.assertEqual(o.method, None) self.assertEqual(o.inputs, []) + def test_generated_files_force(self): + reader = self.reader('generated-files-force') + objs = self.read_topsrcdir(reader) + + self.assertEqual(len(objs), 3) + for o in objs: + self.assertIsInstance(o, GeneratedFile) + self.assertEqual(o.force, 'bar.c' in o.outputs) + def test_localized_generated_files(self): reader = self.reader('localized-generated-files') objs = self.read_topsrcdir(reader) @@ -487,6 +496,16 @@ class TestEmitterBasic(unittest.TestCase): self.assertEqual(o.method, None) self.assertEqual(o.inputs, []) + def test_localized_generated_files_force(self): + reader = self.reader('localized-generated-files-force') + objs = self.read_topsrcdir(reader) + + self.assertEqual(len(objs), 2) + for o in objs: + self.assertIsInstance(o, GeneratedFile) + self.assertTrue(o.localized) + self.assertEqual(o.force, 'abc.ini' in o.outputs) + def test_localized_files_from_generated(self): """Test that using LOCALIZED_GENERATED_FILES and then putting the output in LOCALIZED_FILES as an objdir path works.