diff --git a/Makefile.in b/Makefile.in index 43f54f7ae3f2..8d037f3858dc 100644 --- a/Makefile.in +++ b/Makefile.in @@ -35,7 +35,7 @@ DIST_GARBAGE = config.cache config.log config.status* config-defs.h \ netwerk/necko-config.h xpcom/xpcom-config.h xpcom/xpcom-private.h \ $(topsrcdir)/.mozconfig.mk $(topsrcdir)/.mozconfig.out -default alldep all:: $(topsrcdir)/configure config.status +default alldep all:: CLOBBER $(topsrcdir)/configure config.status $(RM) -r $(DIST)/sdk $(RM) -r $(DIST)/include $(RM) -r $(DIST)/private @@ -43,6 +43,12 @@ default alldep all:: $(topsrcdir)/configure config.status $(RM) $(DIST)/bin/chrome.manifest $(DIST)/bin/components/components.manifest $(RM) -r _tests +CLOBBER: $(topsrcdir)/CLOBBER + @echo "STOP! The CLOBBER file has changed." + @echo "Please run the build through a sanctioned build wrapper, such as" + @echo "'mach build' or client.mk." + @exit 1 + $(topsrcdir)/configure: $(topsrcdir)/configure.in @echo "STOP! configure.in has changed, and your configure is out of date." @echo "Please rerun autoconf and re-configure your build directory." diff --git a/client.mk b/client.mk index bfa5151fb46c..de4de4c86ed6 100644 --- a/client.mk +++ b/client.mk @@ -108,6 +108,9 @@ endef # before evaluation. $(eval $(subst ||,$(CR),$(shell _PYMAKE=$(.PYMAKE) $(TOPSRCDIR)/$(MOZCONFIG_LOADER) $(TOPSRCDIR) 2> $(TOPSRCDIR)/.mozconfig.out | sed 's/$$/||/'))) +ifdef NO_AUTOCLOBBER +export NO_AUTOCLOBBER=1 +endif # Automatically add -jN to make flags if not defined. N defaults to number of cores. ifeq (,$(findstring -j,$(MOZ_MAKE_FLAGS))) @@ -302,9 +305,13 @@ else CONFIGURE = $(TOPSRCDIR)/configure endif +check-clobber: + $(PYTHON) $(TOPSRCDIR)/python/mozbuild/mozbuild/controller/clobber.py $(TOPSRCDIR) $(OBJDIR) + configure-files: $(CONFIGURES) configure-preqs = \ + check-clobber \ configure-files \ $(call mkdir_deps,$(OBJDIR)) \ $(if $(MOZ_BUILD_PROJECTS),$(call mkdir_deps,$(MOZ_OBJDIR))) \ @@ -442,4 +449,23 @@ echo-variable-%: # in parallel. .NOTPARALLEL: -.PHONY: checkout real_checkout depend realbuild build profiledbuild cleansrcdir pull_all build_all clobber clobber_all pull_and_build_all everything configure preflight_all preflight postflight postflight_all $(OBJDIR_TARGETS) +.PHONY: checkout \ + real_checkout \ + depend \ + realbuild \ + build \ + profiledbuild \ + cleansrcdir \ + pull_all \ + build_all \ + check-clobber \ + clobber \ + clobber_all \ + pull_and_build_all \ + everything \ + configure \ + preflight_all \ + preflight \ + postflight \ + postflight_all \ + $(OBJDIR_TARGETS) diff --git a/configure.in b/configure.in index 730a3c68d6bf..7564dbc16cb4 100644 --- a/configure.in +++ b/configure.in @@ -120,30 +120,6 @@ then fi MOZ_BUILD_ROOT=`pwd` -dnl Do not allow building if a clobber is required -dnl ============================================================== -dnl TODO Make this better, ideally this would clobber automaticially -if test -e $_objdir/CLOBBER; then - if test $_topsrcdir/CLOBBER -nt $_objdir/CLOBBER; then - echo " ***" - echo " * The CLOBBER file has been updated, indicating that an incremental build" - echo " * since your last build will probably not work. A full build is required." - echo " * The change that caused this is:" - cat $_topsrcdir/CLOBBER | sed '/^#/d' | sed 's/^/ * /' - echo " * " - echo " * The easiest way to fix this is to manually delete your objdir:" - echo " * rm -rf $_objdir" - echo " * " - echo " * Or, if you know this clobber doesn't apply to you, it can be ignored with:" - echo " * cp '$_topsrcdir/CLOBBER' $_objdir" - echo " ***" - exit 1 - break; - fi -else - touch $_objdir/CLOBBER -fi - MOZ_PYTHON MOZ_DEFAULT_COMPILER diff --git a/python/Makefile.in b/python/Makefile.in index daaf1b277049..29bcb7ad4e17 100644 --- a/python/Makefile.in +++ b/python/Makefile.in @@ -12,6 +12,7 @@ include $(DEPTH)/config/autoconf.mk test_dirs := \ mozbuild/mozbuild/test \ mozbuild/mozbuild/test/backend \ + mozbuild/mozbuild/test/controller \ mozbuild/mozbuild/test/compilation \ mozbuild/mozbuild/test/frontend \ mozbuild/mozpack/test \ diff --git a/python/mozbuild/mozbuild/controller/__init__.py b/python/mozbuild/mozbuild/controller/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/controller/clobber.py b/python/mozbuild/mozbuild/controller/clobber.py new file mode 100644 index 000000000000..f3b6897c22c7 --- /dev/null +++ b/python/mozbuild/mozbuild/controller/clobber.py @@ -0,0 +1,194 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import print_function + +r'''This module contains code for managing clobbering of the tree.''' + +import os +import sys + +try: + from mozfile.mozfile import rmtree +except ImportError: + from shutil import rmtree + + +CLOBBER_MESSAGE = ''' +*** +* The CLOBBER file has been updated, indicating that an incremental build since +* your last build will probably not work. A full/clobber build is required. +* +* The reason for the clobber is: +* +{clobber_reason} +* +* Clobbering can be performed automatically. However, we didn't automatically +* clobber this time because: +* +* {no_reason} +* +* The easiest and fastest way to clobber is to run: +* +* $ mach clobber +* +* If you know this clobber doesn't apply to you or you're feeling lucky - well +* do ya? - you can ignore this clobber requirement by running: +* +* $ touch {clobber_file} +* +*** +'''.strip() + + +class Clobberer(object): + def __init__(self, topsrcdir, topobjdir): + """Create a new object to manage clobbering the tree. + + It is bound to a top source directory and to a specific object + directory. + """ + assert os.path.isabs(topsrcdir) + assert os.path.isabs(topobjdir) + + self.topsrcdir = os.path.normpath(topsrcdir) + self.topobjdir = os.path.normpath(topobjdir) + self.src_clobber = os.path.join(topsrcdir, 'CLOBBER') + self.obj_clobber = os.path.join(topobjdir, 'CLOBBER') + + assert os.path.isfile(self.src_clobber) + + def clobber_needed(self): + """Returns a bool indicating whether a tree clobber is required.""" + + # No object directory clobber file means we're good. + if not os.path.exists(self.obj_clobber): + return False + + # Object directory clobber older than current is fine. + if os.path.getmtime(self.src_clobber) <= \ + os.path.getmtime(self.obj_clobber): + + return False + + return True + + def clobber_cause(self): + """Obtain the cause why a clobber is required. + + This reads the cause from the CLOBBER file. + + This returns a list of lines describing why the clobber was required. + Each line is stripped of leading and trailing whitespace. + """ + with open(self.src_clobber, 'rt') as fh: + lines = [l.strip() for l in fh.readlines()] + return [l for l in lines if l and not l.startswith('#')] + + def ensure_objdir_state(self): + """Ensure the CLOBBER file in the objdir exists. + + This is called as part of the build to ensure the clobber information + is configured properly for the objdir. + """ + if not os.path.exists(self.topobjdir): + os.makedirs(self.topobjdir) + + if not os.path.exists(self.obj_clobber): + # Simply touch the file. + with open(self.obj_clobber, 'a'): + pass + + def maybe_do_clobber(self, cwd, allow_auto=True, fh=sys.stderr): + """Perform a clobber if it is required. Maybe. + + This is the API the build system invokes to determine if a clobber + is needed and to automatically perform that clobber if we can. + + This returns a tuple of (bool, bool, str). The elements are: + + - Whether a clobber was/is required. + - Whether a clobber was performed. + - The reason why the clobber failed or could not be performed. This + will be None if no clobber is required or if we clobbered without + error. + """ + assert cwd + cwd = os.path.normpath(cwd) + + if not self.clobber_needed(): + print('Clobber not needed.', file=fh) + self.ensure_objdir_state() + return False, False, None + + # So a clobber is needed. We only perform a clobber if we are + # allowed to perform an automatic clobber (the default) and if the + # current directory is not under the object directory. The latter is + # because operating systems, filesystems, and shell can throw fits + # if the current working directory is deleted from under you. While it + # can work in some scenarios, we take the conservative approach and + # never try. + if not allow_auto: + return True, False, self._message( + 'Automatic clobbering has been disabled.') + + if cwd.startswith(self.topobjdir) and cwd != self.topobjdir: + return True, False, self._message( + 'Cannot clobber while the shell is inside the object directory.') + + print('Automatically clobbering %s' % self.topobjdir, file=fh) + try: + if cwd == self.topobjdir: + for entry in os.listdir(self.topobjdir): + full = os.path.join(self.topobjdir, entry) + + if os.path.isdir(full): + rmtree(full) + else: + os.unlink(full) + + else: + rmtree(self.topobjdir) + + self.ensure_objdir_state() + print('Successfully completed auto clobber.', file=fh) + return True, True, None + except (IOError) as error: + return True, False, self._message( + 'Error when automatically clobbering: ' + str(error)) + + def _message(self, reason): + lines = ['* ' + line for line in self.clobber_cause()] + + return CLOBBER_MESSAGE.format(clobber_reason='\n'.join(lines), + no_reason=reason, clobber_file=self.obj_clobber) + + +def main(args, env, cwd, fh=sys.stderr): + if len(args) != 2: + print('Usage: clobber.py topsrcdir topobjdir', file=fh) + return 1 + + topsrcdir, topobjdir = args + + if not os.path.isabs(topsrcdir): + topsrcdir = os.path.abspath(topsrcdir) + + if not os.path.isabs(topobjdir): + topobjdir = os.path.abspath(topobjdir) + + auto = False if env.get('NO_AUTOCLOBBER', False) else True + clobber = Clobberer(topsrcdir, topobjdir) + required, performed, message = clobber.maybe_do_clobber(cwd, auto, fh) + + if not required or performed: + return 0 + + print(message, file=fh) + return 1 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:], os.environ, os.getcwd(), sys.stdout)) + diff --git a/python/mozbuild/mozbuild/test/controller/__init__.py b/python/mozbuild/mozbuild/test/controller/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/mozbuild/test/controller/test_clobber.py b/python/mozbuild/mozbuild/test/controller/test_clobber.py new file mode 100644 index 000000000000..c95d7dcb46da --- /dev/null +++ b/python/mozbuild/mozbuild/test/controller/test_clobber.py @@ -0,0 +1,208 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import os +import shutil +import tempfile +import unittest + +from StringIO import StringIO + +from mozunit import main + +from mozbuild.controller.clobber import Clobberer +from mozbuild.controller.clobber import main as clobber + + +class TestClobberer(unittest.TestCase): + def setUp(self): + self._temp_dirs = [] + + return unittest.TestCase.setUp(self) + + def tearDown(self): + for d in self._temp_dirs: + shutil.rmtree(d, ignore_errors=True) + + return unittest.TestCase.tearDown(self) + + def get_tempdir(self): + t = tempfile.mkdtemp() + self._temp_dirs.append(t) + return t + + def get_topsrcdir(self): + t = self.get_tempdir() + p = os.path.join(t, 'CLOBBER') + with open(p, 'a'): + pass + + return t + + def test_no_objdir(self): + """If topobjdir does not exist, no clobber is needed.""" + + tmp = os.path.join(self.get_tempdir(), 'topobjdir') + self.assertFalse(os.path.exists(tmp)) + + c = Clobberer(self.get_topsrcdir(), tmp) + self.assertFalse(c.clobber_needed()) + + # Side-effect is topobjdir is created with CLOBBER file touched. + required, performed, reason = c.maybe_do_clobber(os.getcwd()) + self.assertFalse(required) + self.assertFalse(performed) + self.assertIsNone(reason) + + self.assertTrue(os.path.isdir(tmp)) + self.assertTrue(os.path.exists(os.path.join(tmp, 'CLOBBER'))) + + def test_objdir_no_clobber_file(self): + """If CLOBBER does not exist in topobjdir, treat as empty.""" + + c = Clobberer(self.get_topsrcdir(), self.get_tempdir()) + self.assertFalse(c.clobber_needed()) + + required, performed, reason = c.maybe_do_clobber(os.getcwd()) + self.assertFalse(required) + self.assertFalse(performed) + self.assertIsNone(reason) + + self.assertTrue(os.path.exists(os.path.join(c.topobjdir, 'CLOBBER'))) + + def test_objdir_clobber_newer(self): + """If CLOBBER in topobjdir is newer, do nothing.""" + + c = Clobberer(self.get_topsrcdir(), self.get_tempdir()) + with open(c.obj_clobber, 'a'): + pass + + required, performed, reason = c.maybe_do_clobber(os.getcwd()) + self.assertFalse(required) + self.assertFalse(performed) + self.assertIsNone(reason) + + def test_objdir_clobber_older(self): + """If CLOBBER in topobjdir is older, we clobber.""" + + c = Clobberer(self.get_topsrcdir(), self.get_tempdir()) + with open(c.obj_clobber, 'a'): + pass + + dummy_path = os.path.join(c.topobjdir, 'foo') + with open(dummy_path, 'a'): + pass + + self.assertTrue(os.path.exists(dummy_path)) + + old_time = os.path.getmtime(c.src_clobber) - 60 + os.utime(c.obj_clobber, (old_time, old_time)) + + self.assertTrue(c.clobber_needed()) + + required, performed, reason = c.maybe_do_clobber(os.getcwd(), False) + self.assertTrue(required) + self.assertFalse(performed) + self.assertIn('Automatic clobbering has been disabled', reason) + + # Now let's actually do it. + required, performed, reason = c.maybe_do_clobber(os.getcwd()) + self.assertTrue(required) + self.assertTrue(performed) + + self.assertFalse(os.path.exists(dummy_path)) + self.assertTrue(os.path.exists(c.obj_clobber)) + self.assertGreaterEqual(os.path.getmtime(c.obj_clobber), + os.path.getmtime(c.src_clobber)) + + def test_objdir_is_srcdir(self): + """If topobjdir is the topsrcdir, refuse to clobber.""" + + tmp = self.get_topsrcdir() + c = Clobberer(tmp, tmp) + + self.assertFalse(c.clobber_needed()) + + def test_cwd_is_topobjdir(self): + """If cwd is topobjdir, we can still clobber.""" + c = Clobberer(self.get_topsrcdir(), self.get_tempdir()) + + with open(c.obj_clobber, 'a'): + pass + + dummy_file = os.path.join(c.topobjdir, 'dummy_file') + with open(dummy_file, 'a'): + pass + + dummy_dir = os.path.join(c.topobjdir, 'dummy_dir') + os.mkdir(dummy_dir) + + self.assertTrue(os.path.exists(dummy_file)) + self.assertTrue(os.path.isdir(dummy_dir)) + + old_time = os.path.getmtime(c.src_clobber) - 60 + os.utime(c.obj_clobber, (old_time, old_time)) + + self.assertTrue(c.clobber_needed()) + + required, performed, reason = c.maybe_do_clobber(c.topobjdir) + self.assertTrue(required) + self.assertTrue(performed) + + self.assertFalse(os.path.exists(dummy_file)) + self.assertFalse(os.path.exists(dummy_dir)) + + def test_cwd_under_topobjdir(self): + """If cwd is under topobjdir, we can't clobber.""" + + c = Clobberer(self.get_topsrcdir(), self.get_tempdir()) + + with open(c.obj_clobber, 'a'): + pass + + old_time = os.path.getmtime(c.src_clobber) - 60 + os.utime(c.obj_clobber, (old_time, old_time)) + + d = os.path.join(c.topobjdir, 'dummy_dir') + os.mkdir(d) + + required, performed, reason = c.maybe_do_clobber(d) + self.assertTrue(required) + self.assertFalse(performed) + self.assertIn('Cannot clobber while the shell is inside', reason) + + + def test_mozconfig_overrides_auto_clobber(self): + """If NO_AUTOCLOBBER is in the environment, don't auto clobber.""" + + topsrcdir = self.get_topsrcdir() + topobjdir = self.get_tempdir() + + obj_clobber = os.path.join(topobjdir, 'CLOBBER') + with open(obj_clobber, 'a'): + pass + + dummy_file = os.path.join(topobjdir, 'dummy_file') + with open(dummy_file, 'a'): + pass + + self.assertTrue(os.path.exists(dummy_file)) + + old_time = os.path.getmtime(os.path.join(topsrcdir, 'CLOBBER')) - 60 + os.utime(obj_clobber, (old_time, old_time)) + + env = dict(os.environ) + env['NO_AUTOCLOBBER'] = '1' + + s = StringIO() + status = clobber([topsrcdir, topobjdir], env, os.getcwd(), s) + self.assertEqual(status, 1) + self.assertIn('Automatic clobbering has been disabled', s.getvalue()) + self.assertTrue(os.path.exists(dummy_file)) + + +if __name__ == '__main__': + main()