Bug 837323 - Automatically clobber when CLOBBER is updated; r=ted

Makefile.in

@@ -35,7 +35,7 @@ DIST_GARBAGE = config.cache config.log config.status* config-defs.h \
    netwerk/necko-config.h xpcom/xpcom-config.h xpcom/xpcom-private.h \
    $(topsrcdir)/.mozconfig.mk $(topsrcdir)/.mozconfig.out
 
-default alldep all:: $(topsrcdir)/configure config.status
+default alldep all:: CLOBBER $(topsrcdir)/configure config.status
 	$(RM) -r $(DIST)/sdk
 	$(RM) -r $(DIST)/include
 	$(RM) -r $(DIST)/private
@@ -43,6 +43,12 @@ default alldep all:: $(topsrcdir)/configure config.status
 	$(RM) $(DIST)/bin/chrome.manifest $(DIST)/bin/components/components.manifest
 	$(RM) -r _tests
 
+CLOBBER: $(topsrcdir)/CLOBBER
+	@echo "STOP!  The CLOBBER file has changed."
+	@echo "Please run the build through a sanctioned build wrapper, such as"
+	@echo "'mach build' or client.mk."
+	@exit 1
+
 $(topsrcdir)/configure: $(topsrcdir)/configure.in
 	@echo "STOP!  configure.in has changed, and your configure is out of date."
 	@echo "Please rerun autoconf and re-configure your build directory."

client.mk

@@ -108,6 +108,9 @@ endef
 # before evaluation.
 $(eval $(subst ||,$(CR),$(shell _PYMAKE=$(.PYMAKE) $(TOPSRCDIR)/$(MOZCONFIG_LOADER) $(TOPSRCDIR) 2> $(TOPSRCDIR)/.mozconfig.out | sed 's/$$/||/')))
 
+ifdef NO_AUTOCLOBBER
+export NO_AUTOCLOBBER=1
+endif
 
 # Automatically add -jN to make flags if not defined. N defaults to number of cores.
 ifeq (,$(findstring -j,$(MOZ_MAKE_FLAGS)))
@@ -302,9 +305,13 @@ else
   CONFIGURE = $(TOPSRCDIR)/configure
 endif
 
+check-clobber:
+	$(PYTHON) $(TOPSRCDIR)/python/mozbuild/mozbuild/controller/clobber.py $(TOPSRCDIR) $(OBJDIR)
+
 configure-files: $(CONFIGURES)
 
 configure-preqs = \
+  check-clobber \
   configure-files \
   $(call mkdir_deps,$(OBJDIR)) \
   $(if $(MOZ_BUILD_PROJECTS),$(call mkdir_deps,$(MOZ_OBJDIR))) \
@@ -442,4 +449,23 @@ echo-variable-%:
 # in parallel.
 .NOTPARALLEL:
 
-.PHONY: checkout real_checkout depend realbuild build profiledbuild cleansrcdir pull_all build_all clobber clobber_all pull_and_build_all everything configure preflight_all preflight postflight postflight_all $(OBJDIR_TARGETS)
+.PHONY: checkout \
+    real_checkout \
+    depend \
+    realbuild \
+    build \
+    profiledbuild \
+    cleansrcdir \
+    pull_all \
+    build_all \
+    check-clobber \
+    clobber \
+    clobber_all \
+    pull_and_build_all \
+    everything \
+    configure \
+    preflight_all \
+    preflight \
+    postflight \
+    postflight_all \
+    $(OBJDIR_TARGETS)

configure.in

@@ -120,30 +120,6 @@ then
 fi
 MOZ_BUILD_ROOT=`pwd`
 
-dnl Do not allow building if a clobber is required
-dnl ==============================================================
-dnl TODO Make this better, ideally this would clobber automaticially
-if test -e $_objdir/CLOBBER; then
-  if test $_topsrcdir/CLOBBER -nt $_objdir/CLOBBER; then
-    echo "	***"
-    echo "	*	The CLOBBER file has been updated, indicating that an incremental build"
-    echo "	*	since your last build will probably not work. A full build is required."
-    echo "	*	The change that caused this is:"
-    cat $_topsrcdir/CLOBBER | sed '/^#/d' | sed 's/^/	*	/'
-    echo "	*	"
-    echo "	*	The easiest way to fix this is to manually delete your objdir:"
-    echo "	*	rm -rf $_objdir"
-    echo "	*	"
-    echo "	*	Or, if you know this clobber doesn't apply to you, it can be ignored with:"
-    echo "	*	cp '$_topsrcdir/CLOBBER' $_objdir"
-    echo "	***"
-    exit 1
-    break;
-  fi
-else
-  touch $_objdir/CLOBBER
-fi
-
 MOZ_PYTHON
 
 MOZ_DEFAULT_COMPILER

python/Makefile.in

@@ -12,6 +12,7 @@ include $(DEPTH)/config/autoconf.mk
 test_dirs := \
   mozbuild/mozbuild/test \
   mozbuild/mozbuild/test/backend \
+  mozbuild/mozbuild/test/controller \
   mozbuild/mozbuild/test/compilation \
   mozbuild/mozbuild/test/frontend \
   mozbuild/mozpack/test \

python/mozbuild/mozbuild/controller/clobber.py

@@ -0,0 +1,194 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function
+
+r'''This module contains code for managing clobbering of the tree.'''
+
+import os
+import sys
+
+try:
+    from mozfile.mozfile import rmtree
+except ImportError:
+    from shutil import rmtree
+
+
+CLOBBER_MESSAGE = '''
+***
+* The CLOBBER file has been updated, indicating that an incremental build since
+* your last build will probably not work. A full/clobber build is required.
+*
+* The reason for the clobber is:
+*
+{clobber_reason}
+*
+* Clobbering can be performed automatically. However, we didn't automatically
+* clobber this time because:
+*
+*   {no_reason}
+*
+* The easiest and fastest way to clobber is to run:
+*
+*  $ mach clobber
+*
+* If you know this clobber doesn't apply to you or you're feeling lucky - well
+* do ya? - you can ignore this clobber requirement by running:
+*
+*  $ touch {clobber_file}
+*
+***
+'''.strip()
+
+
+class Clobberer(object):
+    def __init__(self, topsrcdir, topobjdir):
+        """Create a new object to manage clobbering the tree.
+
+        It is bound to a top source directory and to a specific object
+        directory.
+        """
+        assert os.path.isabs(topsrcdir)
+        assert os.path.isabs(topobjdir)
+
+        self.topsrcdir = os.path.normpath(topsrcdir)
+        self.topobjdir = os.path.normpath(topobjdir)
+        self.src_clobber = os.path.join(topsrcdir, 'CLOBBER')
+        self.obj_clobber = os.path.join(topobjdir, 'CLOBBER')
+
+        assert os.path.isfile(self.src_clobber)
+
+    def clobber_needed(self):
+        """Returns a bool indicating whether a tree clobber is required."""
+
+        # No object directory clobber file means we're good.
+        if not os.path.exists(self.obj_clobber):
+            return False
+
+        # Object directory clobber older than current is fine.
+        if os.path.getmtime(self.src_clobber) <= \
+            os.path.getmtime(self.obj_clobber):
+
+            return False
+
+        return True
+
+    def clobber_cause(self):
+        """Obtain the cause why a clobber is required.
+
+        This reads the cause from the CLOBBER file.
+
+        This returns a list of lines describing why the clobber was required.
+        Each line is stripped of leading and trailing whitespace.
+        """
+        with open(self.src_clobber, 'rt') as fh:
+            lines = [l.strip() for l in fh.readlines()]
+            return [l for l in lines if l and not l.startswith('#')]
+
+    def ensure_objdir_state(self):
+        """Ensure the CLOBBER file in the objdir exists.
+
+        This is called as part of the build to ensure the clobber information
+        is configured properly for the objdir.
+        """
+        if not os.path.exists(self.topobjdir):
+            os.makedirs(self.topobjdir)
+
+        if not os.path.exists(self.obj_clobber):
+            # Simply touch the file.
+            with open(self.obj_clobber, 'a'):
+                pass
+
+    def maybe_do_clobber(self, cwd, allow_auto=True, fh=sys.stderr):
+        """Perform a clobber if it is required. Maybe.
+
+        This is the API the build system invokes to determine if a clobber
+        is needed and to automatically perform that clobber if we can.
+
+        This returns a tuple of (bool, bool, str). The elements are:
+
+          - Whether a clobber was/is required.
+          - Whether a clobber was performed.
+          - The reason why the clobber failed or could not be performed. This
+            will be None if no clobber is required or if we clobbered without
+            error.
+        """
+        assert cwd
+        cwd = os.path.normpath(cwd)
+
+        if not self.clobber_needed():
+            print('Clobber not needed.', file=fh)
+            self.ensure_objdir_state()
+            return False, False, None
+
+        # So a clobber is needed. We only perform a clobber if we are
+        # allowed to perform an automatic clobber (the default) and if the
+        # current directory is not under the object directory. The latter is
+        # because operating systems, filesystems, and shell can throw fits
+        # if the current working directory is deleted from under you. While it
+        # can work in some scenarios, we take the conservative approach and
+        # never try.
+        if not allow_auto:
+            return True, False, self._message(
+                'Automatic clobbering has been disabled.')
+
+        if cwd.startswith(self.topobjdir) and cwd != self.topobjdir:
+            return True, False, self._message(
+                'Cannot clobber while the shell is inside the object directory.')
+
+        print('Automatically clobbering %s' % self.topobjdir, file=fh)
+        try:
+            if cwd == self.topobjdir:
+                for entry in os.listdir(self.topobjdir):
+                    full = os.path.join(self.topobjdir, entry)
+
+                    if os.path.isdir(full):
+                        rmtree(full)
+                    else:
+                        os.unlink(full)
+
+            else:
+                rmtree(self.topobjdir)
+
+            self.ensure_objdir_state()
+            print('Successfully completed auto clobber.', file=fh)
+            return True, True, None
+        except (IOError) as error:
+            return True, False, self._message(
+                'Error when automatically clobbering: ' + str(error))
+
+    def _message(self, reason):
+        lines = ['*  ' + line for line in self.clobber_cause()]
+
+        return CLOBBER_MESSAGE.format(clobber_reason='\n'.join(lines),
+            no_reason=reason, clobber_file=self.obj_clobber)
+
+
+def main(args, env, cwd, fh=sys.stderr):
+    if len(args) != 2:
+        print('Usage: clobber.py topsrcdir topobjdir', file=fh)
+        return 1
+
+    topsrcdir, topobjdir = args
+
+    if not os.path.isabs(topsrcdir):
+        topsrcdir = os.path.abspath(topsrcdir)
+
+    if not os.path.isabs(topobjdir):
+        topobjdir = os.path.abspath(topobjdir)
+
+    auto = False if env.get('NO_AUTOCLOBBER', False) else True
+    clobber = Clobberer(topsrcdir, topobjdir)
+    required, performed, message = clobber.maybe_do_clobber(cwd, auto, fh)
+
+    if not required or performed:
+        return 0
+
+    print(message, file=fh)
+    return 1
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:], os.environ, os.getcwd(), sys.stdout))
+

python/mozbuild/mozbuild/test/controller/test_clobber.py

@@ -0,0 +1,208 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import os
+import shutil
+import tempfile
+import unittest
+
+from StringIO import StringIO
+
+from mozunit import main
+
+from mozbuild.controller.clobber import Clobberer
+from mozbuild.controller.clobber import main as clobber
+
+
+class TestClobberer(unittest.TestCase):
+    def setUp(self):
+        self._temp_dirs = []
+
+        return unittest.TestCase.setUp(self)
+
+    def tearDown(self):
+        for d in self._temp_dirs:
+            shutil.rmtree(d, ignore_errors=True)
+
+        return unittest.TestCase.tearDown(self)
+
+    def get_tempdir(self):
+        t = tempfile.mkdtemp()
+        self._temp_dirs.append(t)
+        return t
+
+    def get_topsrcdir(self):
+        t = self.get_tempdir()
+        p = os.path.join(t, 'CLOBBER')
+        with open(p, 'a'):
+            pass
+
+        return t
+
+    def test_no_objdir(self):
+        """If topobjdir does not exist, no clobber is needed."""
+
+        tmp = os.path.join(self.get_tempdir(), 'topobjdir')
+        self.assertFalse(os.path.exists(tmp))
+
+        c = Clobberer(self.get_topsrcdir(), tmp)
+        self.assertFalse(c.clobber_needed())
+
+        # Side-effect is topobjdir is created with CLOBBER file touched.
+        required, performed, reason = c.maybe_do_clobber(os.getcwd())
+        self.assertFalse(required)
+        self.assertFalse(performed)
+        self.assertIsNone(reason)
+
+        self.assertTrue(os.path.isdir(tmp))
+        self.assertTrue(os.path.exists(os.path.join(tmp, 'CLOBBER')))
+
+    def test_objdir_no_clobber_file(self):
+        """If CLOBBER does not exist in topobjdir, treat as empty."""
+
+        c = Clobberer(self.get_topsrcdir(), self.get_tempdir())
+        self.assertFalse(c.clobber_needed())
+
+        required, performed, reason = c.maybe_do_clobber(os.getcwd())
+        self.assertFalse(required)
+        self.assertFalse(performed)
+        self.assertIsNone(reason)
+
+        self.assertTrue(os.path.exists(os.path.join(c.topobjdir, 'CLOBBER')))
+
+    def test_objdir_clobber_newer(self):
+        """If CLOBBER in topobjdir is newer, do nothing."""
+
+        c = Clobberer(self.get_topsrcdir(), self.get_tempdir())
+        with open(c.obj_clobber, 'a'):
+            pass
+
+        required, performed, reason = c.maybe_do_clobber(os.getcwd())
+        self.assertFalse(required)
+        self.assertFalse(performed)
+        self.assertIsNone(reason)
+
+    def test_objdir_clobber_older(self):
+        """If CLOBBER in topobjdir is older, we clobber."""
+
+        c = Clobberer(self.get_topsrcdir(), self.get_tempdir())
+        with open(c.obj_clobber, 'a'):
+            pass
+
+        dummy_path = os.path.join(c.topobjdir, 'foo')
+        with open(dummy_path, 'a'):
+            pass
+
+        self.assertTrue(os.path.exists(dummy_path))
+
+        old_time = os.path.getmtime(c.src_clobber) - 60
+        os.utime(c.obj_clobber, (old_time, old_time))
+
+        self.assertTrue(c.clobber_needed())
+
+        required, performed, reason = c.maybe_do_clobber(os.getcwd(), False)
+        self.assertTrue(required)
+        self.assertFalse(performed)
+        self.assertIn('Automatic clobbering has been disabled', reason)
+
+        # Now let's actually do it.
+        required, performed, reason = c.maybe_do_clobber(os.getcwd())
+        self.assertTrue(required)
+        self.assertTrue(performed)
+
+        self.assertFalse(os.path.exists(dummy_path))
+        self.assertTrue(os.path.exists(c.obj_clobber))
+        self.assertGreaterEqual(os.path.getmtime(c.obj_clobber),
+            os.path.getmtime(c.src_clobber))
+
+    def test_objdir_is_srcdir(self):
+        """If topobjdir is the topsrcdir, refuse to clobber."""
+
+        tmp = self.get_topsrcdir()
+        c = Clobberer(tmp, tmp)
+
+        self.assertFalse(c.clobber_needed())
+
+    def test_cwd_is_topobjdir(self):
+        """If cwd is topobjdir, we can still clobber."""
+        c = Clobberer(self.get_topsrcdir(), self.get_tempdir())
+
+        with open(c.obj_clobber, 'a'):
+            pass
+
+        dummy_file = os.path.join(c.topobjdir, 'dummy_file')
+        with open(dummy_file, 'a'):
+            pass
+
+        dummy_dir = os.path.join(c.topobjdir, 'dummy_dir')
+        os.mkdir(dummy_dir)
+
+        self.assertTrue(os.path.exists(dummy_file))
+        self.assertTrue(os.path.isdir(dummy_dir))
+
+        old_time = os.path.getmtime(c.src_clobber) - 60
+        os.utime(c.obj_clobber, (old_time, old_time))
+
+        self.assertTrue(c.clobber_needed())
+
+        required, performed, reason = c.maybe_do_clobber(c.topobjdir)
+        self.assertTrue(required)
+        self.assertTrue(performed)
+
+        self.assertFalse(os.path.exists(dummy_file))
+        self.assertFalse(os.path.exists(dummy_dir))
+
+    def test_cwd_under_topobjdir(self):
+        """If cwd is under topobjdir, we can't clobber."""
+
+        c = Clobberer(self.get_topsrcdir(), self.get_tempdir())
+
+        with open(c.obj_clobber, 'a'):
+            pass
+
+        old_time = os.path.getmtime(c.src_clobber) - 60
+        os.utime(c.obj_clobber, (old_time, old_time))
+
+        d = os.path.join(c.topobjdir, 'dummy_dir')
+        os.mkdir(d)
+
+        required, performed, reason = c.maybe_do_clobber(d)
+        self.assertTrue(required)
+        self.assertFalse(performed)
+        self.assertIn('Cannot clobber while the shell is inside', reason)
+
+
+    def test_mozconfig_overrides_auto_clobber(self):
+        """If NO_AUTOCLOBBER is in the environment, don't auto clobber."""
+
+        topsrcdir = self.get_topsrcdir()
+        topobjdir = self.get_tempdir()
+
+        obj_clobber = os.path.join(topobjdir, 'CLOBBER')
+        with open(obj_clobber, 'a'):
+            pass
+
+        dummy_file = os.path.join(topobjdir, 'dummy_file')
+        with open(dummy_file, 'a'):
+            pass
+
+        self.assertTrue(os.path.exists(dummy_file))
+
+        old_time = os.path.getmtime(os.path.join(topsrcdir, 'CLOBBER')) - 60
+        os.utime(obj_clobber, (old_time, old_time))
+
+        env = dict(os.environ)
+        env['NO_AUTOCLOBBER'] = '1'
+
+        s = StringIO()
+        status = clobber([topsrcdir, topobjdir], env, os.getcwd(), s)
+        self.assertEqual(status, 1)
+        self.assertIn('Automatic clobbering has been disabled', s.getvalue())
+        self.assertTrue(os.path.exists(dummy_file))
+
+
+if __name__ == '__main__':
+    main()