bug 1286934 - Switch to using sccache2. r=gps

This patch does a few things: 1) Change all the in-tree tooltool manifests to contain sccache2 instead of the existing Python sccache 2) Change mozconfig.cache to point at sccache. 3) Lightly tweak the --with-cccache configure option to support sccache, and detect whether we're using ccache or sccache and set an option appropriately. 4) Add a MOZ_SCCACHE_VERBOSE_STATS option, and add a target in the top-level Makefile to make sccache spit out its stats at the end of the build. This is useful to see the cache hits/errors until we get something better. 5) Add MOZ_USING_SCCACHE to the build telemetry. Not that I think it will be immediately useful, but for future use. MozReview-Commit-ID: 9lrdLwNj5Bm --HG-- extra : rebase_source : d323457df10d0ee0ac5811940e518d9422a7e070
2016-08-25 18:39:57 -04:00 · 2016-08-25 18:39:57 -04:00 · fa343243d0
--- a/Makefile.in
+++ b/Makefile.in
@ -262,6 +262,14 @@ default all::

 include $(topsrcdir)/config/rules.mk

+ifdef SCCACHE_VERBOSE_STATS
+# This won't contain stats for both halves of a universal build, but I can live with that.
+default::
+	@echo "===SCCACHE STATS==="
+	-$(CCACHE) --show-stats
+	@echo "==================="
+endif
+
 distclean::
 	$(RM) $(DIST_GARBAGE)

--- a/browser/config/tooltool-manifests/linux32/clang.manifest
+++ b/browser/config/tooltool-manifests/linux32/clang.manifest
@ -5,6 +5,6 @@
 "digest": "34e219d7e8eaffa81710631c34d21355563d06335b3c00851e94c1f42f9098788fded8463dd0f67dd699f77b47a0245dd7aff754943a7a03fb5fd145a808254f",
 "algorithm": "sha512", 
 "filename": "clang.tar.xz",
-"unpack": true,
+"unpack": true
 }
 ]
--- a/browser/config/tooltool-manifests/linux32/releng.manifest
+++ b/browser/config/tooltool-manifests/linux32/releng.manifest
@ -24,10 +24,12 @@
 "unpack": true
 },
 {
-"size": 167175,
-"digest": "0b71a936edf5bd70cf274aaa5d7abc8f77fe8e7b5593a208f805cc9436fac646b9c4f0b43c2b10de63ff3da671497d35536077ecbc72dba7f8159a38b580f831",
+"version": "sccache rev b21198a7183a2fe226ff49348b1c0b51bae9f4f8",
 "algorithm": "sha512",
-"filename": "sccache.tar.bz2",
-"unpack": true
+"visibility": "public",
+"filename": "sccache2.tar.xz",
+"unpack": true,
+"digest": "b89c40dbf28c2bd54fadf017c15a8789f6e7611252a623cc3a1507e3dd6fc9e5a50d746e81776ba856e33fdc99b4a6413ba7c3ac0aed5f4835705da2b758ef22",
+"size": 1020700
 }
 ]
--- a/browser/config/tooltool-manifests/linux64/clang.manifest
+++ b/browser/config/tooltool-manifests/linux64/clang.manifest
@ -16,10 +16,12 @@
 "unpack": true
 },
 {
-"size": 167175,
-"digest": "0b71a936edf5bd70cf274aaa5d7abc8f77fe8e7b5593a208f805cc9436fac646b9c4f0b43c2b10de63ff3da671497d35536077ecbc72dba7f8159a38b580f831",
+"version": "sccache rev b21198a7183a2fe226ff49348b1c0b51bae9f4f8",
 "algorithm": "sha512",
-"filename": "sccache.tar.bz2",
-"unpack": true
+"visibility": "public",
+"filename": "sccache2.tar.xz",
+"unpack": true,
+"digest": "b89c40dbf28c2bd54fadf017c15a8789f6e7611252a623cc3a1507e3dd6fc9e5a50d746e81776ba856e33fdc99b4a6413ba7c3ac0aed5f4835705da2b758ef22",
+"size": 1020700
 }
 ]
--- a/browser/config/tooltool-manifests/linux64/hazard.manifest
+++ b/browser/config/tooltool-manifests/linux64/hazard.manifest
@ -32,10 +32,12 @@
 "unpack": true
 },
 {
-"filename" : "sccache.tar.bz2",
-"algorithm" : "sha512",
-"digest" : "0b71a936edf5bd70cf274aaa5d7abc8f77fe8e7b5593a208f805cc9436fac646b9c4f0b43c2b10de63ff3da671497d35536077ecbc72dba7f8159a38b580f831",
-"unpack" : true,
-"size" : 167175
+"version": "sccache rev b21198a7183a2fe226ff49348b1c0b51bae9f4f8",
+"algorithm": "sha512",
+"visibility": "public",
+"filename": "sccache2.tar.xz",
+"unpack": true,
+"digest": "b89c40dbf28c2bd54fadf017c15a8789f6e7611252a623cc3a1507e3dd6fc9e5a50d746e81776ba856e33fdc99b4a6413ba7c3ac0aed5f4835705da2b758ef22",
+"size": 1020700
 }
 ]
--- a/browser/config/tooltool-manifests/linux64/releng.manifest
+++ b/browser/config/tooltool-manifests/linux64/releng.manifest
@ -24,10 +24,12 @@
 "unpack": true
 },
 {
-"size": 167175,
-"digest": "0b71a936edf5bd70cf274aaa5d7abc8f77fe8e7b5593a208f805cc9436fac646b9c4f0b43c2b10de63ff3da671497d35536077ecbc72dba7f8159a38b580f831",
+"version": "sccache rev b21198a7183a2fe226ff49348b1c0b51bae9f4f8",
 "algorithm": "sha512",
-"filename": "sccache.tar.bz2",
-"unpack": true
+"visibility": "public",
+"filename": "sccache2.tar.xz",
+"unpack": true,
+"digest": "b89c40dbf28c2bd54fadf017c15a8789f6e7611252a623cc3a1507e3dd6fc9e5a50d746e81776ba856e33fdc99b4a6413ba7c3ac0aed5f4835705da2b758ef22",
+"size": 1020700
 }
 ]
--- a/browser/config/tooltool-manifests/macosx64/clang.manifest
+++ b/browser/config/tooltool-manifests/macosx64/clang.manifest
@ -8,11 +8,13 @@
 "unpack": true
 },
 {
-"size": 167175,
-"digest": "0b71a936edf5bd70cf274aaa5d7abc8f77fe8e7b5593a208f805cc9436fac646b9c4f0b43c2b10de63ff3da671497d35536077ecbc72dba7f8159a38b580f831",
+"version": "sccache rev b21198a7183a2fe226ff49348b1c0b51bae9f4f8",
 "algorithm": "sha512",
-"filename": "sccache.tar.bz2",
-"unpack": true
+"visibility": "public",
+"filename": "sccache2.tar.bz2",
+"unpack": true,
+"digest": "a285c7c6468ad7438262dfec90f65981e84abf2adbb1aa075c0ec1759b4f98ce5d5f14a3d555274f970704210a00738ba7d95db2fc320f7780e6b99bcb0ffb6c",
+"size": 1143715
 },
 {
 "version": "cctools port from commit hash db1f8d906cb28, ld only",
--- a/browser/config/tooltool-manifests/macosx64/cross-releng.manifest
+++ b/browser/config/tooltool-manifests/macosx64/cross-releng.manifest
@ -24,11 +24,13 @@
 "filename": "MacOSX10.7.sdk.tar.bz2"
 },
 {
-"size": 167175,
-"digest": "0b71a936edf5bd70cf274aaa5d7abc8f77fe8e7b5593a208f805cc9436fac646b9c4f0b43c2b10de63ff3da671497d35536077ecbc72dba7f8159a38b580f831",
+"version": "sccache rev b21198a7183a2fe226ff49348b1c0b51bae9f4f8",
 "algorithm": "sha512",
+"visibility": "public",
+"filename": "sccache2.tar.xz",
 "unpack": true,
-"filename": "sccache.tar.bz2"
+"digest": "b89c40dbf28c2bd54fadf017c15a8789f6e7611252a623cc3a1507e3dd6fc9e5a50d746e81776ba856e33fdc99b4a6413ba7c3ac0aed5f4835705da2b758ef22",
+"size": 1020700
 },
 {
 "size": 57060, 
--- a/browser/config/tooltool-manifests/macosx64/releng.manifest
+++ b/browser/config/tooltool-manifests/macosx64/releng.manifest
@ -24,10 +24,12 @@
 "unpack": true
 },
 {
-"size": 167175,
-"digest": "0b71a936edf5bd70cf274aaa5d7abc8f77fe8e7b5593a208f805cc9436fac646b9c4f0b43c2b10de63ff3da671497d35536077ecbc72dba7f8159a38b580f831",
+"version": "sccache rev b21198a7183a2fe226ff49348b1c0b51bae9f4f8",
 "algorithm": "sha512",
-"filename": "sccache.tar.bz2",
-"unpack": true
+"visibility": "public",
+"filename": "sccache2.tar.bz2",
+"unpack": true,
+"digest": "a285c7c6468ad7438262dfec90f65981e84abf2adbb1aa075c0ec1759b4f98ce5d5f14a3d555274f970704210a00738ba7d95db2fc320f7780e6b99bcb0ffb6c",
+"size": 1143715
 }
 ]
--- a/browser/config/tooltool-manifests/win32/build-clang-cl.manifest
+++ b/browser/config/tooltool-manifests/win32/build-clang-cl.manifest
@ -14,11 +14,13 @@
 "unpack": true
 },
 {
-"size": 167175,
-"digest": "0b71a936edf5bd70cf274aaa5d7abc8f77fe8e7b5593a208f805cc9436fac646b9c4f0b43c2b10de63ff3da671497d35536077ecbc72dba7f8159a38b580f831",
+"version": "sccache rev b21198a7183a2fe226ff49348b1c0b51bae9f4f8",
 "algorithm": "sha512",
-"filename": "sccache.tar.bz2",
-"unpack": true
+"visibility": "public",
+"filename": "sccache2.tar.bz2",
+"unpack": true,
+"digest": "7dee5c5602b3830cb8ac45ebaa8542714bbac0e50eabbff58a06972a02ceeab75ed7c56ff22a23f760b8317ae8e9a01cdecfaf75a7acbd2a4cdd817967170d2e",
+"size": 1179901
 },
 {
 "version": "Visual Studio 2015 Update 3 14.0.25425.01 / SDK 10.0.14393.0",
--- a/browser/config/tooltool-manifests/win32/clang.manifest
+++ b/browser/config/tooltool-manifests/win32/clang.manifest
@ -14,11 +14,13 @@
 "unpack": true
 },
 {
-"size": 167175,
-"digest": "0b71a936edf5bd70cf274aaa5d7abc8f77fe8e7b5593a208f805cc9436fac646b9c4f0b43c2b10de63ff3da671497d35536077ecbc72dba7f8159a38b580f831",
+"version": "sccache rev b21198a7183a2fe226ff49348b1c0b51bae9f4f8",
 "algorithm": "sha512",
-"filename": "sccache.tar.bz2",
-"unpack": true
+"visibility": "public",
+"filename": "sccache2.tar.bz2",
+"unpack": true,
+"digest": "7dee5c5602b3830cb8ac45ebaa8542714bbac0e50eabbff58a06972a02ceeab75ed7c56ff22a23f760b8317ae8e9a01cdecfaf75a7acbd2a4cdd817967170d2e",
+"size": 1179901
 },
 {
 "version": "clang 3.9.0/r262971",
--- a/browser/config/tooltool-manifests/win32/releng.manifest
+++ b/browser/config/tooltool-manifests/win32/releng.manifest
@ -14,11 +14,13 @@
 "unpack": true
 },
 {
-"size": 167175,
-"digest": "0b71a936edf5bd70cf274aaa5d7abc8f77fe8e7b5593a208f805cc9436fac646b9c4f0b43c2b10de63ff3da671497d35536077ecbc72dba7f8159a38b580f831",
+"version": "sccache rev b21198a7183a2fe226ff49348b1c0b51bae9f4f8",
 "algorithm": "sha512",
-"filename": "sccache.tar.bz2",
-"unpack": true
+"visibility": "public",
+"filename": "sccache2.tar.bz2",
+"unpack": true,
+"digest": "7dee5c5602b3830cb8ac45ebaa8542714bbac0e50eabbff58a06972a02ceeab75ed7c56ff22a23f760b8317ae8e9a01cdecfaf75a7acbd2a4cdd817967170d2e",
+"size": 1179901
 },
 {
 "version": "Visual Studio 2015 Update 3 14.0.25425.01 / SDK 10.0.14393.0",
--- a/browser/config/tooltool-manifests/win64/clang.manifest
+++ b/browser/config/tooltool-manifests/win64/clang.manifest
@ -15,11 +15,13 @@
 "unpack": true
 },
 {
-"size": 167175,
-"digest": "0b71a936edf5bd70cf274aaa5d7abc8f77fe8e7b5593a208f805cc9436fac646b9c4f0b43c2b10de63ff3da671497d35536077ecbc72dba7f8159a38b580f831",
+"version": "sccache rev b21198a7183a2fe226ff49348b1c0b51bae9f4f8",
 "algorithm": "sha512",
-"filename": "sccache.tar.bz2",
-"unpack": true
+"visibility": "public",
+"filename": "sccache2.tar.bz2",
+"unpack": true,
+"digest": "7dee5c5602b3830cb8ac45ebaa8542714bbac0e50eabbff58a06972a02ceeab75ed7c56ff22a23f760b8317ae8e9a01cdecfaf75a7acbd2a4cdd817967170d2e",
+"size": 1179901
 },
 {
 "version": "clang 3.9.0/r262971",
--- a/browser/config/tooltool-manifests/win64/releng.manifest
+++ b/browser/config/tooltool-manifests/win64/releng.manifest
@ -15,11 +15,13 @@
 "unpack": true
 },
 {
-"size": 167175,
-"digest": "0b71a936edf5bd70cf274aaa5d7abc8f77fe8e7b5593a208f805cc9436fac646b9c4f0b43c2b10de63ff3da671497d35536077ecbc72dba7f8159a38b580f831",
+"version": "sccache rev b21198a7183a2fe226ff49348b1c0b51bae9f4f8",
 "algorithm": "sha512",
-"filename": "sccache.tar.bz2",
-"unpack": true
+"visibility": "public",
+"filename": "sccache2.tar.bz2",
+"unpack": true,
+"digest": "7dee5c5602b3830cb8ac45ebaa8542714bbac0e50eabbff58a06972a02ceeab75ed7c56ff22a23f760b8317ae8e9a01cdecfaf75a7acbd2a4cdd817967170d2e",
+"size": 1179901
 },
 {
 "version": "Visual Studio 2015 Update 3 14.0.25425.01 / SDK 10.0.14393.0",
--- a/build/moz.configure/toolchain.configure
+++ b/build/moz.configure/toolchain.configure
@ -116,11 +116,29 @@ def ccache(value):

 ccache = check_prog('CCACHE', progs=(), input=ccache)

+# Distinguish ccache from sccache.
@depends_if(ccache)
-def using_ccache(ccache):
+def ccache_is_sccache(ccache):
+    return check_cmd_output(ccache, '--version').startswith('sccache')
+
+@depends(ccache, ccache_is_sccache)
+def using_ccache(ccache, ccache_is_sccache):
+    return ccache and not ccache_is_sccache
+
+@depends_if(ccache, ccache_is_sccache)
+def using_sccache(ccache, ccache_is_sccache):
    return True

 set_config('MOZ_USING_CCACHE', using_ccache)
+set_config('MOZ_USING_SCCACHE', using_sccache)
+
+option(env='SCCACHE_VERBOSE_STATS', help='Print verbose sccache stats after build')
+
+@depends(using_sccache, 'SCCACHE_VERBOSE_STATS')
+def sccache_verbose_stats(using_sccache, verbose_stats):
+    return using_sccache and bool(verbose_stats)
+
+set_config('SCCACHE_VERBOSE_STATS', sccache_verbose_stats)

@depends('--with-compiler-wrapper', ccache)
@imports(_from='mozbuild.shellutil', _import='split', _as='shell_split')
--- a/build/mozconfig.cache
+++ b/build/mozconfig.cache
@ -91,6 +91,7 @@ if test -z "$platform"; then
    case "${SYSTEMROOT}" in
    *Windows)
        platform=windows
+        suffix=.exe
        ;;
    esac
 fi
@ -102,8 +103,8 @@ if test -z "$bucket"; then
        ac_add_options --with-ccache
    esac
 else
-    if ! test -e $topsrcdir/sccache/sccache.py; then
-        echo "Sccache missing in the tooltool manifest" >&2
+    if ! test -e $topsrcdir/sccache2/sccache${suffix}; then
+        echo "sccache2 missing in the tooltool manifest" >&2
        exit 1
    fi
    mk_add_options "export SCCACHE_BUCKET=$bucket"
@ -112,13 +113,15 @@ else
        mk_add_options "export SCCACHE_NAMESERVER=169.254.169.253"
        ;;
    esac
-    ac_add_options "--with-compiler-wrapper=python2.7 $topsrcdir/sccache/sccache.py"
+    ac_add_options "--with-ccache=$topsrcdir/sccache2/sccache${suffix}"
+    export SCCACHE_VERBOSE_STATS=1
    mk_add_options MOZ_PREFLIGHT_ALL+=build/sccache.mk
    mk_add_options MOZ_POSTFLIGHT_ALL+=build/sccache.mk
    mk_add_options "UPLOAD_EXTRA_FILES+=sccache.log.gz"
    case "$platform" in
    win*)
        # sccache supports a special flag to create depfiles.
+        #TODO: bug 1318370 - move this all into toolchain.configure
        export _DEPEND_CFLAGS='-deps$(MDDEPDIR)/$(@F).pp'
        # Windows builds have a default wrapper that needs to be overridden
        mk_add_options "export CC_WRAPPER="
--- a/build/sccache.mk
+++ b/build/sccache.mk
@ -11,8 +11,8 @@ endif

 preflight_all:
 	# Terminate any sccache server that might still be around
-	-python2.7 $(TOPSRCDIR)/sccache/sccache.py > /dev/null 2>&1
+	-$(TOPSRCDIR)/sccache2/sccache --stop-server > /dev/null 2>&1

 postflight_all:
 	# Terminate sccache server. This prints sccache stats.
-	-python2.7 $(TOPSRCDIR)/sccache/sccache.py 2>&1 | gzip > $(BASE_DIR)/dist/sccache.log.gz
+	-$(TOPSRCDIR)/sccache2/sccache --stop-server
--- a/mobile/android/config/tooltool-manifests/android-frontend/releng.manifest
+++ b/mobile/android/config/tooltool-manifests/android-frontend/releng.manifest
@ -15,12 +15,13 @@
 "unpack": true
 },
 {
-"size": 167175,
-"visibility": "public",
-"digest": "0b71a936edf5bd70cf274aaa5d7abc8f77fe8e7b5593a208f805cc9436fac646b9c4f0b43c2b10de63ff3da671497d35536077ecbc72dba7f8159a38b580f831",
+"version": "sccache rev b21198a7183a2fe226ff49348b1c0b51bae9f4f8",
 "algorithm": "sha512",
-"filename": "sccache.tar.bz2",
-"unpack": true
+"visibility": "public",
+"filename": "sccache2.tar.xz",
+"unpack": true,
+"digest": "b89c40dbf28c2bd54fadf017c15a8789f6e7611252a623cc3a1507e3dd6fc9e5a50d746e81776ba856e33fdc99b4a6413ba7c3ac0aed5f4835705da2b758ef22",
+"size": 1020700
 },
 {
 "size": 30899096,
--- a/mobile/android/config/tooltool-manifests/android-gradle-dependencies/releng.manifest
+++ b/mobile/android/config/tooltool-manifests/android-gradle-dependencies/releng.manifest
@ -15,12 +15,13 @@
 "unpack": true
 },
 {
-"size": 167175,
-"visibility": "public",
-"digest": "0b71a936edf5bd70cf274aaa5d7abc8f77fe8e7b5593a208f805cc9436fac646b9c4f0b43c2b10de63ff3da671497d35536077ecbc72dba7f8159a38b580f831",
+"version": "sccache rev b21198a7183a2fe226ff49348b1c0b51bae9f4f8",
 "algorithm": "sha512",
-"filename": "sccache.tar.bz2",
-"unpack": true
+"visibility": "public",
+"filename": "sccache2.tar.xz",
+"unpack": true,
+"digest": "b89c40dbf28c2bd54fadf017c15a8789f6e7611252a623cc3a1507e3dd6fc9e5a50d746e81776ba856e33fdc99b4a6413ba7c3ac0aed5f4835705da2b758ef22",
+"size": 1020700
 },
 {
 "size": 30899096,
--- a/mobile/android/config/tooltool-manifests/android-x86/releng.manifest
+++ b/mobile/android/config/tooltool-manifests/android-x86/releng.manifest
@ -24,11 +24,13 @@
 "unpack": true
 },
 {
-"size": 167175,
-"digest": "0b71a936edf5bd70cf274aaa5d7abc8f77fe8e7b5593a208f805cc9436fac646b9c4f0b43c2b10de63ff3da671497d35536077ecbc72dba7f8159a38b580f831",
+"version": "sccache rev b21198a7183a2fe226ff49348b1c0b51bae9f4f8",
 "algorithm": "sha512",
-"filename": "sccache.tar.bz2",
-"unpack": true
+"visibility": "public",
+"filename": "sccache2.tar.xz",
+"unpack": true,
+"digest": "b89c40dbf28c2bd54fadf017c15a8789f6e7611252a623cc3a1507e3dd6fc9e5a50d746e81776ba856e33fdc99b4a6413ba7c3ac0aed5f4835705da2b758ef22",
+"size": 1020700
 },
 {
 "size": 4906080,
--- a/mobile/android/config/tooltool-manifests/android/releng.manifest
+++ b/mobile/android/config/tooltool-manifests/android/releng.manifest
@ -24,12 +24,13 @@
 "unpack": true
 },
 {
-"size": 167175,
-"visibility": "public",
-"digest": "0b71a936edf5bd70cf274aaa5d7abc8f77fe8e7b5593a208f805cc9436fac646b9c4f0b43c2b10de63ff3da671497d35536077ecbc72dba7f8159a38b580f831",
+"version": "sccache rev b21198a7183a2fe226ff49348b1c0b51bae9f4f8",
 "algorithm": "sha512",
-"filename": "sccache.tar.bz2",
-"unpack": true
+"visibility": "public",
+"filename": "sccache2.tar.xz",
+"unpack": true,
+"digest": "b89c40dbf28c2bd54fadf017c15a8789f6e7611252a623cc3a1507e3dd6fc9e5a50d746e81776ba856e33fdc99b4a6413ba7c3ac0aed5f4835705da2b758ef22",
+"size": 1020700
 },
 {
 "size": 4906080,
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@ -509,7 +509,7 @@ class Build(MachCommandBase):
            # to avoid accidentally disclosing PII.
            telemetry_data['substs'] = {}
            try:
-                for key in ['MOZ_ARTIFACT_BUILDS', 'MOZ_USING_CCACHE']:
+                for key in ['MOZ_ARTIFACT_BUILDS', 'MOZ_USING_CCACHE', 'MOZ_USING_SCCACHE']:
                    value = self.substs.get(key, False)
                    telemetry_data['substs'][key] = value
            except BuildEnvironmentNotFoundException: