bug 1355961 - add some sccache stats to build_metrics. r=gps

This commit makes sccache dump JSON stats at the end of the build, and then
reads them in `BuildScript.generate_build_stats` and adds them to the
build_metrics we submit to Perfherder. The stats dumping is done in
Makefile.in where we currently dump verbose sccache stats because sccache
doesn't persist stats to disk right now and it will also shut down its server
process after 5 minutes, so when the post-build automation steps take more
than 5 minutes the server shuts down and the stats are lost.

Currently it's collecting:
* Cache hit rate
* Cache write errors
* Non-cacheable requests (compiler invocations that sccache can't cache)

We can always grow this list later.

MozReview-Commit-ID: J9CwU7XB05I

--HG--
extra : rebase_source : 084b09c3b0621330ac331a99b1bca9a15cf833b7

Makefile.in

@@ -238,8 +238,8 @@ default all::
 include $(topsrcdir)/config/rules.mk
 
 ifdef SCCACHE_VERBOSE_STATS
-# This won't contain stats for both halves of a universal build, but I can live with that.
 default::
+	-$(CCACHE) --show-stats --stats-format=json > sccache-stats.json
 	@echo "===SCCACHE STATS==="
 	-$(CCACHE) --show-stats
 	@echo "==================="

testing/mozharness/mozharness/mozilla/building/buildbase.py

@@ -1884,6 +1884,38 @@ or run without that action (ie: --no-{action})"
 
         return data
 
+
+    def _load_sccache_stats(self):
+        stats_file = os.path.join(
+            self.query_abs_dirs()['abs_obj_dir'], 'sccache-stats.json'
+        )
+        if not os.path.exists(stats_file):
+            self.info('%s does not exist; not loading sccache stats' % stats_file)
+            return
+
+        with open(stats_file, 'rb') as fh:
+            stats = json.load(fh)
+
+        total = stats['stats']['requests_executed']
+        hits = stats['stats']['cache_hits']
+        if total > 0:
+            hits /= float(total)
+
+        yield {
+            'name': 'sccache hit rate',
+            'value': hits,
+            'extraOptions': self.perfherder_resource_options(),
+            'subtests': [],
+        }
+
+        for stat in ['cache_write_errors', 'requests_not_cacheable']:
+            yield {
+                'name': 'sccache %s' % stat,
+                'value': stats['stats'][stat],
+                'extraOptions': self.perfherder_resource_options(),
+                'subtests': [],
+            }
+
     def get_firefox_version(self):
         versionFilePath = os.path.join(
             self.query_abs_dirs()['abs_src_dir'], 'browser/config/version.txt')
@@ -1998,6 +2030,7 @@ or run without that action (ie: --no-{action})"
         build_metrics = self._load_build_resources()
         if build_metrics:
             perfherder_data['suites'].append(build_metrics)
+        perfherder_data['suites'].extend(self._load_sccache_stats())
 
         if self.query_is_nightly():
             for suite in perfherder_data['suites']: