From 694a88e33bc7cec900af3062d3bc287e4dd4df50 Mon Sep 17 00:00:00 2001 From: Dan Minor Date: Tue, 2 Feb 2016 09:32:49 -0500 Subject: [PATCH] Bug 1244160 - Create json-schema for build telemetry data r=gps This adds a simple schema for build telemetry data. We can make it more restrictive once we have a better feeling for what kind of data we want to submit. This also moves more common data about the system to the telemetry handler. We leave psutil derivied information in the resource usage data as not every system will have psutil installed. MozReview-Commit-ID: CFRq1Ow6AOf --HG-- extra : rebase_source : 3022d8f5d20e3d4f9dc871cf2217a6dad2f22e05 --- build/mach_bootstrap.py | 20 +++++++++++++++- build/telemetry-schema.json | 24 +++++++++++++++++++ .../mozbuild/mozbuild/controller/building.py | 24 +++---------------- .../resources/html-build-viewer/index.html | 2 +- 4 files changed, 47 insertions(+), 23 deletions(-) create mode 100644 build/telemetry-schema.json diff --git a/build/mach_bootstrap.py b/build/mach_bootstrap.py index 325bfacabd36..27e26975d3b7 100644 --- a/build/mach_bootstrap.py +++ b/build/mach_bootstrap.py @@ -254,8 +254,26 @@ def bootstrap(topsrcdir, mozilla_dir=None): raise # Add common metadata to help submit sorted data later on. - # For now, we'll just record the mach command that was invoked. data['argv'] = sys.argv + data.setdefault('system', {}).update(dict( + architecture=list(platform.architecture()), + machine=platform.machine(), + python_version=platform.python_version(), + release=platform.release(), + system=platform.system(), + version=platform.version(), + )) + + if platform.system() == 'Linux': + dist = list(platform.linux_distribution()) + data['system']['linux_distribution'] = dist + elif platform.system() == 'Windows': + win32_ver=list((platform.win32_ver())), + data['system']['win32_ver'] = win32_ver + elif platform.system() == 'Darwin': + # mac version is a special Cupertino snowflake + r, v, m = platform.mac_ver() + data['system']['mac_ver'] = [r, list(v), m] with open(os.path.join(outgoing_dir, str(uuid.uuid4()) + '.json'), 'w') as f: diff --git a/build/telemetry-schema.json b/build/telemetry-schema.json new file mode 100644 index 000000000000..6157bf9d5561 --- /dev/null +++ b/build/telemetry-schema.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "argv": {"type": "array"}, + "system": { + "type": "object", + "properties": { + "architecture": {"type": "array"}, + "linux_distribution": {"type": "array"}, + "mac_ver": {"type": "array"}, + "machine": {"type": "string"}, + "python_version": {"type": "string"}, + "release": {"type": "string"}, + "system": {"type": "string"}, + "version": {"type": "string"}, + "win_ver": {"type": "array"} + }, + "required": ["architecture", "machine", "python_version", + "release", "system", "version"] + } + }, + "required": ["argv", "system"] +} diff --git a/python/mozbuild/mozbuild/controller/building.py b/python/mozbuild/mozbuild/controller/building.py index d9c4b26c7972..f7191eaca761 100644 --- a/python/mozbuild/mozbuild/controller/building.py +++ b/python/mozbuild/mozbuild/controller/building.py @@ -377,7 +377,7 @@ class BuildMonitor(MozbuildObject): 'Swap in/out (MB): {sin}/{sout}') o = dict( - version=2, + version=3, argv=sys.argv, start=self.start_time, end=self.end_time, @@ -411,22 +411,15 @@ class BuildMonitor(MozbuildObject): o['resources'].append(entry) - # TODO: it would be nice to collect data on the storage device as well - o['system'] = dict( - architecture=list(platform.architecture()), - machine=platform.machine(), - python_version=platform.python_version(), - release=platform.release(), - system=platform.system(), - version=platform.version(), - ) # If the imports for this file ran before the in-tree virtualenv # was bootstrapped (for instance, for a clobber build in automation), # psutil might not be available. # # Treat psutil as optional to avoid an outright failure to log resources + # TODO: it would be nice to collect data on the storage device as well # in this case. + o['system'] = {} if psutil: o['system'].update(dict( logical_cpu_count=psutil.cpu_count(), @@ -435,17 +428,6 @@ class BuildMonitor(MozbuildObject): vmem_total=psutil.virtual_memory()[0], )) - if platform.system() == 'Linux': - dist = list(platform.linux_distribution()) - o['system']['linux_distribution'] = dist - elif platform.system() == 'Windows': - win32_ver=list((platform.win32_ver())), - o['system']['win32_ver'] = win32_ver - elif platform.system() == 'Darwin': - # mac version is a special Cupertino snowflake - r, v, m = platform.mac_ver() - o['system']['mac_ver'] = [r, list(v), m] - return o def _log_resource_usage(self, prefix, m_type, duration, cpu_percent, diff --git a/python/mozbuild/mozbuild/resources/html-build-viewer/index.html b/python/mozbuild/mozbuild/resources/html-build-viewer/index.html index 026cdbbc0402..290098112ef3 100644 --- a/python/mozbuild/mozbuild/resources/html-build-viewer/index.html +++ b/python/mozbuild/mozbuild/resources/html-build-viewer/index.html @@ -56,7 +56,7 @@ var currentResources; * Interface for a build resources JSON file. */ function BuildResources(data) { - if (data.version != 1 && data.version != 2) { + if (data.version < 1 || data.version > 3) { throw new Error("Unsupported version of the JSON format: " + data.version); }