Bug 1651424: Report build telemetry using Glean r=firefox-build-system-reviewers,Dexter,rstewart

In addition to the existing build telemetry, also gather the stats and
report with Glean. This new telemetry is reported in tandem with the existing
telemetry to allow testing and confidence before a full roll-out.

Additionally, Glean isn't compatible with Python 2, so the new telemetry only runs
on Python 3 mach commands.

Differential Revision: https://phabricator.services.mozilla.com/D83572

build/docs/telemetry.rst

@@ -353,6 +353,15 @@ Time at which this event happened
 :format: ``date-time``
 
 
+Glean Telemetry
+===============
+
+In addition to the existing build-specific telemetry, Mozbuild is also reporting data using
+`Glean <https://mozilla.github.io/glean/>`_ via :ref:`mach_telemetry`.
+The metrics collected are documented :ref:`here<metrics>`.
+As Python 2 is phased out, the old telemetry will be replaced by the new Glean implementation.
+
+
 Error Reporting
 ===============
 

build/mach_bootstrap.py

@@ -2,15 +2,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from __future__ import print_function, unicode_literals
+from __future__ import division, print_function, unicode_literals
 
 import errno
 import json
+import math
 import os
 import platform
 import subprocess
 import sys
 import uuid
+
 if sys.version_info[0] < 3:
     import __builtin__ as builtins
 else:
@@ -282,97 +284,23 @@ def bootstrap(topsrcdir, mozilla_dir=None):
                 # likely automation environment, so do nothing.
                 pass
 
-    def should_skip_telemetry_submission(handler):
-        # The user is performing a maintenance command.
-        if handler.name in (
-                'bootstrap', 'doctor', 'mach-commands', 'vcs-setup',
-                'create-mach-environment',
-                # We call mach environment in client.mk which would cause the
-                # data submission to block the forward progress of make.
-                'environment'):
-            return True
-
-        # Never submit data when running in automation or when running tests.
-        if any(e in os.environ for e in ('MOZ_AUTOMATION', 'TASK_ID', 'MACH_TELEMETRY_NO_SUBMIT')):
-            return True
-
-        return False
-
-    def post_dispatch_handler(context, handler, instance, result,
+    def post_dispatch_handler(context, handler, instance, success,
                               start_time, end_time, depth, args):
         """Perform global operations after command dispatch.
 
 
         For now,  we will use this to handle build system telemetry.
         """
-        # Don't write telemetry data if this mach command was invoked as part of another
-        # mach command.
-        if depth != 1 or os.environ.get('MACH_MAIN_PID') != str(os.getpid()):
+
+        # Don't finalize telemetry data if this mach command was invoked as part of
+        # another mach command.
+        if depth != 1:
             return
 
-        from mozbuild.telemetry import is_telemetry_enabled
-        if not is_telemetry_enabled(context.settings):
-            return
-
-        from mozbuild.telemetry import gather_telemetry
-        from mozbuild.base import MozbuildObject
-        import mozpack.path as mozpath
-
-        if not isinstance(instance, MozbuildObject):
-            instance = MozbuildObject.from_environment()
-
-        try:
-            substs = instance.substs
-        except Exception:
-            substs = {}
-
-        command_attrs = getattr(context, 'command_attrs', {})
-
-        # We gather telemetry for every operation.
-        paths = {
-            instance.topsrcdir: '$topsrcdir/',
-            instance.topobjdir: '$topobjdir/',
-            mozpath.normpath(os.path.expanduser('~')): '$HOME/',
-        }
-        # This might override one of the existing entries, that's OK.
-        # We don't use a sigil here because we treat all arguments as potentially relative
-        # paths, so we'd like to get them back as they were specified.
-        paths[mozpath.normpath(os.getcwd())] = ''
-        data = gather_telemetry(command=handler.name, success=(result == 0),
-                                start_time=start_time, end_time=end_time,
-                                mach_context=context, substs=substs,
-                                command_attrs=command_attrs, paths=paths)
-        if data:
-            telemetry_dir = os.path.join(get_state_dir(), 'telemetry')
-            try:
-                os.mkdir(telemetry_dir)
-            except OSError as e:
-                if e.errno != errno.EEXIST:
-                    raise
-            outgoing_dir = os.path.join(telemetry_dir, 'outgoing')
-            try:
-                os.mkdir(outgoing_dir)
-            except OSError as e:
-                if e.errno != errno.EEXIST:
-                    raise
-
-            with open(os.path.join(outgoing_dir, str(uuid.uuid4()) + '.json'),
-                      'w') as f:
-                json.dump(data, f, sort_keys=True)
-
-        if should_skip_telemetry_submission(handler):
-            return True
-
-        state_dir = get_state_dir()
-
-        machpath = os.path.join(instance.topsrcdir, 'mach')
-        with open(os.devnull, 'wb') as devnull:
-            subprocess.Popen([sys.executable, machpath, 'python',
-                              '--no-virtualenv',
-                              os.path.join(topsrcdir, 'build',
-                                           'submit_telemetry_data.py'),
-                              state_dir],
-                             stdout=devnull, stderr=devnull)
+        _finalize_telemetry_glean(context.telemetry, handler.name == 'bootstrap',
+                                  success)
+        _finalize_telemetry_legacy(context, instance, handler, success, start_time,
+                                   end_time, topsrcdir)
 
     def populate_context(key=None):
         if key is None:
@@ -447,6 +375,107 @@ def bootstrap(topsrcdir, mozilla_dir=None):
     return driver
 
 
+def _finalize_telemetry_legacy(context, instance, handler, success, start_time,
+                               end_time, topsrcdir):
+    """Record and submit legacy telemetry.
+
+    Parameterized by the raw gathered telemetry, this function handles persisting and
+    submission of the data.
+
+    This has been designated as "legacy" telemetry because modern telemetry is being
+    submitted with "Glean".
+    """
+    from mozboot.util import get_state_dir
+    from mozbuild.base import MozbuildObject
+    from mozbuild.telemetry import gather_telemetry
+    from mach.telemetry import (
+        is_telemetry_enabled,
+        is_applicable_telemetry_environment
+    )
+
+    if not (is_applicable_telemetry_environment()
+            and is_telemetry_enabled(context.settings)):
+        return
+
+    if not isinstance(instance, MozbuildObject):
+        instance = MozbuildObject.from_environment()
+
+    command_attrs = getattr(context, 'command_attrs', {})
+
+    # We gather telemetry for every operation.
+    data = gather_telemetry(command=handler.name, success=success,
+                            start_time=start_time, end_time=end_time,
+                            mach_context=context, instance=instance,
+                            command_attrs=command_attrs)
+    if data:
+        telemetry_dir = os.path.join(get_state_dir(), 'telemetry')
+        try:
+            os.mkdir(telemetry_dir)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+        outgoing_dir = os.path.join(telemetry_dir, 'outgoing')
+        try:
+            os.mkdir(outgoing_dir)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+
+        with open(os.path.join(outgoing_dir, str(uuid.uuid4()) + '.json'),
+                  'w') as f:
+            json.dump(data, f, sort_keys=True)
+
+    # The user is performing a maintenance command, skip the upload
+    if handler.name in ('bootstrap', 'doctor', 'mach-commands', 'vcs-setup',
+                        'create-mach-environment',
+                        # We call mach environment in client.mk which would cause the
+                        # data submission to block the forward progress of make.
+                        'environment'):
+        return False
+
+    if 'TEST_MACH_TELEMETRY_NO_SUBMIT' in os.environ:
+        # In our telemetry tests, we want telemetry to be collected for analysis, but
+        # we don't want it submitted.
+        return False
+
+    state_dir = get_state_dir()
+
+    machpath = os.path.join(instance.topsrcdir, 'mach')
+    with open(os.devnull, 'wb') as devnull:
+        subprocess.Popen([sys.executable, machpath, 'python',
+                          '--no-virtualenv',
+                          os.path.join(topsrcdir, 'build',
+                                       'submit_telemetry_data.py'),
+                          state_dir],
+                         stdout=devnull, stderr=devnull)
+
+
+def _finalize_telemetry_glean(telemetry, is_bootstrap, success):
+    """Submit telemetry collected by Glean.
+
+    Finalizes some metrics (command success state and duration, system information) and
+    requests Glean to send the collected data.
+    """
+
+    from mozbuild.telemetry import get_cpu_brand, get_psutil_stats
+
+    system_metrics = telemetry.metrics.mach.system
+    system_metrics.cpu_brand.set(get_cpu_brand())
+
+    has_psutil, logical_cores, physical_cores, memory_total = get_psutil_stats()
+    if has_psutil:
+        # psutil may not be available if a successful build hasn't occurred yet.
+        system_metrics.logical_cores.add(logical_cores)
+        system_metrics.physical_cores.add(physical_cores)
+        if memory_total is not None:
+            system_metrics.memory.accumulate(int(
+                math.ceil(float(memory_total) / (1024 * 1024 * 1024))))
+
+    telemetry.metrics.mach.duration.stop()
+    telemetry.metrics.mach.success.set(success)
+    telemetry.submit(is_bootstrap)
+
+
 # Hook import such that .pyc/.pyo files without a corresponding .py file in
 # the source directory are essentially ignored. See further below for details
 # and caveats.

python/mach/docs/index.rst

@@ -84,3 +84,4 @@ best fit for you.
    driver
    logging
    settings
+   telemetry

python/mach/docs/metrics.md

@@ -0,0 +1,53 @@
+<!-- AUTOGENERATED BY glean_parser.  DO NOT EDIT. -->
+
+# Metrics
+This document enumerates the metrics collected by this project.
+This project may depend on other projects which also collect metrics.
+This means you might have to go searching through the dependency tree to get a full picture of everything collected by this project.
+
+# Pings
+
+ - [usage](#usage)
+
+
+## usage
+
+Sent when the mach invocation is completed (regardless of result). Contains information about the mach invocation that was made, its result, and some details about the current environment and hardware.
+
+
+This ping includes the [client id](https://mozilla.github.io/glean/book/user/pings/index.html#the-client_info-section).
+
+**Data reviews for this ping:**
+
+- <https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34>
+
+**Bugs related to this ping:**
+
+- <https://bugzilla.mozilla.org/show_bug.cgi?id=1291053>
+
+The following metrics are added to the ping:
+
+| Name | Type | Description | Data reviews | Extras | Expiration | [Data Sensitivity](https://wiki.mozilla.org/Firefix/Data_Collection) |
+| --- | --- | --- | --- | --- | --- | --- |
+| mach.argv |[string_list](https://mozilla.github.io/glean/book/user/metrics/string_list.html) |Parameters provided to mach. Absolute paths are sanitized to be relative to one of a few key base paths, such as the "$topsrcdir", "$topobjdir", or "$HOME". For example: "/home/mozilla/dev/firefox/python/mozbuild" would be replaced with "$topsrcdir/python/mozbuild". If a valid replacement base path cannot be found, the path is replaced with "<path omitted>".  |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+| mach.command |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The name of the mach command that was invoked, such as "build", "doc", or "try".  |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+| mach.duration |[timespan](https://mozilla.github.io/glean/book/user/metrics/timespan.html) |How long it took for the command to complete. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+| mach.success |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if the mach invocation succeeded. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+| mach.system.cpu_brand |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |CPU brand string from CPUID. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+| mach.system.logical_cores |[counter](https://mozilla.github.io/glean/book/user/metrics/counter.html) |Number of logical CPU cores present. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+| mach.system.memory |[memory_distribution](https://mozilla.github.io/glean/book/user/metrics/memory_distribution.html) |Amount of system memory. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+| mach.system.physical_cores |[counter](https://mozilla.github.io/glean/book/user/metrics/counter.html) |Number of physical CPU cores present. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+| mozbuild.artifact |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if `--enable-artifact-builds`. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+| mozbuild.ccache |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if `--with-ccache`. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+| mozbuild.clobber |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if the build was a clobber/full build. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1526072#c15)||never | |
+| mozbuild.compiler |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The compiler type in use (CC_TYPE), such as "clang" or "gcc". |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+| mozbuild.debug |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if `--enable-debug`. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+| mozbuild.icecream |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if icecream in use. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+| mozbuild.opt |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if `--enable-optimize`. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+| mozbuild.sccache |[boolean](https://mozilla.github.io/glean/book/user/metrics/boolean.html) |True if ccache in use is sccache. |[1](https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34)||never | |
+
+
+Data categories are [defined here](https://wiki.mozilla.org/Firefox/Data_Collection).
+
+<!-- AUTOGENERATED BY glean_parser.  DO NOT EDIT. -->
+

python/mach/docs/telemetry.rst

@@ -0,0 +1,22 @@
+.. _mach_telemetry:
+
+==============
+Mach Telemetry
+==============
+
+`Glean <https://mozilla.github.io/glean/>`_ is used to collect telemetry, and uses the metrics
+defined in ``/python/mach/metrics.yaml``.
+Associated generated documentation can be found in :ref:`the metrics document<metrics>`.
+
+.. toctree::
+   :maxdepth: 1
+
+   metrics
+
+Updating Generated Metrics Docs
+===============================
+
+When ``metrics.yaml`` is changed, :ref:`the metrics document<metrics>` will need to be updated.
+Glean provides doc-generation tooling for us::
+
+    glean_parser translate -f markdown -o python/mach/docs/ python/mach/metrics.yaml python/mach/pings.yaml

python/mach/mach/base.py

@@ -4,14 +4,18 @@
 
 from __future__ import absolute_import, unicode_literals
 
+from mach.telemetry import Telemetry
+
 
 class CommandContext(object):
     """Holds run-time state so it can easily be passed to command providers."""
-    def __init__(self, cwd=None, settings=None, log_manager=None, commands=None, **kwargs):
+    def __init__(self, cwd=None, settings=None, log_manager=None, commands=None,
+                 telemetry=Telemetry.as_noop(), **kwargs):
         self.cwd = cwd
         self.settings = settings
         self.log_manager = log_manager
         self.commands = commands
+        self.telemetry = telemetry
         self.command_attrs = {}
 
         for k, v in kwargs.items():

python/mach/mach/logging.py

@@ -296,6 +296,10 @@ class LoggingManager(object):
         ``terminal`` and ``json`` determine which log handlers to operate
         on. By default, all known handlers are operated on.
         """
+
+        # Glean makes logs that we're not interested in, so we squelch them.
+        logging.getLogger('glean').setLevel(logging.CRITICAL)
+
         # Remove current handlers from all loggers so we don't double
         # register handlers.
         for logger in self.root_logger.manager.loggerDict.values():

python/mach/mach/main.py

@@ -34,6 +34,7 @@ from .dispatcher import CommandAction
 from .logging import LoggingManager
 from .registrar import Registrar
 from .sentry import register_sentry, NoopErrorReporter
+from .telemetry import report_invocation_metrics, Telemetry
 from .util import setenv, UserError
 
 SUGGEST_MACH_BUSTED_TEMPLATE = r'''
@@ -390,9 +391,10 @@ To see more help for a specific command, run:
             sys.stderr = orig_stderr
 
     def _run(self, argv, sentry):
+        telemetry = Telemetry.from_environment(self.settings)
         context = CommandContext(cwd=self.cwd,
                                  settings=self.settings, log_manager=self.log_manager,
-                                 commands=Registrar)
+                                 commands=Registrar, telemetry=telemetry)
 
         if self.populate_context_handler:
             context = ContextWrapper(context, self.populate_context_handler)
@@ -427,6 +429,7 @@ To see more help for a specific command, run:
             raise MachError('ArgumentParser result missing mach handler info.')
 
         handler = getattr(args, 'mach_handler')
+        report_invocation_metrics(context.telemetry.metrics, handler.name)
 
         # Add JSON logging to a file if requested.
         if args.logfile:

python/mach/mach/registrar.py

@@ -110,7 +110,7 @@ class MachRegistrar(object):
         if not debug_command:
             postrun = getattr(context, 'post_dispatch_handler', None)
             if postrun:
-                postrun(context, handler, instance, result,
+                postrun(context, handler, instance, not result,
                         start_time, end_time, self.command_depth, args=kwargs)
         self.command_depth -= 1
 

python/mach/mach/sentry.py

@@ -10,7 +10,7 @@ from os.path import expanduser
 
 import sentry_sdk
 from mozboot.util import get_state_dir
-from mozbuild.telemetry import is_telemetry_enabled
+from mach.telemetry import is_telemetry_enabled
 from mozversioncontrol import get_repository_object, InvalidRepoPath
 from six import string_types
 

python/mach/mach/telemetry.py

@@ -0,0 +1,110 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function, absolute_import
+
+import os
+import sys
+from mock import Mock
+
+from mozboot.util import get_state_dir
+from mozbuild.base import MozbuildObject, BuildEnvironmentNotFoundException
+from mozbuild.telemetry import filter_args
+
+
+class Telemetry(object):
+    """Records and sends Telemetry using Glean.
+
+    Metrics are defined in python/mozbuild/metrics.yaml.
+    Pings are defined in python/mozbuild/pings.yaml.
+
+    The "metrics" and "pings" properties may be replaced with no-op implementations if
+    Glean isn't available. This allows consumers to report telemetry without having
+    to guard against incompatible environments.
+    """
+    def __init__(self, metrics, pings, failed_glean_import):
+        self.metrics = metrics
+        self._pings = pings
+        self._failed_glean_import = failed_glean_import
+
+    def submit(self, is_bootstrap):
+        self._pings.usage.submit()
+
+        if self._failed_glean_import and not is_bootstrap:
+            print("Glean could not be found, so telemetry will not be reported. "
+                  "You may need to run |mach bootstrap|.", file=sys.stderr)
+
+    @classmethod
+    def as_noop(cls, failed_glean_import=False):
+        return cls(Mock(), Mock(), failed_glean_import)
+
+    @classmethod
+    def from_environment(cls, settings):
+        """Creates and configures a Telemetry instance based on system details.
+
+        If telemetry isn't enabled, the current interpreter isn't Python 3, or Glean
+        can't be imported, then a "mock" telemetry instance is returned that doesn't
+        set or record any data. This allows consumers to optimistically set metrics
+        data without needing to specifically handle the case where the current system
+        doesn't support it.
+        """
+        # Glean is not compatible with Python 2
+        if not (sys.version_info >= (3, 0) and is_applicable_telemetry_environment()):
+            return cls.as_noop()
+
+        try:
+            from glean import Glean, load_metrics, load_pings
+        except ImportError:
+            return cls.as_noop(failed_glean_import=True)
+
+        from pathlib import Path
+
+        Glean.initialize(
+            'mozilla.mach',
+            'Unknown',
+            is_telemetry_enabled(settings),
+            data_dir=Path(get_state_dir()) / 'glean',
+        )
+        from pathlib import Path
+        metrics = load_metrics(Path(__file__).parent.parent / 'metrics.yaml')
+        pings = load_pings(Path(__file__).parent.parent / 'pings.yaml')
+        return cls(metrics, pings, False)
+
+
+def report_invocation_metrics(metrics, command):
+    metrics.mach.command.set(command)
+    metrics.mach.duration.start()
+
+    try:
+        instance = MozbuildObject.from_environment()
+    except BuildEnvironmentNotFoundException:
+        # Mach may be invoked with the state dir as the current working
+        # directory, in which case we're not able to find the topsrcdir (so
+        # we can't create a MozbuildObject instance).
+        # Without this information, we're unable to filter argv paths, so
+        # we skip submitting them to telemetry.
+        return
+    metrics.mach.argv.set(filter_args(command, sys.argv, instance))
+
+
+def is_applicable_telemetry_environment():
+    if os.environ.get('MACH_MAIN_PID') != str(os.getpid()):
+        # This is a child mach process. Since we're collecting telemetry for the parent,
+        # we don't want to collect telemetry again down here.
+        return False
+
+    if any(e in os.environ for e in ('MOZ_AUTOMATION', 'TASK_ID')):
+        return False
+
+    return True
+
+
+def is_telemetry_enabled(settings):
+    if os.environ.get('DISABLE_TELEMETRY') == '1':
+        return False
+
+    try:
+        return settings.build.telemetry
+    except (AttributeError, KeyError):
+        return False

python/mach/mach/test/test_telemetry.py

@@ -37,17 +37,24 @@ def run_mach(tmpdir):
         update_or_create_build_telemetry_config(text_type(tmpdir.join('machrc')))
     env = dict(os.environ)
     env['MOZBUILD_STATE_PATH'] = str(tmpdir)
-    env['MACH_TELEMETRY_NO_SUBMIT'] = '1'
-    # Let whatever mach command we invoke from tests believe it's the main command.
-    del env['MACH_MAIN_PID']
+    env['TEST_MACH_TELEMETRY_NO_SUBMIT'] = '1'
     mach = os.path.join(buildconfig.topsrcdir, 'mach')
 
     def run(*args, **kwargs):
+        # Let whatever mach command we invoke from tests believe it's the main command.
+        mach_main_pid = env.pop('MACH_MAIN_PID')
+        moz_automation = env.pop('MOZ_AUTOMATION', None)
+        task_id = env.pop('TASK_ID', None)
+
         # Run mach with the provided arguments
         out = subprocess.check_output([sys.executable, mach] + list(args),
                                       stderr=subprocess.STDOUT,
                                       env=env,
                                       **kwargs)
+
+        env['MACH_MAIN_PID'] = mach_main_pid
+        env['MOZ_AUTOMATION'] = moz_automation
+        env['TASK_ID'] = task_id
         # Load any telemetry data that was written
         path = tmpdir.join('telemetry', 'outgoing')
         try:

python/mach/metrics.yaml

@@ -0,0 +1,244 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+$schema: moz://mozilla.org/schemas/glean/metrics/1-0-0
+
+mach:
+  command:
+    type: string
+    description: >
+      The name of the mach command that was invoked, such as "build",
+      "doc", or "try".
+    lifetime: application
+    bugs:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
+    notification_emails:
+      - build-telemetry@mozilla.com
+      - mhentges@mozilla.com
+    expires: never
+    send_in_pings:
+      - usage
+  argv:
+    type: string_list
+    description: >
+      Parameters provided to mach. Absolute paths are sanitized to be relative
+      to one of a few key base paths, such as the "$topsrcdir", "$topobjdir",
+      or "$HOME". For example: "/home/mozilla/dev/firefox/python/mozbuild"
+      would be replaced with "$topsrcdir/python/mozbuild".
+      If a valid replacement base path cannot be found, the path is replaced
+      with "<path omitted>".
+    lifetime: application
+    bugs:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
+    notification_emails:
+      - build-telemetry@mozilla.com
+      - mhentges@mozilla.com
+    expires: never
+    send_in_pings:
+      - usage
+  success:
+    type: boolean
+    description: True if the mach invocation succeeded.
+    lifetime: application
+    bugs:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
+    notification_emails:
+      - build-telemetry@mozilla.com
+      - mhentges@mozilla.com
+    expires: never
+    send_in_pings:
+      - usage
+  duration:
+    type: timespan
+    description: How long it took for the command to complete.
+    lifetime: application
+    bugs:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
+    notification_emails:
+      - build-telemetry@mozilla.com
+      - mhentges@mozilla.com
+    expires: never
+    send_in_pings:
+      - usage
+
+mach.system:
+  cpu_brand:
+    type: string
+    description: CPU brand string from CPUID.
+    lifetime: application
+    bugs:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
+    notification_emails:
+      - build-telemetry@mozilla.com
+      - mhentges@mozilla.com
+    expires: never
+    send_in_pings:
+      - usage
+  logical_cores:
+    type: counter
+    description: Number of logical CPU cores present.
+    lifetime: application
+    bugs:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
+    notification_emails:
+      - build-telemetry@mozilla.com
+      - mhentges@mozilla.com
+    expires: never
+    send_in_pings:
+      - usage
+  physical_cores:
+    type: counter
+    description: Number of physical CPU cores present.
+    lifetime: application
+    bugs:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
+    notification_emails:
+      - build-telemetry@mozilla.com
+      - mhentges@mozilla.com
+    expires: never
+    send_in_pings:
+      - usage
+  memory:
+    type: memory_distribution
+    memory_unit: gigabyte
+    description: Amount of system memory.
+    lifetime: application
+    bugs:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
+    notification_emails:
+      - build-telemetry@mozilla.com
+      - mhentges@mozilla.com
+    expires: never
+    send_in_pings:
+      - usage
+
+mozbuild:
+  compiler:
+    type: string
+    description: The compiler type in use (CC_TYPE), such as "clang" or "gcc".
+    lifetime: application
+    bugs:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
+    notification_emails:
+      - build-telemetry@mozilla.com
+      - mhentges@mozilla.com
+    expires: never
+    send_in_pings:
+      - usage
+  artifact:
+    type: boolean
+    description: True if `--enable-artifact-builds`.
+    lifetime: application
+    bugs:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
+    notification_emails:
+      - build-telemetry@mozilla.com
+      - mhentges@mozilla.com
+    expires: never
+    send_in_pings:
+      - usage
+  debug:
+    type: boolean
+    description: True if `--enable-debug`.
+    lifetime: application
+    bugs:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
+    notification_emails:
+      - build-telemetry@mozilla.com
+      - mhentges@mozilla.com
+    expires: never
+    send_in_pings:
+      - usage
+  opt:
+    type: boolean
+    description: True if `--enable-optimize`.
+    lifetime: application
+    bugs:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
+    notification_emails:
+      - build-telemetry@mozilla.com
+      - mhentges@mozilla.com
+    expires: never
+    send_in_pings:
+      - usage
+  ccache:
+    type: boolean
+    description: True if `--with-ccache`.
+    lifetime: application
+    bugs:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
+    notification_emails:
+      - build-telemetry@mozilla.com
+      - mhentges@mozilla.com
+    expires: never
+    send_in_pings:
+      - usage
+  sccache:
+    type: boolean
+    description: True if ccache in use is sccache.
+    lifetime: application
+    bugs:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
+    notification_emails:
+      - build-telemetry@mozilla.com
+      - mhentges@mozilla.com
+    expires: never
+    send_in_pings:
+      - usage
+  icecream:
+    type: boolean
+    description: True if icecream in use.
+    lifetime: application
+    bugs:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
+    notification_emails:
+      - build-telemetry@mozilla.com
+      - mhentges@mozilla.com
+    expires: never
+    send_in_pings:
+      - usage
+  clobber:
+    type: boolean
+    description: True if the build was a clobber/full build.
+    lifetime: application
+    bugs:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1526072
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1526072#c15
+    notification_emails:
+      - build-telemetry@mozilla.com
+      - mhentges@mozilla.com
+    expires: never
+    send_in_pings:
+      - usage

python/mach/pings.yaml

@@ -0,0 +1,19 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+$schema: moz://mozilla.org/schemas/glean/pings/1-0-0
+
+usage:
+  description: >
+    Sent when the mach invocation is completed (regardless of result).
+    Contains information about the mach invocation that was made, its result,
+    and some details about the current environment and hardware.
+  bugs:
+    - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053
+  data_reviews:
+    - https://bugzilla.mozilla.org/show_bug.cgi?id=1291053#c34
+  include_client_id: true
+  notification_emails:
+    - build-telemetry@mozilla.com
+    - mhentges@mozilla.com

python/mozbuild/mozbuild/controller/building.py

@@ -1067,6 +1067,7 @@ class BuildDriver(MozbuildObject):
             monitor.start_resource_recording()
 
             self.mach_context.command_attrs['clobber'] = False
+            self.mach_context.telemetry.metrics.mozbuild.clobber.set(False)
             config = None
             try:
                 config = self.config_environment
@@ -1075,6 +1076,8 @@ class BuildDriver(MozbuildObject):
                 # a fresh objdir or $OBJDIR/config.status has been removed for
                 # some reason, which indicates a clobber of sorts.
                 self.mach_context.command_attrs['clobber'] = True
+                mozbuild_telemetry = self.mach_context.telemetry.metrics.mozbuild
+                mozbuild_telemetry.clobber.set(True)
 
             # Record whether a clobber was requested so we can print
             # a special message later if the build fails.
@@ -1110,6 +1113,21 @@ class BuildDriver(MozbuildObject):
 
                 config = self.reload_config_environment()
 
+            # Collect glean metrics
+            substs = config.substs
+            mozbuild_metrics = self.mach_context.telemetry.metrics.mozbuild
+            mozbuild_metrics.compiler.set(substs.get('CC_TYPE', None))
+
+            def get_substs_flag(name):
+                return bool(substs.get(name, None))
+
+            mozbuild_metrics.artifact.set(get_substs_flag('MOZ_ARTIFACT_BUILDS'))
+            mozbuild_metrics.debug.set(get_substs_flag('MOZ_DEBUG'))
+            mozbuild_metrics.opt.set(get_substs_flag('MOZ_OPTIMIZE'))
+            mozbuild_metrics.ccache.set(get_substs_flag('CCACHE'))
+            mozbuild_metrics.sccache.set(get_substs_flag('MOZ_USING_SCCACHE'))
+            mozbuild_metrics.icecream.set(get_substs_flag('CXX_IS_ICECREAM'))
+
             all_backends = config.substs.get('BUILD_BACKENDS', [None])
             active_backend = all_backends[0]
 
@@ -1581,6 +1599,7 @@ class BuildDriver(MozbuildObject):
         clobber_required, clobber_performed, clobber_message = res
         if self.mach_context is not None and clobber_performed:
             self.mach_context.command_attrs['clobber'] = True
+            self.mach_context.telemetry.metrics.mozbuild.clobber.set(True)
         if not clobber_required or clobber_performed:
             if clobber_performed and env.get('TINDERBOX_OUTPUT'):
                 self.log(logging.WARNING, 'clobber',

python/mozbuild/mozbuild/telemetry.py

@@ -2,7 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from __future__ import absolute_import, print_function, unicode_literals
+from __future__ import division, absolute_import, print_function, unicode_literals
 
 '''
 This file contains a voluptuous schema definition for build system telemetry, and functions
@@ -28,9 +28,7 @@ from voluptuous import (
 from voluptuous.validators import Datetime
 
 import mozpack.path as mozpath
-from .base import (
-    BuildEnvironmentNotFoundException,
-)
+from .base import BuildEnvironmentNotFoundException
 from .configure.constants import CompilerType
 
 schema = Schema({
@@ -173,34 +171,49 @@ def get_cpu_brand():
     }.get(platform.system(), lambda: None)()
 
 
+def get_os_name():
+    return {
+        'Linux': 'linux',
+        'Windows': 'windows',
+        'Darwin': 'macos',
+    }.get(platform.system(), 'other')
+
+
+def get_psutil_stats():
+    '''Return whether psutil exists and its associated stats.
+
+    @returns (bool, int, int, int) whether psutil exists, the logical CPU count,
+        physical CPU count, and total number of bytes of memory.
+    '''
+    try:
+        import psutil
+
+        return (
+            True,
+            psutil.cpu_count(),
+            psutil.cpu_count(logical=False),
+            psutil.virtual_memory().total)
+    except ImportError:
+        return False, None, None, None
+
+
 def get_system_info():
     '''
     Gather info to fill the `system` keys in the schema.
     '''
     # Normalize OS names a bit, and bucket non-tier-1 platforms into "other".
+    has_psutil, logical_cores, physical_cores, memory_total = get_psutil_stats()
     info = {
-        'os': {
-            'Linux': 'linux',
-            'Windows': 'windows',
-            'Darwin': 'macos',
-        }.get(platform.system(), 'other')
+        'os': get_os_name(),
     }
-    try:
-        import psutil
-
-        info['logical_cores'] = psutil.cpu_count()
-        physical_cores = psutil.cpu_count(logical=False)
+    if has_psutil:
+        # `total` on Linux is gathered from /proc/meminfo's `MemTotal`, which is the
+        # total amount of physical memory minus some kernel usage, so round up to the
+        # nearest GB to get a sensible answer.
+        info['memory_gb'] = int(math.ceil(float(memory_total) / (1024 * 1024 * 1024)))
+        info['logical_cores'] = logical_cores
         if physical_cores is not None:
             info['physical_cores'] = physical_cores
-        # `total` on Linux is gathered from /proc/meminfo's `MemTotal`, which is the total
-        # amount of physical memory minus some kernel usage, so round up to the nearest GB
-        # to get a sensible answer.
-        info['memory_gb'] = int(
-            math.ceil(float(psutil.virtual_memory().total) / (1024 * 1024 * 1024)))
-    except ImportError:
-        # TODO: sort out psutil availability on Windows, or write a fallback impl for Windows.
-        # https://bugzilla.mozilla.org/show_bug.cgi?id=1481612
-        pass
     cpu_brand = get_cpu_brand()
     if cpu_brand is not None:
         info['cpu_brand'] = cpu_brand
@@ -249,14 +262,23 @@ def get_build_attrs(attrs):
     return res
 
 
-def filter_args(command, argv, paths):
+def filter_args(command, argv, instance):
     '''
     Given the full list of command-line arguments, remove anything up to and including `command`,
     and attempt to filter absolute pathnames out of any arguments after that.
-
-    `paths` is a dict whose keys are pathnames and values are sigils that should be used to
-    replace those pathnames.
     '''
+
+    # Each key is a pathname and the values are replacement sigils
+    paths = {
+        instance.topsrcdir: '$topsrcdir/',
+        instance.topobjdir: '$topobjdir/',
+        mozpath.normpath(os.path.expanduser('~')): '$HOME/',
+        # This might override one of the existing entries, that's OK.
+        # We don't use a sigil here because we treat all arguments as potentially relative
+        # paths, so we'd like to get them back as they were specified.
+        mozpath.normpath(os.getcwd()): '',
+    }
+
     args = list(argv)
     while args:
         a = args.pop(0)
@@ -273,24 +295,26 @@ def filter_args(command, argv, paths):
     return [filter_path(arg) for arg in args]
 
 
-def gather_telemetry(command='', success=False, start_time=None, end_time=None,
-                     mach_context=None, substs={}, paths={}, command_attrs=None):
+def gather_telemetry(command, success, start_time, end_time, mach_context,
+                     instance, command_attrs):
     '''
     Gather telemetry about the build and the user's system and pass it to the telemetry
     handler to be stored for later submission.
 
-    `paths` is a dict whose keys are pathnames and values are sigils that should be used to
-    replace those pathnames.
-
-    Any absolute paths on the command line will be made relative to `paths` or replaced
-    with a placeholder to avoid including paths from developer's machines.
+    Any absolute paths on the command line will be made relative to a relevant base path
+    or replaced with a placeholder to avoid including paths from developer's machines.
     '''
+    try:
+        substs = instance.substs
+    except BuildEnvironmentNotFoundException:
+        substs = {}
+
     data = {
         'client_id': get_client_id(mach_context.state_dir),
         # Get an rfc3339 datetime string.
         'time': datetime.utcfromtimestamp(start_time).strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
         'command': command,
-        'argv': filter_args(command, sys.argv, paths),
+        'argv': filter_args(command, sys.argv, instance),
         'success': success,
         # TODO: use a monotonic clock: https://bugzilla.mozilla.org/show_bug.cgi?id=1481624
         'duration_ms': int((end_time - start_time) * 1000),
@@ -340,14 +364,3 @@ def verify_statedir(statedir):
         os.mkdir(submitted)
 
     return outgoing, submitted, telemetry_log
-
-
-def is_telemetry_enabled(settings):
-    # Don't write telemetry data for 'mach' when 'DISABLE_TELEMETRY' is set.
-    if os.environ.get('DISABLE_TELEMETRY') == '1':
-        return False
-
-    try:
-        return settings.build.telemetry
-    except (AttributeError, KeyError):
-        return False