bug 1505325 - add a basic test for build telemetry. r=firefox-build-system-reviewers,chmanchester

This change adds python/mach/mach/test/test_telemetry.py which contains
a simple test that running mach with the build telemetry setting enabled
causes us to write a telemetry file. The test fixture in the file should
make it easy to write additional tests.

A necessary precursor to make the tests work was to change mach_bootstrap's
`should_skip_dispatch` function, which would refuse to write telemetry data
if stdout was not a terminal (which it isn't in the tests) and also in
automation. The latter test is moved to ensure that we don't *submit*
telemetry data from automation, but we can still write it to disk. Machines
in automation should never have the telemetry setting enabled outside of
these tests anyway, so this should not change anything in practice.

Differential Revision: https://phabricator.services.mozilla.com/D11172

--HG--
extra : moz-landing-system : lando

build/mach_bootstrap.py

@@ -231,14 +231,6 @@ def bootstrap(topsrcdir, mozilla_dir=None):
                             'environment'):
             return True
 
-        # We are running in automation.
-        if 'MOZ_AUTOMATION' in os.environ or 'TASK_ID' in os.environ:
-            return True
-
-        # The environment is likely a machine invocation.
-        if sys.stdin.closed or not sys.stdin.isatty():
-            return True
-
         return False
 
     def post_dispatch_handler(context, handler, instance, result,
@@ -273,6 +265,10 @@ def bootstrap(topsrcdir, mozilla_dir=None):
                          mach_context=context, substs=substs,
                          paths=[instance.topsrcdir, instance.topobjdir])
 
+        # Never submit data when running in automation.
+        if 'MOZ_AUTOMATION' in os.environ or 'TASK_ID' in os.environ:
+            return True
+
         # But only submit about every n-th operation
         if random.randint(1, TELEMETRY_SUBMISSION_FREQUENCY) != 1:
             return

python/mach/mach/test/python.ini

@@ -7,3 +7,4 @@ skip-if = python == 3
 [test_entry_point.py]
 [test_error_output.py]
 [test_logger.py]
+[test_telemetry.py]

python/mach/mach/test/test_telemetry.py

@@ -0,0 +1,57 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+from __future__ import absolute_import
+
+import json
+import os
+import subprocess
+import sys
+
+import mozunit
+import pytest
+
+from mozboot.bootstrap import update_or_create_build_telemetry_config
+
+
+@pytest.fixture
+def run_mach(tmpdir):
+    """Return a function that runs mach with the provided arguments and then returns
+    a list of the data contained within any telemetry entries generated during the command.
+    """
+    # Use tmpdir as the mozbuild state path, and enable telemetry in
+    # a machrc there.
+    update_or_create_build_telemetry_config(unicode(tmpdir.join('machrc')))
+    env = dict(os.environ)
+    env['MOZBUILD_STATE_PATH'] = str(tmpdir)
+    mach = os.path.normpath(os.path.join(os.path.dirname(__file__),
+                                         '../../../../mach'))
+
+    def run(*args, **kwargs):
+        # Run mach with the provided arguments
+        subprocess.check_output([sys.executable, mach] + list(args),
+                                stderr=subprocess.STDOUT,
+                                env=env,
+                                **kwargs)
+        # Load any telemetry data that was written
+        path = unicode(tmpdir.join('telemetry', 'outgoing'))
+        try:
+            return [json.load(open(os.path.join(path, f), 'rb')) for f in os.listdir(path)]
+        except OSError:
+            return []
+    return run
+
+
+def test_simple(run_mach, tmpdir):
+    data = run_mach('python', '-c', 'pass')
+    assert len(data) == 1
+    d = data[0]
+    assert d['command'] == 'python'
+    assert d['argv'] == ['-c', 'pass']
+    client_id_data = json.load(tmpdir.join('telemetry_client_id.json').open('rb'))
+    assert 'client_id' in client_id_data
+    assert client_id_data['client_id'] == d['client_id']
+
+
+if __name__ == '__main__':
+    mozunit.main()