зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1250656 - Don't block mach command completion when submitting build telemetry data r=gps
This spins up a separate process to submit telemetry data rather than blocking the execution the current mach command. Although the initial Python process needs to wait for the second process to complete prior to exiting, it releases control of the console once it finishes executing Python code, so from the user's perspective, mahc command completion is not blocked by submitting telemetry data. MozReview-Commit-ID: FlKDYd6rNPc --HG-- extra : rebase_source : d10f3f2cdfe6e8c8cdd8b1e02ce9261178b528e6
This commit is contained in:
Родитель
31635f4f9a
Коммит
6b4f6f99fb
|
@ -9,8 +9,8 @@ import json
|
|||
import os
|
||||
import platform
|
||||
import random
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
import __builtin__
|
||||
|
||||
|
@ -189,10 +189,6 @@ CATEGORIES = {
|
|||
}
|
||||
|
||||
|
||||
# Server to which to submit telemetry data
|
||||
BUILD_TELEMETRY_SERVER = 'http://52.88.27.118/build-metrics-dev'
|
||||
|
||||
|
||||
# We submit data to telemetry approximately every this many mach invocations
|
||||
TELEMETRY_SUBMISSION_FREQUENCY = 10
|
||||
|
||||
|
@ -344,6 +340,11 @@ def bootstrap(topsrcdir, mozilla_dir=None):
|
|||
if should_skip_dispatch(context, handler):
|
||||
return
|
||||
|
||||
# We call mach environment in client.mk which would cause the
|
||||
# data submission below to block the forward progress of make.
|
||||
if handler.name in ('environment'):
|
||||
return
|
||||
|
||||
# We have not opted-in to telemetry
|
||||
if 'BUILD_SYSTEM_TELEMETRY' not in os.environ:
|
||||
return
|
||||
|
@ -352,53 +353,12 @@ def bootstrap(topsrcdir, mozilla_dir=None):
|
|||
if random.randint(1, TELEMETRY_SUBMISSION_FREQUENCY) != 1:
|
||||
return
|
||||
|
||||
# No data to work with anyway
|
||||
outgoing = os.path.join(get_state_dir()[0], 'telemetry', 'outgoing')
|
||||
if not os.path.isdir(outgoing):
|
||||
return
|
||||
|
||||
# We can't import requests until after it has been added during the
|
||||
# bootstrapping below.
|
||||
import requests
|
||||
|
||||
submitted = os.path.join(get_state_dir()[0], 'telemetry', 'submitted')
|
||||
try:
|
||||
os.mkdir(submitted)
|
||||
except OSError as e:
|
||||
if e.errno != errno.EEXIST:
|
||||
raise
|
||||
|
||||
session = requests.Session()
|
||||
for filename in os.listdir(outgoing):
|
||||
path = os.path.join(outgoing, filename)
|
||||
if os.path.isdir(path) or not path.endswith('.json'):
|
||||
continue
|
||||
with open(path, 'r') as f:
|
||||
data = f.read()
|
||||
try:
|
||||
r = session.post(BUILD_TELEMETRY_SERVER, data=data,
|
||||
headers={'Content-Type': 'application/json'})
|
||||
except Exception as e:
|
||||
print('Exception posting to telemetry server: %s' % str(e))
|
||||
break
|
||||
# TODO: some of these errors are likely not recoverable, as
|
||||
# written, we'll retry indefinitely
|
||||
if r.status_code != 200:
|
||||
print('Error posting to telemetry: %s %s' %
|
||||
(r.status_code, r.text))
|
||||
continue
|
||||
|
||||
os.rename(os.path.join(outgoing, filename),
|
||||
os.path.join(submitted, filename))
|
||||
|
||||
session.close()
|
||||
|
||||
# Discard submitted data that is >= 30 days old
|
||||
now = time.time()
|
||||
for filename in os.listdir(submitted):
|
||||
ctime = os.stat(os.path.join(submitted, filename)).st_ctime
|
||||
if now - ctime >= 60*60*24*30:
|
||||
os.remove(os.path.join(submitted, filename))
|
||||
with open(os.devnull, 'wb') as devnull:
|
||||
subprocess.Popen([sys.executable,
|
||||
os.path.join(topsrcdir, 'build',
|
||||
'submit_telemetry_data.py'),
|
||||
get_state_dir()[0]],
|
||||
stdout=devnull, stderr=devnull)
|
||||
|
||||
def populate_context(context, key=None):
|
||||
if key is None:
|
||||
|
|
|
@ -0,0 +1,77 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
import errno
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
HERE = os.path.abspath(os.path.dirname(__file__))
|
||||
sys.path.append(os.path.join(HERE, '..', 'python', 'requests'))
|
||||
import requests
|
||||
|
||||
|
||||
# Server to which to submit telemetry data
|
||||
BUILD_TELEMETRY_SERVER = 'http://52.88.27.118/build-metrics-dev'
|
||||
|
||||
|
||||
def submit_telemetry_data(statedir):
|
||||
|
||||
# No data to work with anyway
|
||||
outgoing = os.path.join(statedir, 'telemetry', 'outgoing')
|
||||
if not os.path.isdir(outgoing):
|
||||
return 0
|
||||
|
||||
submitted = os.path.join(statedir, 'telemetry', 'submitted')
|
||||
try:
|
||||
os.mkdir(submitted)
|
||||
except OSError as e:
|
||||
if e.errno != errno.EEXIST:
|
||||
raise
|
||||
|
||||
session = requests.Session()
|
||||
for filename in os.listdir(outgoing):
|
||||
path = os.path.join(outgoing, filename)
|
||||
if os.path.isdir(path) or not path.endswith('.json'):
|
||||
continue
|
||||
with open(path, 'r') as f:
|
||||
data = f.read()
|
||||
try:
|
||||
r = session.post(BUILD_TELEMETRY_SERVER, data=data,
|
||||
headers={'Content-Type': 'application/json'})
|
||||
except Exception as e:
|
||||
logging.error('Exception posting to telemetry '
|
||||
'server: %s' % str(e))
|
||||
break
|
||||
# TODO: some of these errors are likely not recoverable, as
|
||||
# written, we'll retry indefinitely
|
||||
if r.status_code != 200:
|
||||
logging.error('Error posting to telemetry: %s %s' %
|
||||
(r.status_code, r.text))
|
||||
continue
|
||||
|
||||
os.rename(os.path.join(outgoing, filename),
|
||||
os.path.join(submitted, filename))
|
||||
|
||||
session.close()
|
||||
|
||||
# Discard submitted data that is >= 30 days old
|
||||
now = time.time()
|
||||
for filename in os.listdir(submitted):
|
||||
ctime = os.stat(os.path.join(submitted, filename)).st_ctime
|
||||
if now - ctime >= 60*60*24*30:
|
||||
os.remove(os.path.join(submitted, filename))
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) != 2:
|
||||
print('usage: python submit_telemetry_data.py <statedir>')
|
||||
sys.exit(1)
|
||||
statedir = sys.argv[1]
|
||||
logging.basicConfig(filename=os.path.join(statedir, 'telemetry', 'telemetry.log'),
|
||||
format='%(asctime)s %(message)s')
|
||||
sys.exit(submit_telemetry_data(statedir))
|
Загрузка…
Ссылка в новой задаче