gecko-dev/build/submit_telemetry_data.py

154 строки
4.7 KiB
Python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import print_function
import datetime
import json
import logging
import os
import sys
import requests
import voluptuous
import voluptuous.humanize
from mozbuild.telemetry import (
schema as build_telemetry_schema,
verify_statedir,
)
BUILD_TELEMETRY_URL = "https://incoming.telemetry.mozilla.org/{endpoint}"
SUBMIT_ENDPOINT = "submit/eng-workflow/build/1/{ping_uuid}"
STATUS_ENDPOINT = "status"
def delete_expired_files(directory, days=30):
"""Discards files in a directory older than a specified number
of days
"""
now = datetime.datetime.now()
for filename in os.listdir(directory):
filepath = os.path.join(directory, filename)
ctime = os.path.getctime(filepath)
then = datetime.datetime.fromtimestamp(ctime)
if (now - then) > datetime.timedelta(days=days):
os.remove(filepath)
return
def check_edge_server_status(session):
"""Returns True if the Telemetry Edge Server
is ready to accept data
"""
status_url = BUILD_TELEMETRY_URL.format(endpoint=STATUS_ENDPOINT)
response = session.get(status_url)
if response.status_code != 200:
return False
return True
def send_telemetry_ping(session, data, ping_uuid):
"""Sends a single build telemetry ping to the
edge server, returning the response object
"""
resource_url = SUBMIT_ENDPOINT.format(ping_uuid=str(ping_uuid))
url = BUILD_TELEMETRY_URL.format(endpoint=resource_url)
response = session.post(url, json=data)
return response
def submit_telemetry_data(outgoing, submitted):
"""Sends information about `./mach build` invocations to
the Telemetry pipeline
"""
with requests.Session() as session:
# Confirm the server is OK
if not check_edge_server_status(session):
logging.error('Error posting to telemetry: server status is not "200 OK"')
return 1
for filename in os.listdir(outgoing):
path = os.path.join(outgoing, filename)
if os.path.isdir(path) or not path.endswith(".json"):
logging.info("skipping item {}".format(path))
continue
ping_uuid = os.path.splitext(filename)[0] # strip ".json" to get ping UUID
try:
with open(path, "r") as f:
data = json.load(f)
# Verify the data matches the schema
voluptuous.humanize.validate_with_humanized_errors(
data, build_telemetry_schema
)
response = send_telemetry_ping(session, data, ping_uuid)
if response.status_code != 200:
msg = "response code {code} sending {uuid} to telemetry: {body}".format(
body=response.content,
code=response.status_code,
uuid=ping_uuid,
)
logging.error(msg)
continue
# Move from "outgoing" to "submitted"
os.rename(
os.path.join(outgoing, filename), os.path.join(submitted, filename)
)
logging.info("successfully posted {} to telemetry".format(ping_uuid))
except ValueError as ve:
# ValueError is thrown if JSON cannot be decoded
logging.exception("exception parsing JSON at %s: %s" % (path, str(ve)))
os.remove(path)
except voluptuous.Error as e:
# Invalid is thrown if some data does not fit
# the correct Schema
logging.exception("invalid data found at %s: %s" % (path, e.message))
os.remove(path)
except Exception as e:
logging.error("exception posting to telemetry " "server: %s" % str(e))
break
delete_expired_files(submitted)
return 0
if __name__ == "__main__":
if len(sys.argv) != 2:
print("usage: python submit_telemetry_data.py <statedir>")
sys.exit(1)
statedir = sys.argv[1]
try:
outgoing, submitted, telemetry_log = verify_statedir(statedir)
# Configure logging
logging.basicConfig(
filename=telemetry_log,
format="%(asctime)s %(message)s",
level=logging.DEBUG,
)
sys.exit(submit_telemetry_data(outgoing, submitted))
except Exception as e:
# Handle and print messages from `statedir` verification
print(e.message)
sys.exit(1)