# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import print_function import datetime import json import logging import os import sys import requests import voluptuous import voluptuous.humanize from mozbuild.telemetry import ( schema as build_telemetry_schema, verify_statedir, ) BUILD_TELEMETRY_URL = 'https://incoming.telemetry.mozilla.org/{endpoint}' SUBMIT_ENDPOINT = 'submit/eng-workflow/build/1/{ping_uuid}' STATUS_ENDPOINT = 'status' def delete_expired_files(directory, days=30): '''Discards files in a directory older than a specified number of days ''' now = datetime.datetime.now() for filename in os.listdir(directory): filepath = os.path.join(directory, filename) ctime = os.path.getctime(filepath) then = datetime.datetime.fromtimestamp(ctime) if (now - then) > datetime.timedelta(days=days): os.remove(filepath) return def check_edge_server_status(session): '''Returns True if the Telemetry Edge Server is ready to accept data ''' status_url = BUILD_TELEMETRY_URL.format(endpoint=STATUS_ENDPOINT) response = session.get(status_url) if response.status_code != 200: return False return True def send_telemetry_ping(session, data, ping_uuid): '''Sends a single build telemetry ping to the edge server, returning the response object ''' resource_url = SUBMIT_ENDPOINT.format(ping_uuid=str(ping_uuid)) url = BUILD_TELEMETRY_URL.format(endpoint=resource_url) response = session.post(url, json=data) return response def submit_telemetry_data(outgoing, submitted): '''Sends information about `./mach build` invocations to the Telemetry pipeline ''' with requests.Session() as session: # Confirm the server is OK if not check_edge_server_status(session): logging.error('Error posting to telemetry: server status is not "200 OK"') return 1 for filename in os.listdir(outgoing): path = os.path.join(outgoing, filename) if os.path.isdir(path) or not path.endswith('.json'): logging.info('skipping item {}'.format(path)) continue ping_uuid = os.path.splitext(filename)[0] # strip ".json" to get ping UUID try: with open(path, 'r') as f: data = json.load(f) # Verify the data matches the schema voluptuous.humanize.validate_with_humanized_errors( data, build_telemetry_schema ) response = send_telemetry_ping(session, data, ping_uuid) if response.status_code != 200: msg = 'response code {code} sending {uuid} to telemetry: {body}'.format( body=response.content, code=response.status_code, uuid=ping_uuid, ) logging.error(msg) continue # Move from "outgoing" to "submitted" os.rename(os.path.join(outgoing, filename), os.path.join(submitted, filename)) logging.info('successfully posted {} to telemetry'.format(ping_uuid)) except ValueError as ve: # ValueError is thrown if JSON cannot be decoded logging.exception('exception parsing JSON at %s: %s' % (path, str(ve))) os.remove(path) except voluptuous.Error as e: # Invalid is thrown if some data does not fit # the correct Schema logging.exception('invalid data found at %s: %s' % (path, e.message)) os.remove(path) except Exception as e: logging.error('exception posting to telemetry ' 'server: %s' % str(e)) break delete_expired_files(submitted) return 0 if __name__ == '__main__': if len(sys.argv) != 2: print('usage: python submit_telemetry_data.py ') sys.exit(1) statedir = sys.argv[1] try: outgoing, submitted, telemetry_log = verify_statedir(statedir) # Configure logging logging.basicConfig(filename=telemetry_log, format='%(asctime)s %(message)s', level=logging.DEBUG) sys.exit(submit_telemetry_data(outgoing, submitted)) except Exception as e: # Handle and print messages from `statedir` verification print(e.message) sys.exit(1)