gecko-dev/build/submit_telemetry_data.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from __future__ import print_function

import datetime
import json
import logging
import os
import sys

import requests
import voluptuous
import voluptuous.humanize

from mozbuild.telemetry import (
    schema as build_telemetry_schema,
    verify_statedir,
)

BUILD_TELEMETRY_URL = "https://incoming.telemetry.mozilla.org/{endpoint}"
SUBMIT_ENDPOINT = "submit/eng-workflow/build/1/{ping_uuid}"
STATUS_ENDPOINT = "status"


def delete_expired_files(directory, days=30):
    """Discards files in a directory older than a specified number
    of days
    """
    now = datetime.datetime.now()
    for filename in os.listdir(directory):
        filepath = os.path.join(directory, filename)

        ctime = os.path.getctime(filepath)
        then = datetime.datetime.fromtimestamp(ctime)

        if (now - then) > datetime.timedelta(days=days):
            os.remove(filepath)

    return


def check_edge_server_status(session):
    """Returns True if the Telemetry Edge Server
    is ready to accept data
    """
    status_url = BUILD_TELEMETRY_URL.format(endpoint=STATUS_ENDPOINT)
    response = session.get(status_url)
    if response.status_code != 200:
        return False
    return True


def send_telemetry_ping(session, data, ping_uuid):
    """Sends a single build telemetry ping to the
    edge server, returning the response object
    """
    resource_url = SUBMIT_ENDPOINT.format(ping_uuid=str(ping_uuid))
    url = BUILD_TELEMETRY_URL.format(endpoint=resource_url)
    response = session.post(url, json=data)

    return response


def submit_telemetry_data(outgoing, submitted):
    """Sends information about `./mach build` invocations to
    the Telemetry pipeline
    """
    with requests.Session() as session:
        # Confirm the server is OK
        if not check_edge_server_status(session):
            logging.error('Error posting to telemetry: server status is not "200 OK"')
            return 1

        for filename in os.listdir(outgoing):
            path = os.path.join(outgoing, filename)

            if os.path.isdir(path) or not path.endswith(".json"):
                logging.info("skipping item {}".format(path))
                continue

            ping_uuid = os.path.splitext(filename)[0]  # strip ".json" to get ping UUID

            try:
                with open(path, "r") as f:
                    data = json.load(f)

                # Verify the data matches the schema
                voluptuous.humanize.validate_with_humanized_errors(
                    data, build_telemetry_schema
                )

                response = send_telemetry_ping(session, data, ping_uuid)
                if response.status_code != 200:
                    msg = "response code {code} sending {uuid} to telemetry: {body}".format(
                        body=response.content,
                        code=response.status_code,
                        uuid=ping_uuid,
                    )
                    logging.error(msg)
                    continue

                # Move from "outgoing" to "submitted"
                os.rename(
                    os.path.join(outgoing, filename), os.path.join(submitted, filename)
                )

                logging.info("successfully posted {} to telemetry".format(ping_uuid))

            except ValueError as ve:
                # ValueError is thrown if JSON cannot be decoded
                logging.exception("exception parsing JSON at %s: %s" % (path, str(ve)))
                os.remove(path)

            except voluptuous.Error as e:
                # Invalid is thrown if some data does not fit
                # the correct Schema
                logging.exception("invalid data found at %s: %s" % (path, e.message))
                os.remove(path)

            except Exception as e:
                logging.error("exception posting to telemetry " "server: %s" % str(e))
                break

    delete_expired_files(submitted)

    return 0


if __name__ == "__main__":
    if len(sys.argv) != 2:
        print("usage: python submit_telemetry_data.py <statedir>")
        sys.exit(1)

    statedir = sys.argv[1]

    try:
        outgoing, submitted, telemetry_log = verify_statedir(statedir)

        # Configure logging
        logging.basicConfig(
            filename=telemetry_log,
            format="%(asctime)s %(message)s",
            level=logging.DEBUG,
        )

        sys.exit(submit_telemetry_data(outgoing, submitted))

    except Exception as e:
        # Handle and print messages from `statedir` verification
        print(e.message)
        sys.exit(1)