gecko-dev/testing/runtimes/writeruntimes

#!/bin/sh
# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
# vim: set filetype=python:

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

# The beginning of this script is both valid shell and valid python,
# such that the script starts with the shell and is reexecuted python
''':'
which mach > /dev/null 2>&1 && exec mach python "$0" "$@" ||
echo "mach not found, either add it to your \$PATH or run this script via ./mach python testing/runtimes/writeruntimes"; exit  # noqa
'''

from __future__ import absolute_import, print_function

import datetime
import json
import os
import sys
import time
from argparse import ArgumentParser
from collections import defaultdict

import requests

from moztest.resolve import (
    TestManifestLoader,
    TestResolver,
    TEST_SUITES,
)

here = os.path.abspath(os.path.dirname(__file__))
ACTIVE_DATA_URL = "https://activedata.allizom.org/query"
EXCEED_LIMIT = [
    # Suites that exceed 10,000 ActiveData result limit will be defined here.
    'web-platform-tests',
]
MAX_TIMEOUT = 180  # seconds


def construct_query(suite, platform):
    if platform in ('windows', 'android'):
        platform_clause = '{"find":{"run.machine.platform": "%s"}}' % platform
    else:
        platform_clause = '''
            {
                "not": {
                    "or": [
                        {"find":{"run.machine.platform": "windows"}},
                        {"find":{"run.machine.platform": "android"}}
                    ]
                }
            }
        '''

    # By default, this clause should be not be set.
    # However, for some web-platform-test suites, results exceed the 10,000 limit.
    # This permits AD to retrieve the full set of results.
    output_clause = ''
    if suite in EXCEED_LIMIT:
        output_clause = '"destination": "url",\n"format": "list",'

    query = """
{
    "from":"unittest",
    "limit":200000,
    "groupby":["result.test"],
    "select":{"value":"result.duration","aggregate":"average"},
    %s
    "where":{"and":[
        {"eq":{"repo.branch.name": "mozilla-central"}},
        {"in":{"result.status": ["OK", "PASS", "FAIL"]}},
        {"gt":{"run.timestamp": {"date": "today-week"}}},
        {"eq":{"run.suite.fullname":"%s"}},
        %s
    ]}
}
""" % (output_clause, suite, platform_clause)

    return query


def query_activedata(suite, platform):
    query = construct_query(suite, platform)

    print("Querying ActiveData for '{}' tests on '{}' platforms.. "
            .format(suite, platform), end='')
    sys.stdout.flush()
    response = requests.post(ACTIVE_DATA_URL,
                             data=query,
                             stream=True)
    response.raise_for_status()

    # Presence of destination clause in the query requires additional processing
    # to produce the dataset that can be used.
    if "destination" in query:
        total_timeout = 0
        sleep = 10

        status_url = response.json()["status"]
        status_url_response = requests.get(status_url).json()
        while total_timeout <= MAX_TIMEOUT and status_url_response["status"] != "done":
            total_timeout += sleep
            time.sleep(sleep)
            status_url_response = requests.get(status_url).json()

        output_url = response.json()["url"]
        raw_data = requests.get(output_url).json()["data"]

        # Data returned from destination is in format of:
        # {data: [result: {test: test_name, duration: duration}]}
        # Normalize it to the format if destination/format was not specified.
        data = dict([[item['result']['test'], item['result']['duration']] for item in raw_data])
    else:
        data = dict(response.json()["data"])

    print("{} found".format(len(data)))
    return data


def write_runtimes(manifest_runtimes, platform, outdir=here):
    if not os.path.exists(outdir):
        os.makedirs(outdir)

    outfilename = os.path.join(outdir, "manifest-runtimes-{}.json".format(platform))
    with open(outfilename, 'w') as f:
        f.write(json.dumps(manifest_runtimes, indent=2, sort_keys=True))


def compute_manifest_runtimes(suites, platform):
    resolver = TestResolver.from_environment(cwd=here, loader_cls=TestManifestLoader)

    crashtest_prefixes = {
        'http': '/tests/',
        'chrome': '/reftest/content/',
        'file': '/reftest/tests/',
    }
    manifest_runtimes = defaultdict(float)
    for suite in suites:
        data = query_activedata(suite, platform)

        if suite == "web-platform-tests":
            wpt_groups = {t["name"]: t["manifest"]
                          for t in resolver.resolve_tests(flavor="web-platform-tests")}

        for path, duration in data.items():
            # Returned data did not contain a test path, so go to next result.
            if not path:
                continue

            if suite in ('reftest', 'crashtest') and ' ' in path:
                path = path.split()[0]

            if suite == 'crashtest' and '://' in path:
                # Crashtest paths are URLs with various schemes and prefixes.
                # Normalize it to become relative to mozilla-central.
                scheme = path[:path.index('://')]
                if ':' in scheme:
                    scheme = scheme.split(':')[-1]
                prefix = crashtest_prefixes[scheme]
                path = path.split(prefix, 1)[-1]
            elif suite == 'xpcshell' and ':' in path:
                path = path.split(':', 1)[-1]

            if suite == "web-platform-tests":
                if path in wpt_groups:
                    manifest_runtimes[wpt_groups[path]] += duration
                continue

            if path not in resolver.tests_by_path:
                continue

            for test in resolver.tests_by_path[path]:
                manifest = test.get('ancestor_manifest') or test['manifest_relpath']
                manifest_runtimes[manifest] += duration

    manifest_runtimes = {k: round(v, 2) for k, v in manifest_runtimes.items()}
    return manifest_runtimes


def cli(args=sys.argv[1:]):
    default_suites = [suite for suite, obj in TEST_SUITES.items() if 'build_flavor' in obj]
    default_platforms = ['android', 'windows', 'unix']

    parser = ArgumentParser()
    parser.add_argument('-o', '--output-directory', dest='outdir', default=here,
                        help="Directory to save runtime data.")
    parser.add_argument('-s', '--suite', dest='suites', action='append',
                        default=None, choices=default_suites,
                        help="Suite(s) to include in the data set (default: all)")
    parser.add_argument('-p', '--platform', dest='platforms', action='append',
                        default=None, choices=default_platforms,
                        help="Platform(s) to gather runtime information on "
                             "(default: all).")
    args = parser.parse_args(args)

    suites = args.suites or default_suites
    platforms = args.platforms or default_platforms
    for platform in platforms:
        runtimes = compute_manifest_runtimes(suites, platform)
        if not runtimes:
            print("Not creating runtimes file for '{}' as no data was found".format(platform))
            continue

        write_runtimes(runtimes, platform, outdir=args.outdir)


if __name__ == "__main__":
    sys.exit(cli())