478 строки
15 KiB
Python
478 строки
15 KiB
Python
# -*- coding: utf-8 -*-
|
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
|
# You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
import functools
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import shutil
|
|
import sys
|
|
from concurrent.futures import TimeoutError
|
|
from datetime import datetime, timedelta
|
|
|
|
import libmozdata.socorro as socorro
|
|
import libmozdata.utils as utils
|
|
import requests
|
|
from libmozdata.bugzilla import Bugzilla
|
|
from libmozdata.connection import Connection, Query
|
|
from requests.adapters import HTTPAdapter
|
|
from requests.packages.urllib3.util.retry import Retry
|
|
|
|
import gfx_critical_errors
|
|
import versions_util
|
|
|
|
v = versions_util.get(base=True)
|
|
|
|
# http://bugs.python.org/issue7980
|
|
datetime.strptime("", "")
|
|
|
|
|
|
def __trend_handler(default_trend, json, data):
|
|
for facets in json["facets"]["histogram_date"]:
|
|
d = utils.as_utc(datetime.strptime(facets["term"], "%Y-%m-%dT00:00:00+00:00"))
|
|
s = facets["facets"]["signature"]
|
|
for signature in s:
|
|
count = signature["count"]
|
|
sgn = signature["term"]
|
|
if sgn in data:
|
|
data[sgn][d] = count
|
|
else:
|
|
trend = default_trend.copy()
|
|
trend[d] = count
|
|
data[sgn] = trend
|
|
|
|
|
|
def __bug_handler(json, data):
|
|
for bug in json["bugs"]:
|
|
data.append(bug)
|
|
|
|
|
|
def get(
|
|
channel,
|
|
date,
|
|
product="Firefox",
|
|
duration=11,
|
|
tc_limit=50,
|
|
crash_type="all",
|
|
startup=False,
|
|
):
|
|
"""Get crashes info
|
|
|
|
Args:
|
|
channel (str): the channel
|
|
date (str): the final date
|
|
product (Optional[str]): the product
|
|
duration (Optional[int]): the duration to retrieve the data
|
|
tc_limit (Optional[int]): the number of topcrashes to load
|
|
crash_type (Optional[str]): 'all' (default) or 'browser' or 'content' or 'plugin'
|
|
|
|
Returns:
|
|
dict: contains all the info relative to the crashes
|
|
"""
|
|
channel = channel.lower()
|
|
version = v[channel]
|
|
sys.stdout.write("Getting version information from Socorro...")
|
|
sys.stdout.flush()
|
|
versions = versions_util.get_channel_versions(channel, product)
|
|
sys.stdout.write(" ✔\n")
|
|
sys.stdout.flush()
|
|
|
|
if crash_type and isinstance(crash_type, str):
|
|
crash_type = [crash_type]
|
|
|
|
_date = utils.get_date_ymd(date)
|
|
start_date = utils.get_date_str(_date - timedelta(duration - 1))
|
|
end_date = utils.get_date_str(_date)
|
|
|
|
signatures = {}
|
|
|
|
def signature_handler(json):
|
|
for signature in json["facets"]["signature"]:
|
|
signatures[signature["term"]] = [signature["count"], 0, 0, 0, 0, 0]
|
|
|
|
for platform in signature["facets"]["platform"]:
|
|
if platform["term"] == "Linux":
|
|
signatures[signature["term"]][3] = platform["count"]
|
|
elif platform["term"] == "Windows NT":
|
|
signatures[signature["term"]][1] = platform["count"]
|
|
elif platform["term"] == "Mac OS X":
|
|
signatures[signature["term"]][2] = platform["count"]
|
|
|
|
for startup_crash in signature["facets"]["startup_crash"]:
|
|
if startup_crash["term"] in ["1", "T"]:
|
|
signatures[signature["term"]][4] += startup_crash["count"]
|
|
|
|
signatures[signature["term"]][5] = signature["facets"][
|
|
"cardinality_install_time"
|
|
]["value"]
|
|
|
|
params = {
|
|
"product": product,
|
|
"version": versions,
|
|
"date": socorro.SuperSearch.get_search_date(start_date, end_date),
|
|
"release_channel": channel,
|
|
"_aggs.signature": ["platform", "_cardinality.install_time", "startup_crash"],
|
|
"_results_number": 0,
|
|
"_facets_size": tc_limit,
|
|
}
|
|
|
|
if startup:
|
|
params["startup_crash"] = True
|
|
|
|
sys.stdout.write("Getting top signatures from Socorro...")
|
|
sys.stdout.flush()
|
|
socorro.SuperSearch(params=params, handler=signature_handler).wait()
|
|
sys.stdout.write(" ✔\n")
|
|
sys.stdout.flush()
|
|
|
|
bug_flags = [
|
|
"resolution",
|
|
"id",
|
|
"last_change_time",
|
|
"cf_tracking_firefox" + str(version),
|
|
"product",
|
|
"component",
|
|
]
|
|
for i in range(int(version), int(v["nightly"]) + 1):
|
|
bug_flags.append("cf_status_firefox" + str(i))
|
|
|
|
# TODO: too many requests... should be improved with chunks
|
|
bugs = {}
|
|
# TODO: Use regexp, when the Bugzilla bug that prevents them from working will be fixed.
|
|
base = {
|
|
"j_top": "OR",
|
|
"o1": "substring",
|
|
"f1": "cf_crash_signature",
|
|
"v1": None,
|
|
"o2": "substring",
|
|
"f2": "cf_crash_signature",
|
|
"v2": None,
|
|
"o3": "substring",
|
|
"f3": "cf_crash_signature",
|
|
"v3": None,
|
|
"o4": "substring",
|
|
"f4": "cf_crash_signature",
|
|
"v4": None,
|
|
"include_fields": bug_flags,
|
|
}
|
|
|
|
queries = []
|
|
for sgn in signatures.keys():
|
|
cparams = base.copy()
|
|
cparams["v1"] = "[@" + sgn + "]"
|
|
cparams["v2"] = "[@ " + sgn + " ]"
|
|
cparams["v3"] = "[@ " + sgn + "]"
|
|
cparams["v4"] = "[@" + sgn + " ]"
|
|
bugs[sgn] = []
|
|
queries.append(Query(Bugzilla.API_URL, cparams, __bug_handler, bugs[sgn]))
|
|
res_bugs = Bugzilla(queries=queries)
|
|
|
|
# we have stats by signature in self.signatures
|
|
# for each signature get the number of crashes on the last X days
|
|
# so get the signature trend
|
|
trends = {}
|
|
default_trend = {}
|
|
for i in range(duration):
|
|
default_trend[_date - timedelta(i)] = 0
|
|
|
|
base = {
|
|
"product": product,
|
|
"version": versions,
|
|
"signature": None,
|
|
"date": socorro.SuperSearch.get_search_date(start_date, end_date),
|
|
"release_channel": channel,
|
|
"_results_number": 0,
|
|
"_histogram.date": ["signature"],
|
|
"_histogram_interval": 1,
|
|
}
|
|
|
|
queries = []
|
|
for sgns in Connection.chunks(
|
|
list(map(lambda sgn: "=" + sgn, signatures.keys())), 10
|
|
):
|
|
sgn_group = []
|
|
for sgn in sgns:
|
|
if sum(len(s) for s in sgn_group) >= 1000:
|
|
cparams = base.copy()
|
|
cparams["signature"] = sgn_group
|
|
queries.append(
|
|
Query(
|
|
socorro.SuperSearch.URL,
|
|
cparams,
|
|
functools.partial(__trend_handler, default_trend),
|
|
trends,
|
|
)
|
|
)
|
|
sgn_group = []
|
|
|
|
sgn_group.append(sgn)
|
|
|
|
if len(sgn_group) > 0:
|
|
cparams = base.copy()
|
|
cparams["signature"] = sgn_group
|
|
queries.append(
|
|
Query(
|
|
socorro.SuperSearch.URL,
|
|
cparams,
|
|
functools.partial(__trend_handler, default_trend),
|
|
trends,
|
|
)
|
|
)
|
|
|
|
sys.stdout.write("Getting trends for top signatures from Socorro...")
|
|
sys.stdout.flush()
|
|
socorro.SuperSearch(queries=queries).wait()
|
|
sys.stdout.write(" ✔\n")
|
|
sys.stdout.flush()
|
|
|
|
for sgn, trend in trends.items():
|
|
signatures[sgn] = (
|
|
signatures[sgn],
|
|
[trend[key] for key in sorted(trend.keys(), reverse=True)],
|
|
)
|
|
|
|
_signatures = {}
|
|
# order self.signatures by crash count
|
|
sorted_signatures = sorted(
|
|
signatures.items(), key=lambda x: x[1][0][0], reverse=True
|
|
)
|
|
i = 1
|
|
for s in sorted_signatures:
|
|
_signatures[s[0]] = i # top crash rank
|
|
i += 1
|
|
|
|
sys.stdout.write("Getting bugs linked to the top signatures from Bugzilla...")
|
|
sys.stdout.flush()
|
|
while True:
|
|
try:
|
|
for r in res_bugs.results:
|
|
r.result(timeout=2)
|
|
break
|
|
except TimeoutError:
|
|
sys.stdout.write(".")
|
|
sys.stdout.flush()
|
|
sys.stdout.write(" ✔\n")
|
|
sys.stdout.flush()
|
|
|
|
# TODO: In the first query to get the bugs, also get dupe_of and avoid the first query
|
|
# in follow_dup (so modify follow_dup to accept both a bug ID or a bug object).
|
|
queries = []
|
|
for sgn in signatures.keys():
|
|
duplicate_ids = [
|
|
bug["id"] for bug in bugs[sgn] if bug["resolution"] == "DUPLICATE"
|
|
]
|
|
|
|
# Remove bugs resolved as DUPLICATE from the list of bugs associated to the signature.
|
|
bugs[sgn] = [bug for bug in bugs[sgn] if bug["id"] not in duplicate_ids]
|
|
|
|
# Find duplicates for bugs resolved as DUPLICATE.
|
|
duplicates = {
|
|
k: v for k, v in Bugzilla.follow_dup(duplicate_ids).items() if v is not None
|
|
}
|
|
duplicate_targets = [
|
|
bug_id
|
|
for bug_id in duplicates.values()
|
|
if int(bug_id) not in [bug["id"] for bug in bugs[sgn]]
|
|
]
|
|
if len(duplicate_targets) == 0:
|
|
continue
|
|
|
|
# Get info about bugs that the DUPLICATE bugs have been duped to.
|
|
params = {
|
|
"id": ",".join(duplicate_targets),
|
|
"include_fields": bug_flags,
|
|
}
|
|
queries.append(Query(Bugzilla.API_URL, params, __bug_handler, bugs[sgn]))
|
|
sys.stdout.write(
|
|
"Resolving duplicate bugs to the bugs they've been duplicated to..."
|
|
)
|
|
sys.stdout.flush()
|
|
Bugzilla(queries=queries).wait()
|
|
sys.stdout.write(" ✔\n")
|
|
sys.stdout.flush()
|
|
|
|
for sgn, stats in signatures.items():
|
|
# stats is 2-uple: ([count, win_count, mac_count, linux_count, startup_count], trend)
|
|
startup_percent = float(stats[0][4]) / float(stats[0][0])
|
|
_signatures[sgn] = {
|
|
"tc_rank": _signatures[sgn],
|
|
"crash_count": stats[0][0],
|
|
"estimated_user_count": stats[0][5],
|
|
"startup_percent": startup_percent,
|
|
"bugs": bugs[sgn],
|
|
}
|
|
|
|
return {
|
|
"start_date": start_date,
|
|
"end_date": end_date,
|
|
"versions": list(versions),
|
|
"signatures": _signatures,
|
|
}
|
|
|
|
|
|
def get_with_retries(url, params=None, headers=None):
|
|
retries = Retry(total=16, backoff_factor=1, status_forcelist=[429])
|
|
|
|
s = requests.Session()
|
|
s.mount(
|
|
"https://analysis-output.telemetry.mozilla.org",
|
|
HTTPAdapter(max_retries=retries),
|
|
)
|
|
|
|
return s.get(url, params=params, headers=headers)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
DATE = "yesterday"
|
|
DURATION = 11
|
|
TC_LIMIT = 200
|
|
|
|
Bugzilla.TOKEN = os.environ.get("BUGZILLA_TOKEN")
|
|
|
|
try:
|
|
shutil.rmtree("dist")
|
|
except OSError:
|
|
pass
|
|
os.mkdir("dist")
|
|
os.mkdir("dist/images")
|
|
|
|
for channel in ["release", "beta", "nightly"]:
|
|
for startup in [False, True]:
|
|
print(
|
|
"Getting top-"
|
|
+ str(TC_LIMIT)
|
|
+ (" startup " if startup else " ")
|
|
+ "crashes for the '"
|
|
+ channel
|
|
+ "' channel"
|
|
)
|
|
|
|
stats = get(
|
|
channel,
|
|
DATE,
|
|
duration=int(DURATION),
|
|
tc_limit=int(TC_LIMIT),
|
|
startup=startup,
|
|
)
|
|
|
|
print("\n\n")
|
|
|
|
with open(
|
|
"dist/" + channel + ("-startup" if startup else "") + ".json", "w"
|
|
) as f:
|
|
json.dump(stats, f, allow_nan=False)
|
|
|
|
with open("dist/graphics_critical_errors.json", "w") as f:
|
|
json.dump(gfx_critical_errors.analyze_gfx_critical_errors(), f)
|
|
|
|
paths = [
|
|
"index.html",
|
|
"correlations.js",
|
|
"buildid_changeset.js",
|
|
"style.css",
|
|
"exclamation_mark.svg",
|
|
"question_mark.svg",
|
|
"rocket_fly.png",
|
|
"spin.svg",
|
|
"correlations.html",
|
|
"correlations_page.js",
|
|
"missing_uplifts.html",
|
|
"missing_uplifts.js",
|
|
"all_missing_uplifts.html",
|
|
"all_missing_uplifts.js",
|
|
"compare-betas.html",
|
|
"compare-betas.js",
|
|
"scomp.html",
|
|
"scomp.css",
|
|
"scomp.js",
|
|
"beta-stability-pushlog.html",
|
|
"beta-stability-pushlog.js",
|
|
"graphics_critical_errors.html",
|
|
"graphics_critical_errors.js",
|
|
"buildid_changeset.html",
|
|
"buildid_changeset_page.js",
|
|
"addon_related_signatures.html",
|
|
"addon_related_signatures.js",
|
|
"supergraph.html",
|
|
"supergraph.js",
|
|
"rerank.html",
|
|
"rerank.js",
|
|
"common_landings.html",
|
|
"common_landings.js",
|
|
"channels_diff.html",
|
|
"channels_diff.js",
|
|
] + ["images/" + image for image in os.listdir("images")]
|
|
|
|
for path in paths:
|
|
shutil.copyfile(path, "dist/" + path)
|
|
|
|
for product in ["firefox"]:
|
|
base_url = (
|
|
"https://analysis-output.telemetry.mozilla.org/top-signatures-correlations/data/"
|
|
if product == "firefox"
|
|
else "https://analysis-output.telemetry.mozilla.org/top-fennec-signatures-correlations/data/"
|
|
)
|
|
|
|
# Check that correlations were generated.
|
|
r = get_with_retries(base_url + "all.json.gz")
|
|
if r.status_code != 200:
|
|
print(product + " correlations weren't generated.")
|
|
print(r.text)
|
|
raise Exception(r)
|
|
|
|
generation_date = datetime.strptime(
|
|
r.headers["last-modified"], "%a, %d %b %Y %H:%M:%S %Z"
|
|
).date()
|
|
|
|
if datetime.utcnow().date() - timedelta(1) > generation_date:
|
|
print(
|
|
product
|
|
+ " correlations weren't generated yesterday, they were last generated on "
|
|
+ str(generation_date)
|
|
+ "."
|
|
)
|
|
raise Exception(r)
|
|
|
|
for channel in ["release", "beta", "nightly"]:
|
|
# Check that the OOM | small correlations contain "moz_crash_reason" as a sanity check.
|
|
r = get_with_retries(
|
|
base_url
|
|
+ channel
|
|
+ "/"
|
|
+ hashlib.sha1("OOM | small".encode("utf-8")).hexdigest()
|
|
+ ".json.gz"
|
|
)
|
|
if r.status_code != 200:
|
|
print(
|
|
"Failure downloading "
|
|
+ product
|
|
+ " "
|
|
+ channel
|
|
+ ' correlations for "OOM | small".'
|
|
)
|
|
print(r.text)
|
|
raise Exception(r)
|
|
|
|
if not any(
|
|
any(
|
|
key in result["item"].keys()
|
|
for key in [
|
|
"CPU Info",
|
|
"reason",
|
|
"moz_crash_reason",
|
|
"platform_version",
|
|
]
|
|
)
|
|
for result in r.json()["results"]
|
|
):
|
|
print(
|
|
product
|
|
+ " "
|
|
+ channel
|
|
+ ' correlations failing "OOM | small" sanity check.'
|
|
)
|
|
print(r.json())
|
|
raise Exception(r)
|