mozilla
/
kitsune
зеркало из https://github.com/mozilla/kitsune.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность
kitsune/scripts/year_in_review.py

469 строки
13 KiB
Python
Executable File
Исходник Постоянная ссылка Ответственный История

#!/usr/bin/env python
import json
import logging
import subprocess
import sys
import textwrap
import xmlrpc.client
USAGE = 'Usage: year_in_review.py [--json] <YEAR>'
# Note: Most of the bugzila api code comes from Scrumbugz.
cache = {}
log = logging.getLogger(__name__)
BZ_URL = 'http://bugzilla.mozilla.org/xmlrpc.cgi'
SESSION_COOKIES_CACHE_KEY = 'bugzilla-session-cookies'
PRODUCTS = [
'support.mozilla.org'
]
BZ_RESOLUTIONS = [
'',
'FIXED',
'INVALID',
'WONTFIX',
'DUPLICATE',
'WORKSFORME',
'INCOMPLETE',
'SUPPORT',
'EXPIRED',
'MOVED'
]
BZ_FIELDS = [
'id',
'status',
'resolution',
'summary',
'whiteboard',
'assigned_to',
'priority',
'severity',
'product',
'component',
'blocks',
'depends_on',
'creator',
'creation_time',
'last_change_time',
'target_milestone',
]
UNWANTED_COMPONENT_FIELDS = [
'sort_key',
'is_active',
'default_qa_contact',
'default_assigned_to',
'description'
]
class SessionTransport(xmlrpc.client.SafeTransport):
"""
XML-RPC HTTPS transport that stores auth cookies in the cache.
"""
_session_cookies = None
@property
def session_cookies(self):
if self._session_cookies is None:
cookie = cache.get(SESSION_COOKIES_CACHE_KEY)
if cookie:
self._session_cookies = cookie
return self._session_cookies
def parse_response(self, response):
cookies = self.get_cookies(response)
if cookies:
self._session_cookies = cookies
cache.set(SESSION_COOKIES_CACHE_KEY,
self._session_cookies, 0)
log.debug('Got cookie: %s', self._session_cookies)
return xmlrpc.client.Transport.parse_response(self, response)
def send_host(self, connection, host):
cookies = self.session_cookies
if cookies:
for cookie in cookies:
connection.putheader('Cookie', cookie)
log.debug('Sent cookie: %s', cookie)
return xmlrpc.client.Transport.send_host(self, connection, host)
def get_cookies(self, response):
cookie_headers = None
if hasattr(response, 'msg'):
cookies = response.msg.getheaders('set-cookie')
if cookies:
log.debug('Full cookies: %s', cookies)
cookie_headers = [c.split(';', 1)[0] for c in cookies]
return cookie_headers
class BugzillaAPI(xmlrpc.client.ServerProxy):
def get_bug_ids(self, **kwargs):
"""Return list of ids of bugs from a search."""
kwargs.update({
'include_fields': ['id'],
})
log.debug('Searching bugs with kwargs: %s', kwargs)
bugs = self.Bug.search(kwargs)
return [bug['id'] for bug in bugs.get('bugs', [])]
def get_bugs(self, **kwargs):
defaults = {
'include_fields': BZ_FIELDS,
}
get_history = kwargs.pop('history', True)
get_comments = kwargs.pop('comments', True)
defaults.update(kwargs)
if 'ids' in defaults:
defaults['permissive'] = True
log.debug('Getting bugs with kwargs: %s', defaults)
bugs = self.Bug.get(defaults)
else:
log.debug('Searching bugs with kwargs: %s', defaults)
bugs = self.Bug.search(defaults)
bug_ids = [bug['id'] for bug in bugs.get('bugs', [])]
if not bug_ids:
return bugs
# mix in history and comments
history = comments = {}
if get_history:
history = self.get_history(bug_ids)
if get_comments:
comments = self.get_comments(bug_ids)
for bug in bugs['bugs']:
bug['history'] = history.get(bug['id'], [])
bug['comments'] = comments.get(bug['id'], {}).get('comments', [])
bug['comments_count'] = len(comments.get(bug['id'], {})
.get('comments', []))
return bugs
def get_history(self, bug_ids):
log.debug('Getting history for bugs: %s', bug_ids)
try:
history = self.Bug.history({'ids': bug_ids}).get('bugs')
except xmlrpc.client.Fault:
log.exception('Problem getting history for bug ids: %s', bug_ids)
return {}
return dict((h['id'], h['history']) for h in history)
def get_comments(self, bug_ids):
log.debug('Getting comments for bugs: %s', bug_ids)
try:
comments = self.Bug.comments({
'ids': bug_ids,
'include_fields': ['id', 'creator', 'time', 'text'],
}).get('bugs')
except xmlrpc.client.Fault:
log.exception('Problem getting comments for bug ids: %s', bug_ids)
return {}
return dict((int(bid), cids) for bid, cids in comments.items())
def wrap(text, indent=' '):
text = text.split('\n\n')
text = [textwrap.fill(part, expand_tabs=True, initial_indent=indent,
subsequent_indent=indent)
for part in text]
return '\n\n'.join(text)
def parse_whiteboard(whiteboard):
bits = {
'u': '',
'c': '',
'p': '',
's': ''
}
for part in whiteboard.split(' '):
part = part.split('=')
if len(part) != 2:
continue
if part[0] in bits:
bits[part[0]] = part[1]
return bits
def bugzilla_stats(year):
stats = []
bugzilla = BugzillaAPI(
BZ_URL,
transport=SessionTransport(use_datetime=True),
allow_none=True)
# -------------------------------------------
# Bugs created this year
# -------------------------------------------
bugs = bugzilla.get_bugs(
product=PRODUCTS,
creation_time='%s-01-01' % year,
include_fields=['id', 'creator', 'creation_time'],
history=False,
comments=False)
bugs = bugs['bugs']
total = 0
creators = {}
for bug in bugs:
# We can only get creation_time >= somedate, so we need to nix
# the bugs that are after the year we're looking for.
if bug['creation_time'].year != int(year):
continue
total += 1
creators[bug['creator']] = creators.get(bug['creator'], 0) + 1
creators = sorted(list(creators.items()), key=lambda item: item[1], reverse=True)
stats.append(('Bugs created', {
'total': total,
'breakdown': [
{'name': mem[0].split('@')[0], 'count': mem[1]}
for mem in creators[:10]]
}))
# -------------------------------------------
# Bugs resolved this year
# -------------------------------------------
bugs = bugzilla.get_bugs(
product=PRODUCTS,
last_change_time='%s-01-01' % year,
include_fields=['id', 'summary', 'assigned_to', 'last_change_time', 'resolution'],
status=['RESOLVED', 'VERIFIED', 'CLOSED'],
history=True,
comments=False)
bugs = bugs['bugs']
total = 0
peeps = {}
resolutions = {}
traceback_bugs = []
research_bugs = []
tracker_bugs = []
for bug in bugs:
# We can only get last_change_time >= somedate, so we need to
# nix the bugs that are after the year we're looking for.
if bug['last_change_time'].year != int(year):
continue
if bug['summary'].lower().startswith('[traceback]'):
traceback_bugs.append(bug)
if bug['summary'].lower().startswith('[research]'):
research_bugs.append(bug)
if bug['summary'].lower().startswith('[tracker]'):
tracker_bugs.append(bug)
for hist in bug['history']:
for change in hist['changes']:
if not change['field_name'] == 'resolution':
continue
# I think this history item comes from clearing the
# resolution. i.e. reopening.
if change['added'] == '':
continue
total += 1
# If the bug is marked FIXED, we assume that whoever
# it was assigned to should get the "credit". If it
# wasn't marked FIXED, then it's probably someone
# doing triage and so whoever changed the resolution
# should get "credit".
if (change['added'] == 'FIXED'
and not 'nobody' in bug['assigned_to']):
person = bug['assigned_to']
else:
person = hist['who']
peeps_dict = peeps.setdefault(person, {})
key = change['added']
peeps_dict[key] = peeps_dict.get(key, 0) + 1
resolutions[change['added']] = resolutions.get(
change['added'], 0) + 1
peeps = sorted(list(peeps.items()), key=lambda item: sum(item[1].values()), reverse=True)
stats.append(('Bugs resolved', {
'total': total,
'breakdown': [
{'name': mem[0].split('@')[0],
'total': sum(mem[1].values()),
'breakdown': list(mem[1].items())}
for mem in peeps[:10]
]
}))
# -------------------------------------------
# Resolution stats
# -------------------------------------------
resolutions = sorted(list(resolutions.items()), key=lambda item: item[1])
stats.append(('Bugs resolved breakdown', resolutions))
# -------------------------------------------
# Research bugs
# -------------------------------------------
stats.append(('Research bugs', [
{'id': bug['id'], 'summary': bug['summary']}
for bug in research_bugs
]))
# -------------------------------------------
# Trackers
# -------------------------------------------
stats.append(('Tracker bugs', [
{'id': bug['id'], 'summary': bug['summary']}
for bug in tracker_bugs
]))
return stats
def git(*args):
return subprocess.check_output(args)
def git_stats(year):
stats = []
# Get the shas for all the commits we're going to look at.
all_commits = subprocess.check_output([
'git', 'log',
'--after=%s-01-01' % year,
'--before=%s-01-01' % (int(year) + 1),
'--format=%H'
])
all_commits = all_commits.splitlines()
# Person -> # commits
committers = {}
# Person -> (# files changed, # inserted, # deleted)
changes = {}
for commit in all_commits:
author = git('git', 'log', '--format=%an',
'{0}~..{1}'.format(commit, commit))
author = author.strip()
# FIXME - this is lame. what's going on is that there are
# merge commits which have multiple authors, so we just grab
# the second one.
if '\n' in author:
author = author.splitlines()[1]
committers[author] = committers.get(author, 0) + 1
diff_data = git('git', 'diff', '--numstat', '--find-copies-harder',
'{0}~..{1}'.format(commit, commit))
total_added = 0
total_deleted = 0
total_files = 0
for line in diff_data.splitlines():
added, deleted, fn = line.split('\t')
if fn.startswith('vendor/'):
continue
if added != '-':
total_added += int(added)
if deleted != '-':
total_deleted += int(deleted)
total_files += 1
old_changes = changes.get(author, (0, 0, 0))
changes[author] = (
old_changes[0] + total_added,
old_changes[1] + total_deleted,
old_changes[2] + total_files
)
print('Total commits:', len(all_commits))
print('')
committers = sorted(
list(committers.items()), key=lambda item: item[1], reverse=True)
committers_data = []
for person, count in committers:
committers_data.append({
'name': person,
'data': {
'commits': count,
'added': changes[person][0],
'deleted': changes[person][1],
'files': changes[person][2]
}
})
stats.append(('Git commit data', {
'total commits': len(all_commits),
'total lines added': sum([item[0] for item in list(changes.values())]),
'total lines deleted': sum([item[1] for item in list(changes.values())]),
'total files changed': sum([item[2] for item in list(changes.values())])
}))
stats.append(('Git committer data', committers_data))
return stats
def main(argv):
# XXX: This helps debug bugzilla xmlrpc bits.
# logging.basicConfig(level=logging.DEBUG)
do_json = False
if not argv:
print(USAGE)
print('Error: Must specify the year. e.g. 2012')
return 1
if '--json' in argv:
print('>>> OMGWTFBBQ! You want it in JSON!')
do_json = True
argv.remove('--json')
year = argv[0]
output = []
output.append(('Year', year))
print('>>> Generating bugzilla stats....')
output.extend(bugzilla_stats(year))
print('>>> Generating git stats....')
output.extend(git_stats(year))
print('')
if do_json:
print(json.dumps(output, indent=2))
else:
for mem in output:
print('')
print(mem[0])
print('=' * len(mem[0]))
print('')
# FIXME - this is gross
print(json.dumps(mem[1], indent=2))
if __name__ == '__main__':
sys.exit(main(sys.argv[1:]))
Ссылка в новой задаче Показать git blame Копировать постоянную ссылку