mozilla
/
bugbug
зеркало из https://github.com/mozilla/bugbug.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Add script to compute success rate for component models (#190)

This commit is contained in:
John Giannelos 2019-02-26 15:16:39 +01:00 коммит произвёл Marco
Родитель ee3b63c277
Коммит d29621b84d
2 изменённых файлов: 181 добавлений и 73 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

147
bugbug/models/component.py
Просмотреть файл

@ -16,6 +16,62 @@ from bugbug.model import Model
class ComponentModel(Model):
PRODUCTS = {
'Core', 'External Software Affecting Firefox', 'DevTools', 'Firefox for Android', 'Firefox', 'Toolkit',
'WebExtensions'
}
CONFLATED_COMPONENTS = [
'Core::Audio/Video', 'Core::Graphics', 'Core::IPC', 'Core::JavaScript', 'Core::Layout', 'Core::Networking',
'Core::Print', 'Core::WebRTC', 'Firefox::Activity Streams', 'Toolkit::Password Manager',
]
MEANINGFUL_COMPONENTS = [
'Core::CSS Parsing and Computation', 'Core::Canvas: 2D', 'Core::Canvas: WebGL', 'Core::DOM',
'Core::DOM: Animation', 'Core::DOM: CSS Object Model', 'Core::DOM: Content Processes',
'Core::DOM: Device Interfaces', 'Core::DOM: Events', 'Core::DOM: IndexedDB',
'Core::DOM: Push Notifications', 'Core::DOM: Security', 'Core::DOM: Service Workers',
'Core::DOM: Web Payments', 'Core::DOM: Workers', 'Core::Disability Access APIs',
'Core::Document Navigation', 'Core::Drag and Drop', 'Core::Editor', 'Core::Event Handling',
'Core::Gecko Profiler', 'Core::Geolocation', 'Core::HTML: Parser', 'Core::ImageLib',
'Core::Internationalization', 'Core::MFBT', 'Core::MathML', 'Core::Memory Allocator',
'Core::Panning and Zooming', 'Core::Plug-ins', 'Core::Preferences: Backend', 'Core::SVG',
'Core::Security', 'Core::Security: PSM', 'Core::Security: Process Sandboxing', 'Core::Selection',
'Core::Spelling checker', 'Core::String', 'Core::Web Audio', 'Core::Web Painting', 'Core::Web Replay',
'Core::Web Speech', 'Core::WebVR', 'Core::Widget', 'Core::Widget: Android', 'Core::Widget: Cocoa',
'Core::Widget: Gtk', 'Core::Widget: Win32', 'Core::XPCOM', 'Core::XPConnect', 'Core::XUL',
'DevTools::Accessibility Tools', 'DevTools::Animation Inspector', 'DevTools::CSS Rules Inspector',
'DevTools::Console', 'DevTools::Debugger', 'DevTools::Font Inspector', 'DevTools::Framework',
'DevTools::Inspector', 'DevTools::JSON Viewer', 'DevTools::Memory', 'DevTools::Netmonitor',
'DevTools::Performance Tools (Profiler/Timeline)', 'DevTools::Responsive Design Mode',
'DevTools::Shared Components', 'DevTools::Storage Inspector', 'DevTools::Style Editor', 'DevTools::WebIDE',
'DevTools::about:debugging',
'External Software Affecting Firefox::Other',
'Firefox for Android::Activity Stream', 'Firefox for Android::Android Sync',
'Firefox for Android::Audio/Video', 'Firefox for Android::Awesomescreen',
'Firefox for Android::Firefox Accounts', 'Firefox for Android::GeckoView',
'Firefox for Android::Keyboards and IME', 'Firefox for Android::Metrics',
'Firefox for Android::Settings and Preferences', 'Firefox for Android::Testing',
'Firefox for Android::Theme and Visual Design', 'Firefox for Android::Toolbar',
'Firefox for Android::Web Apps',
'Firefox::Address Bar', 'Firefox::Device Permissions', 'Firefox::Downloads Panel',
'Firefox::Enterprise Policies', 'Firefox::Extension Compatibility', 'Firefox::File Handling',
'Firefox::Installer', 'Firefox::Keyboard Navigation', 'Firefox::Menus', 'Firefox::Migration',
'Firefox::New Tab Page', 'Firefox::Normandy Client', 'Firefox::PDF Viewer', 'Firefox::Pocket',
'Firefox::Preferences', 'Firefox::Private Browsing', 'Firefox::Screenshots', 'Firefox::Search',
'Firefox::Security', 'Firefox::Session Restore', 'Firefox::Shell Integration',
'Firefox::Site Identity and Permission Panels', 'Firefox::Sync', 'Firefox::Tabbed Browser',
'Firefox::Theme', 'Firefox::Toolbars and Customization', 'Firefox::Tours',
'Firefox::Tracking Protection', 'Firefox::WebPayments UI',
'Toolkit::Add-ons Manager', 'Toolkit::Application Update', 'Toolkit::Blocklist Policy Requests',
'Toolkit::Crash Reporting', 'Toolkit::Downloads API', 'Toolkit::Find Toolbar', 'Toolkit::Form Autofill',
'Toolkit::Form Manager', 'Toolkit::Notifications and Alerts', 'Toolkit::Performance Monitoring',
'Toolkit::Places', 'Toolkit::Reader Mode', 'Toolkit::Safe Browsing', 'Toolkit::Startup and Profile System',
'Toolkit::Storage', 'Toolkit::Telemetry', 'Toolkit::Themes', 'Toolkit::Video/Audio Controls',
'Toolkit::XUL Widgets',
]
def __init__(self, lemmatization=False):
Model.__init__(self, lemmatization)
@ -57,86 +113,31 @@ class ComponentModel(Model):
self.clf = xgboost.XGBClassifier(n_jobs=16)
self.clf.set_params(predictor='cpu_predictor')
def filter_component(self, bug_data):
if bug_data['product'] not in self.PRODUCTS:
return None
full_comp = f'{bug_data["product"]}::{bug_data["component"]}'
for conflated_component in self.CONFLATED_COMPONENTS:
if full_comp.startswith(conflated_component):
return conflated_component
if full_comp in self.MEANINGFUL_COMPONENTS:
return full_comp
return bug_data['product']
def get_labels(self):
products = {
'Core', 'External Software Affecting Firefox', 'DevTools', 'Firefox for Android', 'Firefox', 'Toolkit',
'WebExtensions'
}
classes = {}
conflated_components = [
'Core::Audio/Video', 'Core::Graphics', 'Core::IPC', 'Core::JavaScript', 'Core::Layout', 'Core::Networking',
'Core::Print', 'Core::WebRTC', 'Firefox::Activity Streams', 'Toolkit::Password Manager',
]
meaningful_components = [
'Core::CSS Parsing and Computation', 'Core::Canvas: 2D', 'Core::Canvas: WebGL', 'Core::DOM',
'Core::DOM: Animation', 'Core::DOM: CSS Object Model', 'Core::DOM: Content Processes',
'Core::DOM: Device Interfaces', 'Core::DOM: Events', 'Core::DOM: IndexedDB',
'Core::DOM: Push Notifications', 'Core::DOM: Security', 'Core::DOM: Service Workers',
'Core::DOM: Web Payments', 'Core::DOM: Workers', 'Core::Disability Access APIs',
'Core::Document Navigation', 'Core::Drag and Drop', 'Core::Editor', 'Core::Event Handling',
'Core::Gecko Profiler', 'Core::Geolocation', 'Core::HTML: Parser', 'Core::ImageLib',
'Core::Internationalization', 'Core::MFBT', 'Core::MathML', 'Core::Memory Allocator',
'Core::Panning and Zooming', 'Core::Plug-ins', 'Core::Preferences: Backend', 'Core::SVG',
'Core::Security', 'Core::Security: PSM', 'Core::Security: Process Sandboxing', 'Core::Selection',
'Core::Spelling checker', 'Core::String', 'Core::Web Audio', 'Core::Web Painting', 'Core::Web Replay',
'Core::Web Speech', 'Core::WebVR', 'Core::Widget', 'Core::Widget: Android', 'Core::Widget: Cocoa',
'Core::Widget: Gtk', 'Core::Widget: Win32', 'Core::XPCOM', 'Core::XPConnect', 'Core::XUL',
'DevTools::Accessibility Tools', 'DevTools::Animation Inspector', 'DevTools::CSS Rules Inspector',
'DevTools::Console', 'DevTools::Debugger', 'DevTools::Font Inspector', 'DevTools::Framework',
'DevTools::Inspector', 'DevTools::JSON Viewer', 'DevTools::Memory', 'DevTools::Netmonitor',
'DevTools::Performance Tools (Profiler/Timeline)', 'DevTools::Responsive Design Mode',
'DevTools::Shared Components', 'DevTools::Storage Inspector', 'DevTools::Style Editor', 'DevTools::WebIDE',
'DevTools::about:debugging',
'External Software Affecting Firefox::Other',
'Firefox for Android::Activity Stream', 'Firefox for Android::Android Sync',
'Firefox for Android::Audio/Video', 'Firefox for Android::Awesomescreen',
'Firefox for Android::Firefox Accounts', 'Firefox for Android::GeckoView',
'Firefox for Android::Keyboards and IME', 'Firefox for Android::Metrics',
'Firefox for Android::Settings and Preferences', 'Firefox for Android::Testing',
'Firefox for Android::Theme and Visual Design', 'Firefox for Android::Toolbar',
'Firefox for Android::Web Apps',
'Firefox::Address Bar', 'Firefox::Device Permissions', 'Firefox::Downloads Panel',
'Firefox::Enterprise Policies', 'Firefox::Extension Compatibility', 'Firefox::File Handling',
'Firefox::Installer', 'Firefox::Keyboard Navigation', 'Firefox::Menus', 'Firefox::Migration',
'Firefox::New Tab Page', 'Firefox::Normandy Client', 'Firefox::PDF Viewer', 'Firefox::Pocket',
'Firefox::Preferences', 'Firefox::Private Browsing', 'Firefox::Screenshots', 'Firefox::Search',
'Firefox::Security', 'Firefox::Session Restore', 'Firefox::Shell Integration',
'Firefox::Site Identity and Permission Panels', 'Firefox::Sync', 'Firefox::Tabbed Browser',
'Firefox::Theme', 'Firefox::Toolbars and Customization', 'Firefox::Tours',
'Firefox::Tracking Protection', 'Firefox::WebPayments UI',
'Toolkit::Add-ons Manager', 'Toolkit::Application Update', 'Toolkit::Blocklist Policy Requests',
'Toolkit::Crash Reporting', 'Toolkit::Downloads API', 'Toolkit::Find Toolbar', 'Toolkit::Form Autofill',
'Toolkit::Form Manager', 'Toolkit::Notifications and Alerts', 'Toolkit::Performance Monitoring',
'Toolkit::Places', 'Toolkit::Reader Mode', 'Toolkit::Safe Browsing', 'Toolkit::Startup and Profile System',
'Toolkit::Storage', 'Toolkit::Telemetry', 'Toolkit::Themes', 'Toolkit::Video/Audio Controls',
'Toolkit::XUL Widgets',
]
for bug_data in bugzilla.get_bugs():
if bug_data['product'] not in products:
continue
bug_id = int(bug_data['id'])
full_comp = f'{bug_data["product"]}::{bug_data["component"]}'
component = self.filter_component(bug_data)
for conflated_component in conflated_components:
if full_comp.startswith(conflated_component):
classes[bug_id] = conflated_component
break
if bug_id in classes:
continue
if full_comp in meaningful_components:
classes[bug_id] = full_comp
if bug_id in classes:
continue
classes[bug_id] = bug_data['product']
if component:
classes[bug_id] = component
component_counts = Counter(classes.values()).most_common()
top_components = set(component for component, count in component_counts)

107
scripts/get_untriaged.py Normal file
Просмотреть файл

@ -0,0 +1,107 @@
# -*- coding: utf-8 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import argparse
import csv
import json
import os
import sys
from datetime import date
from datetime import datetime
from datetime import timedelta
# Inject project path
sys.path.append('../')
def fetch_untriaged(args):
from bugbug import bugzilla
today = date.today()
three_months_ago = today - timedelta(days=args.days_back)
# Set bugzilla token and download bugs
bugzilla.set_token(args.token)
bug_ids = bugzilla.download_bugs_between(three_months_ago, today)
# Get untriaged bugs
bugs = bugzilla.get_bugs()
untriaged_bugs = []
for bug in bugs:
if bug['id'] not in bug_ids:
continue
for history in bug['history']:
for change in history['changes']:
if change['field_name'] == 'component' and change['removed'] == 'Untriaged':
untriaged_bugs.append(bug)
with open('bugs-{}.json'.format(datetime.now().strftime('%s')), 'w') as f:
json.dump(untriaged_bugs, f)
return untriaged_bugs
def run_untriaged(untriaged_bugs):
from bugbug.models.component import ComponentModel
from bugbug.models.component_nn import ComponentNNModel
models = [
(ComponentModel, '../componentmodel'),
(ComponentNNModel, '../componentnnmodel')
]
for (model_class, model_file_name) in models:
rows = []
model = model_class.load(model_file_name)
for bug in untriaged_bugs:
p = model.classify(bug, probabilities=True)
url = f'https://bugzilla.mozilla.org/show_bug.cgi?id={bug["id"]}'
classifiable = p[p >= 0.7].size >= 1
classification = ''
expected_component = model.filter_component(bug)
if not expected_component:
print('Skipping bug: {}'.format(bug['id']))
continue
if classifiable:
print('Classifying bug with ID: {}'.format(bug['id']))
classification = model.classify(bug)[0]
print('Classified bug as: {}'.format(classification))
else:
print('Not classifiable bug: {}'.format(bug['id']))
correct_prediction = expected_component == classification
rows.append([url, classifiable, expected_component, classification, correct_prediction, bug['summary']])
os.makedirs('sheets', exist_ok=True)
class_name = model.__class__.__name__
timestamp = datetime.utcnow().strftime('%Y-%m-%d')
sheet_name = f'{class_name}-{timestamp}-labels.csv'
with open(os.path.join('sheets', sheet_name), 'w') as f:
writer = csv.writer(f)
writer.writerows(rows)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--token',
help='Bugzilla API token')
parser.add_argument('--days-back',
type=int,
help='Days since to fetch bugs')
parser.add_argument('--file_path',
help='Days since to fetch bugs')
args = parser.parse_args()
if args.file_path:
with open(args.file_path, 'r') as f:
bugs = json.load(f)
else:
bugs = fetch_untriaged(args)
run_untriaged(bugs)