Change the database layout so that the breakdown table now references
tests relationally in a new awfy_suite_test table. This reduces the breakdown table size about 10% and greatly improves the performance of querying test names. This update requires running db_upgrade.py and using the new UDPATE.php and internals.php changes. In addition, this patch skips the condensing step when we already have a condense file for a previous month. This improves the update.py running time by about 50%.
This commit is contained in:
David Anderson
AreWeFastYet
Родитель
3f9f931fa4
Коммит
f3647984f8
|
|
@ -86,6 +86,14 @@ CREATE TABLE `awfy_suite` (
|
|||
UNIQUE KEY `name_UNIQUE` (`name`)
|
||||
) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=9 ;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `awfy_suite_test` (
|
||||
`id` int(10) unsigned NOT NULL AUTO_INCREMENT,
|
||||
`suite_id` int(11) NOT NULL,
|
||||
`name` varchar(128) NOT NULL,
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `suite_id` (`suite_id`,`name`)
|
||||
) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=1 ;
|
||||
|
||||
--
|
||||
-- Dumping data for table `awfy_suite`
|
||||
--
|
||||
|
|
@ -156,14 +164,13 @@ CREATE TABLE `awfy_breakdown` (
|
|||
`run_id` int(11) DEFAULT NULL,
|
||||
`suite_id` int(11) DEFAULT NULL,
|
||||
`mode_id` int(11) DEFAULT NULL,
|
||||
`test` varchar(45) DEFAULT NULL,
|
||||
`score` varchar(45) DEFAULT NULL,
|
||||
`test_id` int(10) DEFAULT NULL,
|
||||
PRIMARY KEY (`id`),
|
||||
KEY `run_id` (`run_id`),
|
||||
KEY `suite_id` (`suite_id`),
|
||||
KEY `mode_id` (`mode_id`),
|
||||
KEY `suite_id_2` (`suite_id`,`test`)
|
||||
) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=7636046 ;
|
||||
) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=1 ;
|
||||
|
||||
-- --------------------------------------------------------
|
||||
|
||||
|
|
@ -178,7 +185,7 @@ CREATE TABLE `awfy_build` (
|
|||
`cset` varchar(256) NOT NULL,
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `index2` (`run_id`,`mode_id`)
|
||||
) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=136734 ;
|
||||
) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=1 ;
|
||||
|
||||
-- --------------------------------------------------------
|
||||
|
||||
|
|
@ -195,7 +202,7 @@ CREATE TABLE `awfy_machine` (
|
|||
`last_checked` int(10) unsigned NOT NULL,
|
||||
`contact` mediumtext NOT NULL
|
||||
PRIMARY KEY (`id`)
|
||||
) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=16 ;
|
||||
) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=1 ;
|
||||
|
||||
-- --------------------------------------------------------
|
||||
|
||||
|
|
@ -212,7 +219,7 @@ CREATE TABLE `awfy_score` (
|
|||
PRIMARY KEY (`id`),
|
||||
KEY `run_id` (`run_id`),
|
||||
KEY `mode_id` (`mode_id`)
|
||||
) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=480043 ;
|
||||
) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=1 ;
|
||||
|
||||
-- --------------------------------------------------------
|
||||
|
||||
|
|
@ -228,5 +235,5 @@ CREATE TABLE `fast_run` (
|
|||
`status` int(11) NOT NULL,
|
||||
`error` mediumtext NOT NULL,
|
||||
PRIMARY KEY (`id`)
|
||||
) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=33848 ;
|
||||
) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=1 ;
|
||||
|
||||
|
|
|
|||
|
|
@ -9,10 +9,20 @@ import sys
|
|||
import awfy
|
||||
import json
|
||||
from profiler import Profiler
|
||||
from datetime import datetime
|
||||
|
||||
SecondsPerDay = 60 * 60 * 24
|
||||
MaxRecentRuns = 30
|
||||
|
||||
def should_export(name, when):
|
||||
path = os.path.join(awfy.path, name)
|
||||
if not os.path.exists(path):
|
||||
return True
|
||||
now = datetime.now()
|
||||
if now.year == when[0] and now.month == when[1]:
|
||||
return True
|
||||
return False
|
||||
|
||||
def export(name, j):
|
||||
path = os.path.join(awfy.path, name)
|
||||
if os.path.exists(path):
|
||||
|
|
@ -208,6 +218,11 @@ def condense(cx, suite, prefix, name):
|
|||
|
||||
for when, graph in graphs:
|
||||
new_name = prefix + 'condensed-' + name + '-' + str(when[0]) + '-' + str(when[1])
|
||||
|
||||
# Don't condense if it already exists...
|
||||
if not should_export(new_name + '.json', when):
|
||||
continue
|
||||
|
||||
sys.stdout.write('Condensing ' + new_name + '... ')
|
||||
sys.stdout.flush()
|
||||
with Profiler() as p:
|
||||
|
|
@ -233,13 +248,13 @@ def condense_suite(cx, machine, suite):
|
|||
# the combine graph back to our caller.
|
||||
suite_aggregate = condense(cx, suite, '', name)
|
||||
|
||||
for test in suite.tests:
|
||||
test_name = suite.name + '-' + test + '-' + str(machine.id)
|
||||
test_aggregate = condense(cx, suite, 'bk-', test_name)
|
||||
for test_id, test_name in suite.tests:
|
||||
test_path = suite.name + '-' + test_name + '-' + str(machine.id)
|
||||
test_aggregate = condense(cx, suite, 'bk-', test_path)
|
||||
j = { 'version': awfy.version,
|
||||
'graph': test_aggregate
|
||||
}
|
||||
export('bk-aggregate-' + test_name + '.json', j)
|
||||
export('bk-aggregate-' + test_path + '.json', j)
|
||||
|
||||
return suite_aggregate
|
||||
|
||||
|
|
|
|||
|
|
@ -17,12 +17,9 @@ class Benchmark(object):
|
|||
# Get a list of individual tests
|
||||
self.tests = []
|
||||
c = awfy.db.cursor()
|
||||
c.execute("SELECT test FROM awfy_breakdown \
|
||||
WHERE suite_id = %s \
|
||||
GROUP BY test",
|
||||
[suite_id])
|
||||
c.execute("select id, name from awfy_suite_test where suite_id = %s", (suite_id,))
|
||||
for row in c.fetchall():
|
||||
self.tests.append(row[0])
|
||||
self.tests.append((row[0], row[1]))
|
||||
|
||||
def export(self):
|
||||
return { "id": self.id,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,66 @@
|
|||
# vim: set sts=4 ts=8 sw=4 tw=99 et:
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
import awfy
|
||||
|
||||
awfy.Startup()
|
||||
|
||||
c = awfy.db.cursor()
|
||||
c.execute("show tables;")
|
||||
tables = set()
|
||||
for row in c.fetchall():
|
||||
tables.add(row[0])
|
||||
|
||||
if not 'awfy_suite_test' in tables:
|
||||
c.execute("""
|
||||
create table if not exists `awfy_suite_test` (
|
||||
`id` int(10) unsigned not null auto_increment,
|
||||
`suite_id` int(11) not null,
|
||||
`name` varchar(128) not null,
|
||||
primary key(`id`),
|
||||
unique key `suite_id` (`suite_id`, `name`)
|
||||
) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=1;
|
||||
""")
|
||||
|
||||
awfy_breakdown_columns = set()
|
||||
c.execute("show columns from awfy_breakdown")
|
||||
for row in c.fetchall():
|
||||
awfy_breakdown_columns.add(row[0])
|
||||
|
||||
if 'test_id' not in awfy_breakdown_columns:
|
||||
print('Creating new column on awfy_breakdown_columns...')
|
||||
c.execute("alter table `awfy_breakdown` add `test_id` int unsigned not null")
|
||||
|
||||
if 'test' in awfy_breakdown_columns:
|
||||
print('Finding tests...')
|
||||
|
||||
suite_map = {}
|
||||
c.execute("select distinct suite_id, test from awfy_breakdown where test is not null")
|
||||
for row in c.fetchall():
|
||||
suite_id = row[0]
|
||||
test_name = row[1]
|
||||
if suite_id not in suite_map:
|
||||
suite_map[suite_id] = {}
|
||||
test_map = suite_map[suite_id]
|
||||
c.execute("select id from awfy_suite_test where suite_id = %s and name = %s",
|
||||
(suite_id, test_name))
|
||||
result = c.fetchone()
|
||||
if not result:
|
||||
c.execute("insert into awfy_suite_test (suite_id, name) values (%s, %s)",
|
||||
(suite_id, test_name))
|
||||
test_id = c.lastrowid
|
||||
else:
|
||||
test_id = result[0]
|
||||
test_map[test_name] = test_id
|
||||
|
||||
for suite_id in suite_map:
|
||||
test_map = suite_map[suite_id]
|
||||
for test_name in test_map:
|
||||
test_id = test_map[test_name]
|
||||
print('Updating columns for suite_id {0} test {1} to test_id {2}'.format(suite_id, test_name, test_id))
|
||||
c.execute("update awfy_breakdown set test_id = %s where suite_id = %s and test = %s",
|
||||
(test_id, suite_id, test_name))
|
||||
|
||||
print('Dropping old column...')
|
||||
c.execute("alter table `awfy_breakdown` drop `test`")
|
||||
|
|
@ -51,7 +51,7 @@ def fetch_test_scores(machine_id, suite_id, name, earliest_run_id):
|
|||
JOIN fast_run r ON s.run_id = r.id \
|
||||
JOIN awfy_build b ON (s.run_id = b.run_id AND s.mode_id = b.mode_id) \
|
||||
WHERE s.suite_id = %s \
|
||||
AND s.test = %s \
|
||||
AND s.test_id = %s \
|
||||
AND r.status = 1 \
|
||||
AND r.machine = %s \
|
||||
AND r.id > %s \
|
||||
|
|
@ -254,11 +254,11 @@ def update(cx, machine, suite):
|
|||
if not new_rows:
|
||||
return
|
||||
|
||||
for test in suite.tests:
|
||||
for test_id, test_name in suite.tests:
|
||||
def fetch_test(earliest_run_id):
|
||||
return fetch_test_scores(machine.id, suite.id, test, earliest_run_id)
|
||||
return fetch_test_scores(machine.id, suite.id, test_id, earliest_run_id)
|
||||
|
||||
prefix = 'bk-raw-' + suite.name + '-' + test + '-' + str(machine.id)
|
||||
prefix = 'bk-raw-' + suite.name + '-' + test_name + '-' + str(machine.id)
|
||||
perform_update(cx, suite, prefix, fetch_test)
|
||||
|
||||
def export_master(cx):
|
||||
|
|
|
|||
|
|
@ -49,10 +49,11 @@ if (isset($_GET['run']) && $_GET['run'] == 'yes') {
|
|||
($run, $suite_id, $mode_id, $time)")
|
||||
or die("ERROR: " . mysql_error());
|
||||
} else {
|
||||
$test_id = find_or_add_test($suite_id, $name);
|
||||
mysql_query("INSERT INTO awfy_breakdown
|
||||
(run_id, suite_id, mode_id, test, score)
|
||||
(run_id, suite_id, mode_id, score, test_id)
|
||||
VALUES
|
||||
($run, $suite_id, $mode_id, '$name', $time)")
|
||||
($run, $suite_id, $mode_id, $time, $test_id)")
|
||||
or die("ERROR: " . mysql_error());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -57,6 +57,19 @@ function find_suite($suite)
|
|||
return intval($row[0]);
|
||||
}
|
||||
|
||||
function find_or_add_test($suite_id, $name)
|
||||
{
|
||||
$query = "select id from awfy_suite_test where suite_id = $suite_id and name = '$name'";
|
||||
$results = mysql_query($query);
|
||||
if (!$results || mysql_num_rows($results) < 1) {
|
||||
$query = "insert into awfy_suite_test (suite_id, name) values($suite_id, $name)";
|
||||
mysql_query($query);
|
||||
return mysql_insert_id();
|
||||
}
|
||||
$row = mysql_fetch_array($results);
|
||||
return intval($row[0]);
|
||||
}
|
||||
|
||||
function awfy_query($query)
|
||||
{
|
||||
$result = mysql_query($query) or die(mysql_error());
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче