diff --git a/database/schema.sql b/database/schema.sql index 69ad732..b6687b6 100644 --- a/database/schema.sql +++ b/database/schema.sql @@ -86,6 +86,14 @@ CREATE TABLE `awfy_suite` ( UNIQUE KEY `name_UNIQUE` (`name`) ) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=9 ; +CREATE TABLE IF NOT EXISTS `awfy_suite_test` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `suite_id` int(11) NOT NULL, + `name` varchar(128) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `suite_id` (`suite_id`,`name`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=1 ; + -- -- Dumping data for table `awfy_suite` -- @@ -156,14 +164,13 @@ CREATE TABLE `awfy_breakdown` ( `run_id` int(11) DEFAULT NULL, `suite_id` int(11) DEFAULT NULL, `mode_id` int(11) DEFAULT NULL, - `test` varchar(45) DEFAULT NULL, `score` varchar(45) DEFAULT NULL, + `test_id` int(10) DEFAULT NULL, PRIMARY KEY (`id`), KEY `run_id` (`run_id`), KEY `suite_id` (`suite_id`), KEY `mode_id` (`mode_id`), - KEY `suite_id_2` (`suite_id`,`test`) -) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=7636046 ; +) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=1 ; -- -------------------------------------------------------- @@ -178,7 +185,7 @@ CREATE TABLE `awfy_build` ( `cset` varchar(256) NOT NULL, PRIMARY KEY (`id`), UNIQUE KEY `index2` (`run_id`,`mode_id`) -) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=136734 ; +) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=1 ; -- -------------------------------------------------------- @@ -195,7 +202,7 @@ CREATE TABLE `awfy_machine` ( `last_checked` int(10) unsigned NOT NULL, `contact` mediumtext NOT NULL PRIMARY KEY (`id`) -) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=16 ; +) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=1 ; -- -------------------------------------------------------- @@ -212,7 +219,7 @@ CREATE TABLE `awfy_score` ( PRIMARY KEY (`id`), KEY `run_id` (`run_id`), KEY `mode_id` (`mode_id`) -) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=480043 ; +) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=1 ; -- -------------------------------------------------------- @@ -228,5 +235,5 @@ CREATE TABLE `fast_run` ( `status` int(11) NOT NULL, `error` mediumtext NOT NULL, PRIMARY KEY (`id`) -) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=33848 ; +) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=1 ; diff --git a/server/condenser.py b/server/condenser.py index 498e74d..5c786d9 100644 --- a/server/condenser.py +++ b/server/condenser.py @@ -9,10 +9,20 @@ import sys import awfy import json from profiler import Profiler +from datetime import datetime SecondsPerDay = 60 * 60 * 24 MaxRecentRuns = 30 +def should_export(name, when): + path = os.path.join(awfy.path, name) + if not os.path.exists(path): + return True + now = datetime.now() + if now.year == when[0] and now.month == when[1]: + return True + return False + def export(name, j): path = os.path.join(awfy.path, name) if os.path.exists(path): @@ -208,6 +218,11 @@ def condense(cx, suite, prefix, name): for when, graph in graphs: new_name = prefix + 'condensed-' + name + '-' + str(when[0]) + '-' + str(when[1]) + + # Don't condense if it already exists... + if not should_export(new_name + '.json', when): + continue + sys.stdout.write('Condensing ' + new_name + '... ') sys.stdout.flush() with Profiler() as p: @@ -233,13 +248,13 @@ def condense_suite(cx, machine, suite): # the combine graph back to our caller. suite_aggregate = condense(cx, suite, '', name) - for test in suite.tests: - test_name = suite.name + '-' + test + '-' + str(machine.id) - test_aggregate = condense(cx, suite, 'bk-', test_name) + for test_id, test_name in suite.tests: + test_path = suite.name + '-' + test_name + '-' + str(machine.id) + test_aggregate = condense(cx, suite, 'bk-', test_path) j = { 'version': awfy.version, 'graph': test_aggregate } - export('bk-aggregate-' + test_name + '.json', j) + export('bk-aggregate-' + test_path + '.json', j) return suite_aggregate diff --git a/server/data.py b/server/data.py index 1740138..f8f2420 100644 --- a/server/data.py +++ b/server/data.py @@ -17,12 +17,9 @@ class Benchmark(object): # Get a list of individual tests self.tests = [] c = awfy.db.cursor() - c.execute("SELECT test FROM awfy_breakdown \ - WHERE suite_id = %s \ - GROUP BY test", - [suite_id]) + c.execute("select id, name from awfy_suite_test where suite_id = %s", (suite_id,)) for row in c.fetchall(): - self.tests.append(row[0]) + self.tests.append((row[0], row[1])) def export(self): return { "id": self.id, diff --git a/server/db_upgrade.py b/server/db_upgrade.py new file mode 100644 index 0000000..c8516f5 --- /dev/null +++ b/server/db_upgrade.py @@ -0,0 +1,66 @@ +# vim: set sts=4 ts=8 sw=4 tw=99 et: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import awfy + +awfy.Startup() + +c = awfy.db.cursor() +c.execute("show tables;") +tables = set() +for row in c.fetchall(): + tables.add(row[0]) + +if not 'awfy_suite_test' in tables: + c.execute(""" + create table if not exists `awfy_suite_test` ( + `id` int(10) unsigned not null auto_increment, + `suite_id` int(11) not null, + `name` varchar(128) not null, + primary key(`id`), + unique key `suite_id` (`suite_id`, `name`) + ) ENGINE=MyISAM DEFAULT CHARSET=latin1 AUTO_INCREMENT=1; + """) + +awfy_breakdown_columns = set() +c.execute("show columns from awfy_breakdown") +for row in c.fetchall(): + awfy_breakdown_columns.add(row[0]) + +if 'test_id' not in awfy_breakdown_columns: + print('Creating new column on awfy_breakdown_columns...') + c.execute("alter table `awfy_breakdown` add `test_id` int unsigned not null") + +if 'test' in awfy_breakdown_columns: + print('Finding tests...') + + suite_map = {} + c.execute("select distinct suite_id, test from awfy_breakdown where test is not null") + for row in c.fetchall(): + suite_id = row[0] + test_name = row[1] + if suite_id not in suite_map: + suite_map[suite_id] = {} + test_map = suite_map[suite_id] + c.execute("select id from awfy_suite_test where suite_id = %s and name = %s", + (suite_id, test_name)) + result = c.fetchone() + if not result: + c.execute("insert into awfy_suite_test (suite_id, name) values (%s, %s)", + (suite_id, test_name)) + test_id = c.lastrowid + else: + test_id = result[0] + test_map[test_name] = test_id + + for suite_id in suite_map: + test_map = suite_map[suite_id] + for test_name in test_map: + test_id = test_map[test_name] + print('Updating columns for suite_id {0} test {1} to test_id {2}'.format(suite_id, test_name, test_id)) + c.execute("update awfy_breakdown set test_id = %s where suite_id = %s and test = %s", + (test_id, suite_id, test_name)) + + print('Dropping old column...') + c.execute("alter table `awfy_breakdown` drop `test`") diff --git a/server/update.py b/server/update.py index 3d8ef82..a7a177f 100644 --- a/server/update.py +++ b/server/update.py @@ -51,7 +51,7 @@ def fetch_test_scores(machine_id, suite_id, name, earliest_run_id): JOIN fast_run r ON s.run_id = r.id \ JOIN awfy_build b ON (s.run_id = b.run_id AND s.mode_id = b.mode_id) \ WHERE s.suite_id = %s \ - AND s.test = %s \ + AND s.test_id = %s \ AND r.status = 1 \ AND r.machine = %s \ AND r.id > %s \ @@ -254,11 +254,11 @@ def update(cx, machine, suite): if not new_rows: return - for test in suite.tests: + for test_id, test_name in suite.tests: def fetch_test(earliest_run_id): - return fetch_test_scores(machine.id, suite.id, test, earliest_run_id) + return fetch_test_scores(machine.id, suite.id, test_id, earliest_run_id) - prefix = 'bk-raw-' + suite.name + '-' + test + '-' + str(machine.id) + prefix = 'bk-raw-' + suite.name + '-' + test_name + '-' + str(machine.id) perform_update(cx, suite, prefix, fetch_test) def export_master(cx): diff --git a/website/UPDATE.php b/website/UPDATE.php index 416e964..44860b7 100644 --- a/website/UPDATE.php +++ b/website/UPDATE.php @@ -49,10 +49,11 @@ if (isset($_GET['run']) && $_GET['run'] == 'yes') { ($run, $suite_id, $mode_id, $time)") or die("ERROR: " . mysql_error()); } else { + $test_id = find_or_add_test($suite_id, $name); mysql_query("INSERT INTO awfy_breakdown - (run_id, suite_id, mode_id, test, score) + (run_id, suite_id, mode_id, score, test_id) VALUES - ($run, $suite_id, $mode_id, '$name', $time)") + ($run, $suite_id, $mode_id, $time, $test_id)") or die("ERROR: " . mysql_error()); } } diff --git a/website/internals.php b/website/internals.php index 349e1bc..366bd90 100755 --- a/website/internals.php +++ b/website/internals.php @@ -57,6 +57,19 @@ function find_suite($suite) return intval($row[0]); } +function find_or_add_test($suite_id, $name) +{ + $query = "select id from awfy_suite_test where suite_id = $suite_id and name = '$name'"; + $results = mysql_query($query); + if (!$results || mysql_num_rows($results) < 1) { + $query = "insert into awfy_suite_test (suite_id, name) values($suite_id, $name)"; + mysql_query($query); + return mysql_insert_id(); + } + $row = mysql_fetch_array($results); + return intval($row[0]); +} + function awfy_query($query) { $result = mysql_query($query) or die(mysql_error());