mozilla
/
arewefastyet
зеркало из
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Update towards having out of order runs. In the meantime organize php code better

This commit is contained in:
Hannes Verschore 2015-10-30 17:53:42 +01:00
Родитель dabba6ac18
Коммит 3d608ab128
13 изменённых файлов: 426 добавлений и 82 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

49
slave/submitter.py
Просмотреть файл

@ -67,6 +67,27 @@ class RemoteSubmitter(Submitter):
except urllib2.URLError:
self.runIds[i] = None
def startOutOfOrder(self, mode, revision, run_before, run_after):
self.assertMachine()
for i in range(len(self.urls)):
try:
url = self.urls[i]
url += '?run=ooo'
url += '&MACHINE=' + str(self.machine)
url += '&mode=' + str(mode)
url += '&revision=' + str(revision)
url += '&run_before_id=' + str(run_before)
url += '&run_after_id=' + str(run_after)
url = urllib2.urlopen(url)
contents = url.read()
m = re.search('id=(\d+)', contents)
if m == None:
self.runIds[i] = None
else:
self.runIds[i] = int(m.group(1))
except urllib2.URLError:
self.runIds[i] = None
def createBuild(self, engine_type, config, cset):
mode = self.mode(engine_type, config)
for i in range(len(self.urls)):
@ -190,12 +211,25 @@ if __name__ == "__main__":
from optparse import OptionParser
parser = OptionParser(usage="usage: %prog [options]")
parser.add_option("-f", "--finish", action="store_true", dest="finish", default=False)
parser.add_option("-s", "--session", dest="session", type="string")
# create a session arguments
parser.add_option("-c", "--create", action="store_true", dest="create", default=False)
parser.add_option("-o", "--output", dest="output", type="string",
help="The place to write the session too.")
parser.add_option("-m", "--machine", dest="machine", type="int",
help="Give the machine number to submit to.")
parser.add_option("--mode", dest="mode", type="string",
help="OutOfOrder: Give the mode")
parser.add_option("--revision", dest="rev", type="string",
help="OutOfOrder: Give the revision")
parser.add_option("--run_before", dest="run_before", type="id",
help="OutOfOrder: Give the run id whereafter this new run needs to come")
parser.add_option("--run_after", dest="run_after", type="id",
help="OutOfOrder: Give the run id before which this new run needs to come")
# all other cases need session info
parser.add_option("-s", "--session", dest="session", type="string")
parser.add_option("-f", "--finish", action="store_true", dest="finish", default=False)
(options, args) = parser.parse_args()
utils.config.init("awfy.config")
@ -203,8 +237,13 @@ if __name__ == "__main__":
if options.create:
submitter = RemoteSubmitter()
submitter.setMachine(options.machine)
submitter.start()
print json.dumps(submitter.runIds)
if options.mode:
submitter.startOutOfOrder(options.mode, options.rev, options.run_before, options.run_after)
else:
submitter.start()
fp = open(options.output, "w")
json.dump(submitter.runIds, fp)
elif options.finish:
fp = open(options.session, "r")
session = json.load(fp)

96
website/UPDATE.php
Просмотреть файл

@ -7,61 +7,77 @@ require_once("internals.php");
init_database();
require_once("lib/DB/Run.php");
require_once("lib/DB/Build.php");
require_once("lib/DB/Mode.php");
require_once("lib/RunReporter.php");
// Start a full benchmark run. Request a token/number used to report/group
// benchmark scores.
if (isset($_GET['run']) && $_GET['run'] == 'yes') {
// TODO: sort_order is not a date anymore. Adjust this logic to not take an
// order, but only set this afterwards.
$MACHINE = GET_int('MACHINE');
$stamp = GET_int('stamp');
if (GET_string("run") == 'yes') {
$machine_id = GET_int('MACHINE');
$query = mysql_query("SELECT max(sort_order) as maximum
FROM awfy_run
WHERE machine = '".$MACHINE."'") or die(mysql_error());
$run = mysql_fetch_object($query);
$run = RunReporter::createForMachine($machine_id);
echo "id=".$run->id;
die();
}
// Start an out of order run. Retriggering a particular mode.
if (GET_string("run") == 'ooo') {
$machine_id = GET_int('MACHINE');
$mode = Mode::FromMode(GET_string('name'));
$revision = GET_string('revision');
$run_before_id = GET_int('run_before_id');
$run_after_id = GET_int('run_after_id');
$run = RunReporter::createOutOfOrder($machine_id, $mode->id, $revision,
$run_before_id, $run_after_id);
echo "id=".$run->id;
mysql_query("INSERT INTO awfy_run (machine, sort_order, approx_stamp)
VALUES
($MACHINE, {$run->maximum}, UNIX_TIMESTAMP())")
or die("ERROR: " . mysql_error());
print("id=" . mysql_insert_id());
die();
}
// Finish a full benchmark run. Scores will only become visible from now on
// (when status equals 1).
if (isset($_GET['run']) && $_GET['run'] == 'finish') {
$runid = GET_run_id('runid');
if (GET_string("run") == 'finish') {
$run_id = GET_int('runid');
$status = GET_int('status');
if (isset($_GET['error']))
$error = '\'' . mysql_real_escape_string(GET_string('error')) . '\'';
else
$error = 'NULL';
mysql_query("UPDATE awfy_run
SET status = $status,
error = $error,
finish_stamp = UNIX_TIMESTAMP()
WHERE id = $runid")
or die("ERROR: " . mysql_error());
$run = new Run($run_id);
if ($run->isFinished() || $run->hasError())
throw new Error("Run was already finished or error'ed");
$error = GET_string('error');
$run->finish($status, $error);
die();
}
if (isset($_GET['run']) && $_GET['run'] == 'addEngine') {
$runid = GET_run_id('runid');
$mode_id = find_mode(GET_string('name'));
$cset = mysql_real_escape_string(GET_string('cset'));
mysql_query("INSERT INTO awfy_build
(run_id, mode_id, cset)
VALUES
($runid, $mode_id, '$cset')")
or die("ERROR: " . mysql_error());
if (GET_string("run") == 'addEngine') {
$run = new Run(GET_int('runid'));
$revision = GET_string('cset');
$mode = Mode::FromMode(GET_string('name'));
if ($run->isFinished() || $run->hasError())
throw new Error("Run was already finished or error'ed");
if ($run->isOutOfOrder()) {
// out of order builds cannot add extra modes. The available
// mode have already been added.
if (!Build::withRunAndMode($run->id, $mode->id))
$run->finish(0, "Tried to add extra modes to out of order run.");
die();
}
Build::insert($run, $mode->id, $revision);
die();
}
// Report that a slave is still awake when there are no benchmarks results
// to send.
if (isset($_GET['awake']) && $_GET['awake'] == 'yes') {
if (GET_string('awake') == 'yes') {
$MACHINE = GET_int('MACHINE');
mysql_query("UPDATE awfy_machine
SET last_checked = UNIX_TIMESTAMP()
@ -71,12 +87,16 @@ if (isset($_GET['awake']) && $_GET['awake'] == 'yes') {
}
// Report score of a benchmark total or subtest.
$run_id = GET_int('run');
$run = new Run($run_id);
if ($run->isFinished() || $run->hasError())
throw new Error("Run was already finished or error'ed");
$time = mysql_real_escape_string(GET_string('time'));
$mode_id = find_mode(GET_string('mode'));
$run = GET_run_id('run');
$version = GET_string('suite');
$score = GET_int('score');
$build = find_build($run, $mode_id);
$build = find_build($run_id, $mode_id);
if (isset($_GET['version']))
$version = GET_string('version');
$suite_version_id = find_or_add_suite_version(GET_string('suite'), $version);

9
website/internals.php
Просмотреть файл

@ -206,14 +206,5 @@ function awfy_query($query)
return $result;
}
function GET_run_id($name)
{
$runid = GET_int($name);
$results = mysql_query("SELECT id from awfy_run WHERE id = $runid AND status = 0");
if (!$results || mysql_num_rows($results) < 1)
return 0;
return $runid;
}
// Init
$config = new Config("/etc/awfy-server.config");

8
website/lib/BashInterpreter.php
Просмотреть файл

@ -3,8 +3,8 @@
class BashInterpreter {
public static function matchCommand($text, $command) {
$command = str_replace("/", "\/", $command);
preg_match_all("/".$command." .*[;|$|\r\n]/", $text, $matches);
return $matches[0];
preg_match_all("/(".$command." .*)[;$\r\n#]/", $text, $matches);
return $matches[1];
}
public static function matchFlag($command, $flag) {
@ -23,6 +23,10 @@ class BashInterpreter {
return str_replace($command, $command." ".$full_flag, $text);
}
public static function addFlagToCommand($text, $full_command, $full_flag) {
return str_replace($full_command, $full_command." ".$full_flag, $text);
}
public static function removeFlagFromCommands($text, $command, $full_flag) {
foreach(BashInterpreter::matchCommand($text, $command) as $match) {
$text = BashInterpreter::removeFlagFromCommand($text, $match, $full_flag);

45
website/lib/DB/Build.php Normal file
Просмотреть файл

@ -0,0 +1,45 @@
<?php
require_once("DB.php");
class Build {
public static $db = "awfy_build";
public function __construct($id) {
$this->id = $id;
}
public static function withRunAndMode($run_id, $mode_id) {
$qBuild = mysql_query("SELECT id FROM awfy_build
WHERE run_id = $run_id AND
mode_id = $mode_id
LIMIT 1") or die(mysql_error());
$build = mysql_fetch_object($qBuild);
return new Build($build->id);
}
public static function insert($run, $mode_id, $revision) {
if ($run->isFinished())
throw new Exception("Cannot info to a run that is finished.");
mysql_query("INSERT INTO awfy_build
(run_id, mode_id, revision)
VALUES
({$run->id}, $mode_id, '".mysql_real_escape_string($revision)."')")
or die("ERROR: " . mysql_error());
return new Build(mysql_insert_id());
}
public function revision() {
return $this->select("revision");
}
public function run_id() {
return $this->select("run_id");
}
public function mode_id() {
return $this->select("mode_id");
}
}

12
website/lib/DB/DB.php Normal file
Просмотреть файл

@ -0,0 +1,12 @@
<?php
class DB {
public function select($field) {
$qField = mysql_query("SELECT $field as field from {$this::$db}
WHERE id = {$this->id}") or die(mysql_error());
$field = mysql_fetch_object($qField);
return $field->field;
}
}

41
website/lib/DB/Mode.php
Просмотреть файл

@ -1,30 +1,35 @@
<?php
class Mode {
require_once("Vendor.php");
require_once("DB.php");
// db: awfy_mode
function __construct($id) {
class Mode extends DB {
public static $db = "awfy_mode";
public function __construct($id) {
$this->id = $id;
}
function vendor() {
$qMode = mysql_query("SELECT vendor_id
FROM awfy_mode
WHERE id = {$this->id}");
$mode = mysql_fetch_object($qMode);
return $mode->vendor_id;
public static function FromMode($mode) {
$query = "SELECT id FROM awfy_mode
WHERE mode = '" . mysql_real_escape_string($mode) . "'";
$results = mysql_query($query) or die(mysql_error());
if (!$results || mysql_num_rows($results) < 1)
return null;
$row = mysql_fetch_array($results);
return new Mode($row[0]);
}
function mode() {
$qMode = mysql_query("SELECT mode
FROM awfy_mode
WHERE id = {$this->id}");
$mode = mysql_fetch_object($qMode);
return $mode->mode;
public function vendor_id() {
return $this->select("vendor_id");
}
function id() {
return $this->id;
public function vendor() {
return new Vendor($this->vendor_id());
}
public function mode() {
return $this->select("mode");
}
}

73
website/lib/DB/Run.php Normal file
Просмотреть файл

@ -0,0 +1,73 @@
<?php
require_once("DB.php");
class Run extends DB {
public static $db = "awfy_run";
public function __construct($id) {
$this->id = $id;
}
public static function insert($machine_id, $sort_order, $approx_stamp = 0) {
$approx_stamp = intval($approx_stamp);
if ($approx_stamp == 0)
$approx_stamp = "UNIX_TIMESTAMP()";
mysql_query("INSERT INTO awfy_run
(machine, sort_order, approx_stamp)
VALUES
($machine_id, $sort_order, $approx_stamp)")
or die("ERROR: " . mysql_error());
return new Run(mysql_insert_id());
}
public function finish($status, $error = "") {
if (empty($error))
$error = "NULL";
else
$error = "'" . mysql_real_escape_string($error) . "'";
mysql_query("UPDATE awfy_run
SET status = $status,
error = $error,
finish_stamp = UNIX_TIMESTAMP()
WHERE id = $runid")
or die("ERROR: " . mysql_error());
}
public function isFinished() {
return $this->select("status") > 0;
}
public function isOutOfOrder() {
return !!$this->select("out_of_order");
}
public function hasError() {
return $this->select("error") != "";
}
public function sort_order() {
return $this->select("sort_order");
}
public function approx_stamp() {
return $this->select("approx_stamp");
}
public function builds() {
$qRun = mysql_query("SELECT approx_stamp from awfy_builds
WHERE id = {$this->id}") or die(mysql_error());
$run = mysql_fetch_object($qRun);
return $run->approx_stamp;
}
public function isBuildInfoComplete() {
// The set of builds cannot change anymore, when the run is finished
// or if this run is an out of order run. Out of order runs immediately
// add their build info.
return $this->isFinished() || $this->isOutOfOrder() || $this->hasError();
}
}

18
website/lib/DB/Vendor.php Normal file
Просмотреть файл

@ -0,0 +1,18 @@
<?php
class Vendor {
// db: awfy_mode
function __construct($id) {
$this->id = $id;
}
function csetURL() {
$qVendor = mysql_query("SELECT csetURL
FROM awfy_vendor
WHERE id = {$this->id}");
$vendor = mysql_fetch_object($qVendor);
return $vendor->csetURL;
}
}

26
website/lib/ManipulateTask.php
Просмотреть файл

@ -38,7 +38,6 @@ class ManipulateTask extends Task {
$source_matches = BashInterpreter::matchFlag($command, "-s");
$source_rules = $this->source_rules();
$engine = $source_rules[$source_matches[0]];
print_r($removed_engines);
if (in_array($engine, $removed_engines)) {
$this->removeBuildOrDownloadCommand($command);
}
@ -63,16 +62,31 @@ class ManipulateTask extends Task {
$this->update_configs($configs);
}
public function setTipRevision() {
$this->removeRevisionInfo();
public function setBuildRevisionToTip() {
$this->removeBuildRevisionInfo();
}
public function setRevision($new_revision) {
$this->removeRevisionInfo();
public function setBuildRevision($new_revision) {
$this->removeBuildRevisionInfo();
$this->task = BashInterpreter::addFlagToCommands($this->task, "python build.py", "-r ".$new_revision);
}
private function removeRevisionInfo() {
public function setSubmitterOutOfOrder($mode_name, $revision, $run_before_id, $run_after_id) {
$commands = BashInterpreter::matchCommand($this->task, "python submitter.py");
foreach ($commands as $command) {
if (count(BashInterpreter::matchFlag($command, "-c")) > 0 ||
count(BashInterpreter::matchFlag($command, "--create")) > 0)
{
$flag = "--revision ".$revision." ";
$flag .= "--mode ".$mode_name." ";
$flag .= "--run_before ".$run_before_id." ";
$flag .= "--run_after ".$run_after_id;
$this->task = BashInterpreter::addFlagToCommand($this->task, $command, $flag);
}
}
}
private function removeBuildRevisionInfo() {
$commands = BashInterpreter::matchCommand($this->task, "python download.py");
if (count($commands) != 0)
throw new Exception("Not yet supported to specify revision for downloaded builds.");

11
website/lib/RetriggerController.php
Просмотреть файл

@ -33,7 +33,7 @@ class RetriggerController {
$task = new ManipulateTask($task->task);
if ($mode_id != 0)
$task->update_modes($mode->mode());
$task->update_modes(Array($mode->mode()));
$retrigger->tasks[] = $task;
@ -45,9 +45,13 @@ class RetriggerController {
return $retrigger;
}
public function setRevision($revision) {
public function convertToRevision($mode_id, $revision, $run_before_id, $run_after_id) {
$mode = new Mode($mode_id);
foreach ($this->tasks as $task) {
$task->setRevision($revision);
$task->update_modes(Array("jmim"/*$mode->mode()*/));
$task->setBuildRevision($revision);
$task->setSubmitterOutOfOrder("jmim"/*$mode->mode()*/, $revision, $run_before_id, $run_after_id);
}
}
@ -62,4 +66,3 @@ class RetriggerController {
}
}
}

78
website/lib/RunReporter.php Normal file
Просмотреть файл

@ -0,0 +1,78 @@
<?php
require_once("DB/Run.php");
require_once("DB/Mode.php");
require_once("DB/Build.php");
class RunReporter {
public static function createForMachine($machine_id) {
$query = mysql_query("SELECT max(sort_order) as maximum
FROM awfy_run
WHERE machine = $machine_id") or die(mysql_error());
$run = mysql_fetch_object($query);
return Run::insert($machine_id, $run->maximum);
}
public static function createOutOfOrder($machine_id, $mode_id, $revision,
$run_before_id, $run_after_id)
{
$run_before = new Run($run_before_id);
$run_after = new Run($run_after_id);
// Find the sorting order where we could add this revision;
$sort_order = RunReporter::findSortOrder($run_before, $mode_id, $revision);
// sanity check.
if ($sort_order >= run_after->sort_order())
throw new Exception("Given run bounds were incorrect.");
// Create a space at the given sort_order, by shifting all sort_order,
// equal or higher than the given sort_order.
RunReporter::increaseNextSortOrder($machine_id, $sort_order);
$run = Run::insert($machine_id, $sort_order, $run_before->approx_stamp());
$build = Build::insert($run, $mode_id, $revision);
return $run;
}
private static function findSortOrder($run, $mode_id, $revision)
{
$version_control = VersionControl::forMode($mode_id);
while (True) {
if (!$run->isBuildInfoComplete())
throw new Exception("Encountered an incomplete run.");
$build = Build::withRunAndMode($run_before->id, $mode_id);
// We can safely ignore runs that have no results with the requested mode_id
if (!$build) {
$run = $run->next();
continue;
}
// We skip to the next run, if revisions are the same.
// To make sure that new runs are shown after existing ones.
if ($version_control->equal($build->revision(), $revision)) {
$run = $run->next();
continue;
}
// Using version control take a peek if the revision
// is later/earlier than this one.
if ($version_control->isAfter($build->revision(), $revision))
return $run->sort_order();
$run = $run->next();
}
}
private static function increaseNextSortOrder($machine_id, $sort_order) {
mysql_query("UPDATE awfy_run
SET sort_order = sort_order + 1
WHERE machine = $machine_id AND
sort_order >= $sort_order") or die(mysql_error());
}
}

42
website/lib/VersionControl.php Normal file
Просмотреть файл

@ -0,0 +1,42 @@
<?php
class VersionControl {
public function forMode($mode_id) {
$mode = new Mode($mode_id);
$vendor = new Vendor($mode->vendor_id());
$url = $vendor->csetURL();
if (strpos($url, "hg.mozilla.org") !== false)
return new HGWeb($url);
throw new Exception("Not implemented version control system.");
}
}
class HGWeb {
public function __construct($url) {
$this->url = str_replace("/rev/", "/", $url);
}
public function equal($revision1, $revision2) {
return $revision1 == $revision2;
}
public function isAfter($revision1, $revision2) {
// test if is before
$html = file_get_contents($this->url."pushloghtml?fromchange=$revision1&tochange=$revision2");
if (strpos($html, "pushlogentry") !== false)
return false;
// test if is after
$html = file_get_contents($this->url."pushloghtml?fromchange=$revision2&tochange=$revision1");
if (strpos($html, "pushlogentry") !== false)
return true;
throw new Exception("Could find relationship between $revision1 and $revision2.");
}
}