diff --git a/detector/improve_detection.py b/detector/improve_detection.py
new file mode 100644
index 0000000..54a6a33
--- /dev/null
+++ b/detector/improve_detection.py
@@ -0,0 +1,107 @@
+# vim: set ts=4 sw=4 tw=99 et:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import awfy
+import tables
+
+RUNS = 20
+
+def isOutliner(item):
+ score = item.score()
+ nexts = [i.get("score") for i in score.nexts_no_outliners(RUNS)]
+ prevs = [i.get("score") for i in score.prevs_no_outliners(RUNS)]
+ if len(nexts) != RUNS:
+ return False
+ if len(prevs) != RUNS:
+ return False
+ avg_nexts = sum(nexts)/len(nexts)
+ avg_prevs = sum(prevs)/len(prevs)
+
+ if abs(avg_nexts - avg_prevs) < score.noise() / tables.NOISE_FACTOR:
+ return True
+ return False
+
+def isEmptyRegression(item):
+ score = item.score()
+ if score.prev().get("build").get("cset") == score.get("build").get("cset"):
+ return True
+ return False
+
+def isBiModal(item):
+ def divide(li):
+ h = max(li)
+ l = min(li)
+ highs = []
+ lows = []
+ for i in li:
+ if h-i < i-l:
+ highs.append(i)
+ else:
+ lows.append(i)
+ return highs, lows
+
+
+ score = item.score()
+ nexts = [i.get("score") for i in score.nexts(RUNS)]
+ prevs = [i.get("score") for i in score.prevs(RUNS)]
+ nexts_h, nexts_l = divide(nexts)
+ prevs_h, prevs_l = divide(prevs)
+
+ if len(nexts_h) < 4:
+ return False
+ if len(nexts_l) < 4:
+ return False
+ if len(prevs_h) < 4:
+ return False
+ if len(prevs_l) < 4:
+ return False
+ avg_nexts_h = sum(nexts_h)/len(nexts_h)
+ avg_nexts_l = sum(nexts_l)/len(nexts_l)
+ avg_prevs_h = sum(prevs_h)/len(prevs_h)
+ avg_prevs_l = sum(prevs_l)/len(prevs_l)
+
+ if abs(avg_nexts_h - avg_prevs_h) >= score.noise() / tables.NOISE_FACTOR:
+ return False
+ if abs(avg_nexts_l - avg_prevs_l) >= score.noise() / tables.NOISE_FACTOR:
+ return False
+ return True
+
+modes = [14,16,20,21,22,23,25,26,27,28,29,31,32,33,35]
+for regression in tables.Regression.where({'status':'unconfirmed'}):
+ if regression.get("build").get("mode_id") not in modes:
+ continue
+
+ allRemoved = True
+ for item in regression.regressions():
+ if item.get("noise"):
+ continue
+
+ print "item"
+
+ if isEmptyRegression(item):
+ print "remove", regression.id
+ item.score().dump()
+ item.update({"noise":"1"})
+ continue
+
+ if isOutliner(item):
+ print "remove", regression.id
+ item.score().dump()
+ item.update({"noise":"1"})
+ continue
+
+ if isBiModal(item):
+ print "remove", regression.id
+ item.score().dump()
+ item.update({"noise":"1"})
+ continue
+
+ allRemoved = False
+
+ if allRemoved:
+ print "remove item"
+ regression.update({"status":"noise"})
+
+awfy.db.commit()
diff --git a/detector/regression_detector.py b/detector/regression_detector.py
index b41b1e1..effe3f7 100644
--- a/detector/regression_detector.py
+++ b/detector/regression_detector.py
@@ -12,7 +12,7 @@ from optparse import OptionParser
parser = OptionParser(usage="usage: %prog [options]")
parser.add_option( "--dry-run", dest="dryrun", action="store_true", default=False,
- help="Don't cmomit the new regressions to the database yet.")
+ help="Don't commit the new regressions to the database yet.")
(options, args) = parser.parse_args()
def notProcessedRuns():
@@ -103,16 +103,30 @@ if __name__ == "__main__":
score.dump()
if not options.dryrun:
build = score.get("build_id")
- try:
- id_ = tables.Regression.insert({"build_id": build})
- tables.RegressionStatus.insert({"regression_id": id_, "name": "awfy", "extra": "Submitted", "stamp":"UNIX_TIMESTAMP()"})
- except:
- pass
+ prev_build = score.prev().get("build_id")
+ regression = [regression for regression in tables.Regression.where({
+ "build_id": build,
+ "prev_build_id": prev_build
+ })]
+ if len(regression) == 0:
+ regression_id = tables.Regression.insert({
+ "build_id": build,
+ "prev_build_id": prev_build
+ })
+ tables.RegressionStatus.insert({"regression_id": regression_id,
+ "name": "awfy",
+ "extra": "Submitted",
+ "stamp":"UNIX_TIMESTAMP()"})
+ else:
+ regression_id = regression[0].id
+
try:
if score.__class__ == tables.Score:
- tables.RegressionScore.insert({"build_id": build, "score_id": score.get("id")})
+ tables.RegressionScore.insert({"regression_id": regression_id,
+ "score_id": score.get("id")})
elif score.__class__ == tables.Breakdown:
- tables.RegressionBreakdown.insert({"build_id": build, "breakdown_id": score.get("id")})
+ tables.RegressionBreakdown.insert({"regression_id": regression_id,
+ "breakdown_id": score.get("id")})
else:
assert False
except:
diff --git a/detector/regression_detector_old.py b/detector/regression_detector_old.py
deleted file mode 100644
index a7683da..0000000
--- a/detector/regression_detector_old.py
+++ /dev/null
@@ -1,111 +0,0 @@
-# vim: set ts=4 sw=4 tw=99 et:
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import awfy
-import sys
-import time
-import tables_old as tables
-
-
-def notProcessedRuns():
- # Only look at reports in the last week
- newer = int(time.time() - 60 * 60 * 24 * 7)
- c = awfy.db.cursor()
- c.execute("SELECT id \
- FROM awfy_run \
- WHERE stamp > "+str(newer)+" AND \
- status = 1 AND \
- detector != 1 AND \
- machine in (28,29)")
- runs = []
- for row in c.fetchall():
- runs.append(tables.Run(row[0]))
- return runs
-
-def regressed(score):
- # Lower than threshold, no regression.
- change = score.change()
- if change is None:
- return None
- if abs(change) <= score.noise():
- return False
-
- #Don't report outliners
- if score.outliner():
- return False
-
- # Don't report if same revision
- if score.prev() is not None:
- if score.get('build').get('cset') == score.prev().get('build').get('cset'):
- return False
-
- # average change over multiple runs.
- change = score.avg_change()
-
- # No change, so wait for more data before reporting.
- if change is None:
- return None
-
- # Next is not available. Wait for that before reporting.
- if not score.next():
- return None
- if score.next().avg_change() is None:
- return None
-
- # Next has a bigger change. Regression is more likely to be that.
- if change >= 0 and score.next().avg_change() > change:
- return False
- if change <= 0 and score.next().avg_change() < change:
- return False
-
- # If there is a prev, test that prev change is smaller
- if score.prev():
- if change >= 0 and score.prev().avg_change() >= change:
- return False
- if change <= 0 and score.prev().avg_change() <= change:
- return False
-
- return True
-
-if __name__ == "__main__":
- import os
- import time
- os.environ['TZ'] = "Europe/Amsterdam"
- time.tzset()
-
- start = time.time()
- for run in notProcessedRuns():
- scores = run.getScoresAndBreakdowns()
- finish = True
- print "run:", run.get("id")
- for score in scores:
- regressed_ = regressed(score)
-
- # Not enough info yet
- if regressed_ is None:
- finish = False
-
- if regressed_ is True:
- score.dump()
- build = score.get("build_id")
- try:
- id_ = tables.Regression.insert({"build_id": build})
- tables.RegressionStatus.insert({"regression_id": id_, "name": "awfy", "extra": "Submitted", "stamp":"UNIX_TIMESTAMP()"})
- except:
- pass
- try:
- if score.__class__ == tables.Score:
- tables.RegressionScore.insert({"build_id": build, "score_id": score.get("id")})
- elif score.__class__ == tables.Breakdown:
- tables.RegressionBreakdown.insert({"build_id": build, "breakdown_id": score.get("id")})
- else:
- assert False
- except:
- pass
- if finish:
- run.update({"detector": "1"})
- tables.DBTable.maybeflush()
-
- awfy.db.commit()
diff --git a/detector/tables.py b/detector/tables.py
index 50efd1a..1767554 100644
--- a/detector/tables.py
+++ b/detector/tables.py
@@ -84,6 +84,11 @@ class DBTable:
SET "+",".join(sets)+" \
WHERE id = %s", (self.id, ))
+ def delete(self):
+ c = awfy.db.cursor()
+ c.execute("DELETE FROM "+self.table()+" \
+ WHERE id = %s", (self.id, ))
+
@staticmethod
def valuefy(value):
if "'" in str(value):
@@ -109,6 +114,14 @@ class DBTable:
for row in c.fetchall():
yield class_(row[0])
+ @classmethod
+ def where(class_, data):
+ where = [name+" = "+DBTable.valuefy(data[name]) for name in data]
+ c = awfy.db.cursor()
+ c.execute("SELECT id FROM "+class_.table()+" WHERE "+" AND ".join(where))
+ for row in c.fetchall():
+ yield class_(row[0])
+
@classmethod
def maybeflush(class_):
#TODO
@@ -211,27 +224,29 @@ class Regression(DBTable):
def __init__(self, id):
DBTable.__init__(self, id)
+ def regressions(self):
+ c = awfy.db.cursor()
+ c.execute("SELECT id FROM awfy_regression_breakdown \
+ WHERE regression_id = %s", (self.id,))
+ for row in c.fetchall():
+ if row[0] == 0:
+ continue
+ yield RegressionBreakdown(row[0])
+ c.execute("SELECT id FROM awfy_regression_score \
+ WHERE regression_id = %s", (self.id,))
+ for row in c.fetchall():
+ if row[0] == 0:
+ continue
+ yield RegressionScore(row[0])
+
@staticmethod
def table():
return "awfy_regression"
class RegressionScore(DBTable):
- def __init__(self, build, score):
- c = awfy.db.cursor()
- c.execute("SELECT id FROM "+self.table()+" \
- WHERE build_id = %s AND \
- score_id = %s", (build.get("id"), score.get("id")))
- row = c.fetchone()
- id = row[0] if row else 0
- DBTable.__init__(self, id)
- def regression(self):
- c = awfy.db.cursor()
- c.execute("SELECT id FROM awfy_regression \
- WHERE build_id = %s", (self.get("build_id"),))
- row = c.fetchone()
- id = row[0] if row else 0
- return Regression(id)
+ def score(self):
+ return self.get("score")
@staticmethod
def table():
@@ -271,22 +286,9 @@ class RegressionScoreNoise(DBTable):
return "awfy_regression_score_noise"
class RegressionBreakdown(DBTable):
- def __init__(self, build, breakdown):
- c = awfy.db.cursor()
- c.execute("SELECT id FROM "+self.table()+" \
- WHERE build_id = %s AND \
- breakdown_id = %s", (build.get("id"), breakdown.get("id")))
- row = c.fetchone()
- id = row[0] if row else 0
- DBTable.__init__(self, id)
- def regression(self):
- c = awfy.db.cursor()
- c.execute("SELECT id FROM awfy_regression \
- WHERE build_id = %s", (self.get("build_id"),))
- row = c.fetchone()
- id = row[0] if row else 0
- return Regression(id)
+ def score(self):
+ return self.get("breakdown")
@staticmethod
def table():
diff --git a/detector/tables_old.py b/detector/tables_old.py
deleted file mode 100644
index 94aa749..0000000
--- a/detector/tables_old.py
+++ /dev/null
@@ -1,759 +0,0 @@
-# vim: set ts=4 sw=4 tw=99 et:
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import awfy
-import types
-
-RUNS_FACTOR = 1
-NOISE_FACTOR = 3
-
-def get_class(field):
- try:
- identifier = globals()[field]
- except AttributeError:
- raise NameError("%s doesn't exist." % field)
- if isinstance(identifier, (types.ClassType, types.TypeType)):
- return identifier
- raise TypeError("%s is not a class." % field)
-
-def camelcase(string):
- """Convert string or unicode from lower-case underscore to camel-case"""
- splitted_string = string.split('_')
- # use string's class to work on the string to keep its type
- class_ = string.__class__
- return class_.join('', map(class_.capitalize, splitted_string))
-
-class DBTable:
- globalcache = {}
-
- def __init__(self, id):
- self.id = int(id)
- self.initialized = False
- self.cached = None
-
- def prefetch(self):
- if self.table() not in self.__class__.globalcache:
- self.__class__.globalcache[self.table()] = {}
-
- c = awfy.db.cursor()
- c.execute("SELECT * \
- FROM "+self.table()+" \
- WHERE id > %s - 100 AND \
- id < %s + 100 \
- ", (self.id, self.id))
- for row in c.fetchall():
- cache = {}
- for i in range(len(row)):
- cache[c.description[i][0]] = row[i]
- self.__class__.globalcache[self.table()][cache["id"]] = cache
-
- def initialize(self):
- if self.initialized:
- return
-
- self.initialized = True
- if self.table() in self.__class__.globalcache:
- if self.id in self.__class__.globalcache[self.table()]:
- self.cached = self.__class__.globalcache[self.table()][self.id]
- return
-
- self.prefetch()
- self.cached = self.__class__.globalcache[self.table()][self.id]
- return
-
- def get(self, field):
- self.initialize()
-
- if field in self.cached:
- return self.cached[field]
-
- if field+"_id" in self.cached:
- id_ = self.cached[field+"_id"]
- class_ = get_class(camelcase(field))
- value = class_(id_)
- self.cached[field] = value
- return self.cached[field]
- assert False
-
- def update(self, data):
- sets = [key + " = " + DBTable.valuefy(data[key]) for key in data]
- c = awfy.db.cursor()
- c.execute("UPDATE "+self.table()+" \
- SET "+",".join(sets)+" \
- WHERE id = %s", (self.id, ))
-
- @staticmethod
- def valuefy(value):
- if "'" in str(value):
- raise TypeError("' is not allowed as value.")
- if value == "UNIX_TIMESTAMP()":
- return value
- else:
- return "'"+str(value)+"'"
-
- @classmethod
- def insert(class_, data):
- values = [DBTable.valuefy(value) for value in data.values()]
- c = awfy.db.cursor()
- c.execute("INSERT INTO "+class_.table()+" \
- ("+",".join(data.keys())+") \
- VALUES ("+",".join(values)+")")
- return c.lastrowid
-
- @classmethod
- def all(class_):
- c = awfy.db.cursor()
- c.execute("SELECT id FROM "+class_.table())
- for row in c.fetchall():
- yield class_(row[0])
-
- @classmethod
- def maybeflush(class_):
- #TODO
- records = 0
- for i in class_.globalcache:
- records += len(class_.globalcache[i].keys())
-
-class Run(DBTable):
- def __init__(self, id):
- DBTable.__init__(self, id)
-
- @staticmethod
- def table():
- return "awfy_run"
-
- def initialize(self):
- if self.initialized:
- return
- DBTable.initialize(self)
- if "machine_id" not in self.cached:
- self.cached["machine_id"] = self.cached["machine"]
- del self.cached["machine"]
-
- def getScoresAndBreakdowns(self):
- c = awfy.db.cursor()
- c.execute("SELECT id \
- FROM awfy_build \
- WHERE run_id = %s", (self.id,))
- scores = []
- for row in c.fetchall():
- scores += Build(row[0]).getScoresAndBreakdowns()
- return scores
-
- def getScores(self):
- c = awfy.db.cursor()
- c.execute("SELECT id \
- FROM awfy_build \
- WHERE run_id = %s", (self.id,))
- scores = []
- for row in c.fetchall():
- scores += Build(row[0]).getScores()
- return scores
-
- def finishStamp(self):
- pass
-
-class SuiteTest(DBTable):
- def __init__(self, id):
- DBTable.__init__(self, id)
-
- @staticmethod
- def table():
- return "awfy_suite_test"
-
-class SuiteVersion(DBTable):
- def __init__(self, id):
- DBTable.__init__(self, id)
-
- @staticmethod
- def table():
- return "awfy_suite_version"
-
-class Suite(DBTable):
- def __init__(self, id):
- DBTable.__init__(self, id)
-
- @staticmethod
- def table():
- return "awfy_suite"
-
-class Machine(DBTable):
- def __init__(self, id):
- DBTable.__init__(self, id)
-
- @staticmethod
- def table():
- return "awfy_machine"
-
-class Mode(DBTable):
- def __init__(self, id):
- DBTable.__init__(self, id)
-
- @classmethod
- def allWith(class_, machine):
- c = awfy.db.cursor()
- c.execute("SELECT distinct(awfy_build.mode_id) \
- FROM awfy_build \
- LEFT JOIN awfy_run ON awfy_build.run_id = awfy_run.id \
- WHERE machine = %s", (machine.get("id"),))
- for row in c.fetchall():
- yield Mode(row[0])
-
- @staticmethod
- def table():
- return "awfy_mode"
-
-class Regression(DBTable):
- def __init__(self, id):
- DBTable.__init__(self, id)
-
- @staticmethod
- def table():
- return "awfy_regression"
-
-class RegressionScore(DBTable):
- def __init__(self, build, score):
- c = awfy.db.cursor()
- c.execute("SELECT id FROM "+self.table()+" \
- WHERE build_id = %s AND \
- score_id = %s", (build.get("id"), score.get("id")))
- row = c.fetchone()
- id = row[0] if row else 0
- DBTable.__init__(self, id)
-
- def regression(self):
- c = awfy.db.cursor()
- c.execute("SELECT id FROM awfy_regression \
- WHERE build_id = %s", (self.get("build_id"),))
- row = c.fetchone()
- id = row[0] if row else 0
- return Regression(id)
-
- @staticmethod
- def table():
- return "awfy_regression_score"
-
-class RegressionScoreNoise(DBTable):
- def __init__(self, machine, suite, mode):
- c = awfy.db.cursor()
- c.execute("SELECT id FROM "+self.table()+" \
- WHERE machine_id = %s AND \
- mode_id = %s AND \
- suite_version_id = %s", (machine.get("id"), mode.get("id"), suite.get("id")))
- row = c.fetchone()
- id = row[0] if row else 0
- DBTable.__init__(self, id)
-
- @classmethod
- def insertOrUpdate(class_, machine, suite, mode, noise):
- try:
- RegressionScoreNoise.insert({
- "machine_id": machine.get("id"),
- "suite_version_id": suite.get("id"),
- "mode_id": mode.get("id"),
- "noise": noise
- })
- except:
- c = awfy.db.cursor()
- c.execute("UPDATE "+class_.table()+" \
- SET noise = %s \
- WHERE machine_id = %s AND \
- mode_id = %s AND \
- suite_version_id = %s", (noise, machine.get("id"), mode.get("id"),
- suite.get("id")))
-
- @staticmethod
- def table():
- return "awfy_regression_score_noise"
-
-class RegressionBreakdown(DBTable):
- def __init__(self, build, breakdown):
- c = awfy.db.cursor()
- c.execute("SELECT id FROM "+self.table()+" \
- WHERE build_id = %s AND \
- breakdown_id = %s", (build.get("id"), breakdown.get("id")))
- row = c.fetchone()
- id = row[0] if row else 0
- DBTable.__init__(self, id)
-
- def regression(self):
- c = awfy.db.cursor()
- c.execute("SELECT id FROM awfy_regression \
- WHERE build_id = %s", (self.get("build_id"),))
- row = c.fetchone()
- id = row[0] if row else 0
- return Regression(id)
-
- @staticmethod
- def table():
- return "awfy_regression_breakdown"
-
-class RegressionBreakdownNoise(DBTable):
- def __init__(self, machine, suite, mode):
- c = awfy.db.cursor()
- c.execute("SELECT id FROM "+self.table()+" \
- WHERE machine_id = %s AND \
- mode_id = %s AND \
- suite_test_id = %s", (machine.get("id"), mode.get("id"), suite.get("id")))
- row = c.fetchone()
- id = row[0] if row else 0
- DBTable.__init__(self, id)
-
- @classmethod
- def insertOrUpdate(class_, machine, suite, mode, noise):
- try:
- RegressionBreakdownNoise.insert({
- "machine_id": machine.get("id"),
- "suite_test_id": suite.get("id"),
- "mode_id": mode.get("id"),
- "noise": noise
- })
- except:
- c = awfy.db.cursor()
- c.execute("UPDATE "+class_.table()+" \
- SET noise = %s \
- WHERE machine_id = %s AND \
- mode_id = %s AND \
- suite_test_id = %s", (noise, machine.get("id"), mode.get("id"),
- suite.get("id")))
-
- @staticmethod
- def table():
- return "awfy_regression_breakdown_noise"
-
-class RegressionStatus(DBTable):
- def __init__(self, id):
- DBTable.__init__(self, id)
-
- @staticmethod
- def table():
- return "awfy_regression_status"
-
-class Build(DBTable):
- def __init__(self, id):
- DBTable.__init__(self, id)
-
- @staticmethod
- def table():
- return "awfy_build"
-
- def getScores(self):
- scores = []
- c = awfy.db.cursor()
- c.execute("SELECT id \
- FROM awfy_score \
- WHERE build_id = %s", (self.id,))
- for row in c.fetchall():
- scores.append(Score(row[0]))
- return scores
-
- def getScoresAndBreakdowns(self):
- scores = self.getScores()
- c = awfy.db.cursor()
- c.execute("SELECT id \
- FROM awfy_breakdown \
- WHERE build_id = %s", (self.id,))
- for row in c.fetchall():
- scores.append(Breakdown(row[0]))
- return scores
-
-class RegressionTools(DBTable):
- def __init__(self, id):
- DBTable.__init__(self, id)
-
- def outliner(self):
- if self.next() is None:
- return False
- prevs, _ = self.avg_prevs_nexts()
- _, nexts = self.next().avg_prevs_nexts()
- if prevs is None or nexts is None:
- return False
- if abs(prevs-nexts) <= self.noise():
- if (abs(self.get('score') - self.prev().get('score')) > self.noise() and
- abs(self.get('score') - self.next().get('score')) > self.noise()):
- return True
- return False
-
- def next(self):
- self.initialize()
- if "next" not in self.cached:
- self.cached["next"] = self.compute_next()
- return self.cached["next"]
-
- def compute_next(self):
- nexts = self.prefetch_next(10)
-
- prev = self
- prev.cached["next"] = None
- for score in nexts:
- prev.initialize()
- prev.cached["next"] = score
- score.initialize()
- score.cached["prev"] = prev
- prev = score
-
- return self.cached["next"]
-
- def prev(self):
- self.initialize()
- if "prev" not in self.cached:
- self.cached["prev"] = self.compute_prev()
- if self.cached["prev"]:
- self.cached["prev"].initialize()
- self.cached["prev"].cached["next"] = self
- else:
- pass
- return self.cached["prev"]
-
- def compute_prev(self):
- prevs = self.prefetch_prev(10)
-
- next_ = self
- next_.cached["prev"] = None
- for score in prevs:
- next_.initialize()
- next_.cached["prev"] = score
- score.initialize()
- score.cached["next"] = next_
- next_ = score
-
- return self.cached["prev"]
-
- def prevs(self, amount):
- prevs = []
- point = self
- while len(prevs) < amount:
- point = point.prev()
- if not point:
- break
- prevs.append(point)
- return prevs
-
- def nexts(self, amount):
- nexts = []
- point = self
- while len(nexts) < amount:
- point = point.next()
- if not point:
- break
- nexts.append(point)
- return nexts
-
- def prevs_no_outliners(self, amount):
- # note this removes outliners
- prevs = []
- point = self
- while len(prevs) < amount:
- point = point.prev()
- if not point:
- break
- if point.outliner():
- continue
- prevs.append(point)
- return prevs
-
- def nexts_no_outliners(self, amount):
- # note this removes outliners
- nexts = []
- point = self
- while len(nexts) < amount:
- point = point.next()
- if not point:
- break
- if point.outliner():
- continue
- nexts.append(point)
- return nexts
-
- def avg_prevs_no_outliners(self):
- avg_prevs, _ = self.avg_prevs_nexts_no_outliners()
- return avg_prevs
-
- def avg_nexts_no_outliners(self):
- _, avg_nexts = self.avg_prevs_nexts_no_outliners()
- return avg_nexts
-
- def avg_prevs_nexts_no_outliners(self):
- self.initialize()
- if "avg_prevs_no_outliners" not in self.cached:
- self.cached["avg_prevs_no_outliners"], self.cached["avg_nexts_no_outliners"] = self.compute_avg_prevs_nexts_no_outliners()
- return self.cached["avg_prevs_no_outliners"], self.cached["avg_nexts_no_outliners"]
-
- def avg_prevs_nexts(self):
- self.initialize()
- if "avg_prevs" not in self.cached:
- self.cached["avg_prevs"], self.cached["avg_nexts"] = self.compute_avg_prevs_nexts()
- return self.cached["avg_prevs"], self.cached["avg_nexts"]
-
- def compute_avg_prevs_nexts_no_outliners(self):
- "Compute the change in runs before and after the current run"
- # How many runs do we need to test?
- runs = self.runs()
-
- # Get scores before and after this run.
- prevs = [i.get('score') for i in self.prevs_no_outliners(runs)]
- nexts = [self.get('score')] + [i.get('score') for i in self.nexts_no_outliners(runs - 1)]
-
- p_weight = [len(prevs)-i for i in range(len(prevs))]
- n_weight = [len(nexts)-i for i in range(len(nexts))]
- prevs = [prevs[i]*p_weight[i] for i in range(len(prevs))]
- nexts = [nexts[i]*n_weight[i] for i in range(len(nexts))]
-
- # Not enough data to compute change.
- if len(nexts) != runs:
- return None, None
-
- avg_prevs = sum(prevs)
- avg_nexts = sum(nexts)
-
- # Handle edge cases.
- if avg_prevs != 0:
- avg_prevs /= sum(p_weight)
- if avg_nexts != 0:
- avg_nexts /= sum(n_weight)
-
- return avg_prevs, avg_nexts
-
- def compute_avg_prevs_nexts(self):
- "Compute the change in runs before and after the current run"
- # How many runs do we need to test?
- runs = self.runs()
-
- # Get scores before and after this run.
- prevs = [i.get('score') for i in self.prevs(runs)]
- nexts = [self.get('score')] + [i.get('score') for i in self.nexts(runs - 1)]
-
- p_weight = [len(prevs)-i for i in range(len(prevs))]
- n_weight = [len(nexts)-i for i in range(len(nexts))]
- prevs = [prevs[i]*p_weight[i] for i in range(len(prevs))]
- nexts = [nexts[i]*n_weight[i] for i in range(len(nexts))]
-
- # Not enough data to compute change.
- if len(nexts) != runs:
- return None, None
-
- avg_prevs = sum(prevs)
- avg_nexts = sum(nexts)
-
- # Handle edge cases.
- if avg_prevs != 0:
- avg_prevs /= sum(p_weight)
- if avg_nexts != 0:
- avg_nexts /= sum(n_weight)
-
- return avg_prevs, avg_nexts
-
- def change(self):
- prevs, nexts = self.avg_prevs_nexts_no_outliners()
- if not prevs or not nexts:
- return None
-
- return abs(prevs - nexts)
-
- def avg_change(self):
- prevs, nexts = self.avg_prevs_nexts_no_outliners()
- if not prevs or not nexts:
- return None
-
- if prevs == 0:
- return float("inf")
-
- change = (prevs - nexts) / (prevs)
- return change
-
-
-class Score(RegressionTools):
- def __init__(self, id):
- RegressionTools.__init__(self, id)
-
- @staticmethod
- def table():
- return "awfy_score"
-
- def prefetch_next(self, limit = 1):
- stamp = self.get("build").get("run").get("stamp")
- machine = self.get("build").get("run").get("machine_id")
- mode = self.get("build").get("mode_id")
- suite = self.get("suite_version_id")
-
- c = awfy.db.cursor()
- c.execute("SELECT awfy_score.id \
- FROM awfy_score \
- INNER JOIN awfy_build ON awfy_build.id = awfy_score.build_id \
- INNER JOIN awfy_run ON awfy_run.id = awfy_build.run_id \
- WHERE stamp > %s AND \
- machine = %s AND \
- mode_id = %s AND \
- suite_version_id = %s AND \
- status = 1 \
- ORDER BY stamp ASC \
- LIMIT "+str(limit), (stamp, machine, mode, suite))
- rows = c.fetchall()
- return [Score(row[0]) for row in rows]
-
- def prefetch_prev(self, limit = 1):
- stamp = self.get("build").get("run").get("stamp")
- machine = self.get("build").get("run").get("machine_id")
- mode = self.get("build").get("mode_id")
- suite = self.get("suite_version_id")
-
- c = awfy.db.cursor()
- c.execute("SELECT awfy_score.id \
- FROM awfy_score \
- INNER JOIN awfy_build ON awfy_build.id = awfy_score.build_id \
- INNER JOIN awfy_run ON awfy_run.id = awfy_build.run_id \
- WHERE stamp < %s AND \
- machine = %s AND \
- mode_id = %s AND \
- suite_version_id = %s AND \
- status = 1 \
- ORDER BY stamp DESC \
- LIMIT 1", (stamp, machine, mode, suite))
- rows = c.fetchall()
- return [Score(row[0]) for row in rows]
-
- def runs(self):
- runs = max(1, self.get('build').get('run').get('machine').get("confidence_runs"))
- runs *= self.get('suite_version').get('suite').get("confidence_factor")
- runs *= RUNS_FACTOR
- runs = int(round(runs))
- return runs
-
- def noise(self):
- noise = RegressionScoreNoise(self.get('build').get('run').get('machine'),
- self.get('suite_version'),
- self.get('build').get('mode')).get('noise')
- return NOISE_FACTOR*noise
-
- @classmethod
- def first(class_, machine, suite, mode):
- assert machine.__class__ == Machine
- assert suite.__class__ == SuiteVersion
- assert mode.__class__ == Mode
-
- c = awfy.db.cursor()
- c.execute("SELECT awfy_score.id \
- FROM awfy_score \
- INNER JOIN awfy_build ON awfy_build.id = awfy_score.build_id \
- INNER JOIN awfy_run ON awfy_run.id = awfy_build.run_id \
- WHERE machine = %s AND \
- mode_id = %s AND \
- suite_version_id = %s AND \
- status = 1 \
- ORDER BY stamp ASC \
- LIMIT 1", (machine.get("id"), mode.get("id"), suite.get("id")))
- row = c.fetchone()
- if row:
- return Score(row[0])
- return None
-
- def dump(self):
- if self.get("build").get("mode").get("name") != "Ion":
- return
- import datetime
- print datetime.datetime.fromtimestamp(
- int(self.get("build").get("run").get("stamp"))
- ).strftime('%Y-%m-%d %H:%M:%S'),
- print "", self.get("build").get("run").get("machine").get("description"),
- print "", self.get("build").get("mode").get("name"),
- print "", self.get("suite_version").get("name")+":", self.avg_change(),
- print "", self.prev().get("score") if self.prev() else "", self.get("score"),
- print " ("+str(self.runs())+" runs, "+str(self.noise())+")"
-
-class Breakdown(RegressionTools):
- def __init__(self, id):
- RegressionTools.__init__(self, id)
-
- @staticmethod
- def table():
- return "awfy_breakdown"
-
- def prefetch_next(self, limit = 1):
- stamp = self.get("build").get("run").get("stamp")
- machine = self.get("build").get("run").get("machine_id")
- mode = self.get("build").get("mode_id")
- suite = self.get("suite_test_id")
-
- c = awfy.db.cursor()
- c.execute("SELECT awfy_breakdown.id \
- FROM awfy_breakdown \
- INNER JOIN awfy_build ON awfy_build.id = awfy_breakdown.build_id \
- INNER JOIN awfy_run ON awfy_run.id = awfy_build.run_id \
- WHERE stamp > %s AND \
- machine = %s AND \
- mode_id = %s AND \
- suite_test_id = %s AND \
- status = 1 \
- ORDER BY stamp ASC \
- LIMIT "+str(limit), (stamp, machine, mode, suite))
- rows = c.fetchall()
- return [Breakdown(row[0]) for row in rows]
-
- def prefetch_prev(self, limit = 1):
- stamp = self.get("build").get("run").get("stamp")
- machine = self.get("build").get("run").get("machine_id")
- mode = self.get("build").get("mode_id")
- suite = self.get("suite_test_id")
-
- c = awfy.db.cursor()
- c.execute("SELECT awfy_breakdown.id \
- FROM awfy_breakdown \
- INNER JOIN awfy_build ON awfy_build.id = awfy_breakdown.build_id \
- INNER JOIN awfy_run ON awfy_run.id = awfy_build.run_id \
- WHERE stamp < %s AND \
- machine = %s AND \
- mode_id = %s AND \
- suite_test_id = %s AND \
- status = 1 \
- ORDER BY stamp DESC \
- LIMIT "+str(limit), (stamp, machine, mode, suite))
- rows = c.fetchall()
- return [Breakdown(row[0]) for row in rows]
-
- @classmethod
- def first(class_, machine, suite, mode):
- assert machine.__class__ == Machine
- assert suite.__class__ == SuiteTest
- assert mode.__class__ == Mode
-
- c = awfy.db.cursor()
- c.execute("SELECT awfy_breakdown.id \
- FROM awfy_breakdown \
- INNER JOIN awfy_build ON awfy_build.id = awfy_breakdown.build_id \
- INNER JOIN awfy_run ON awfy_run.id = awfy_build.run_id \
- WHERE machine = %s AND \
- mode_id = %s AND \
- suite_test_id = %s AND \
- status = 1 \
- ORDER BY stamp ASC \
- LIMIT 1", (machine.get("id"), mode.get("id"), suite.get("id")))
- row = c.fetchone()
- if row:
- return Breakdown(row[0])
- return None
-
- def runs(self):
- runs = max(1, self.get('build').get('run').get('machine').get("confidence_runs"))
- runs *= self.get('suite_test').get("confidence_factor")
- runs *= RUNS_FACTOR
- runs = int(round(runs))
- return runs
-
- def noise(self):
- noise = RegressionBreakdownNoise(self.get('build').get('run').get('machine'),
- self.get('suite_test'),
- self.get('build').get('mode')).get('noise')
- return NOISE_FACTOR*noise
-
- def dump(self):
- import datetime
- print datetime.datetime.fromtimestamp(
- int(self.get("build").get("run").get("stamp"))
- ).strftime('%Y-%m-%d %H:%M:%S'),
- #print "", self.get("build").get("run").get("machine").get("description"),
- print "", self.get("build").get("mode").get("name"),
- print "", self.get("suite_test").get("suite_version").get("name")+":", self.get("suite_test").get("name")+":", self.avg_change(),
- print "", self.prev().get("score") if self.prev() else "", self.get("score"),
- print " ("+str(self.runs())+" runs, "+str(self.noise())+", "+str(self.change())+")"
diff --git a/detector/tests.py b/detector/tests.py
deleted file mode 100644
index 9d3a097..0000000
--- a/detector/tests.py
+++ /dev/null
@@ -1,108 +0,0 @@
-# vim: set ts=4 sw=4 tw=99 et:
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import awfy
-import sys
-import time
-import tables
-import tables_old
-import regression_detector
-import regression_detector_old
-from collections import defaultdict
-
-import os
-import time
-import datetime
-os.environ['TZ'] = "Europe/Amsterdam"
-time.tzset()
-
-def testRuns():
- c = awfy.db.cursor()
- newer = int(time.time() - 60 * 60 * 24 * 30)
- older = int(time.time() - 60 * 60 * 24 * 1)
- c.execute("SELECT id \
- FROM awfy_run \
- WHERE stamp > "+str(newer)+" AND \
- stamp < "+str(older)+" AND \
- status = 1 AND \
- detector = 1 AND \
- machine in (28, 29)")
- runs = []
- for row in c.fetchall():
- runs.append(tables.Run(row[0]))
- return runs
-
-
-changes = defaultdict(int)
-for run in testRuns():
- scores = run.getScoresAndBreakdowns()
- for score in scores:
- if score.get("build").get("mode").get("name") != "Ion":
- continue
-
- if score.__class__ == tables.Score:
- regression = tables.RegressionScore(score.get("build"), score)
- score_old = tables_old.Score(score.id)
- elif score.__class__ == tables.Breakdown:
- regression = tables.RegressionBreakdown(score.get("build"), score)
- score_old = tables_old.Breakdown(score.id)
-
- status_db = "noregression"
- if regression.id != 0:
- if regression.get("noise") == 1:
- status_db = "marked noregression"
- elif regression.regression().get('status') == "noise":
- status_db = "marked noregression"
- elif regression.regression().get('status') == "unconfirmed":
- status_db = "unconfirmed"
- else:
- status_db = "regressed"
-
- status_old = "noregression"
- regressed_ = regression_detector_old.regressed(score_old)
- if regressed_ is None:
- status_old = "nodata"
- elif regressed_:
- status_old = "regressed"
-
- status_now = "noregression"
- regressed_ = regression_detector.regressed(score)
- if regressed_ is None:
- status_now = "nodata"
- elif regressed_:
- status_now = "regressed"
-
- key = status_db+"-"+status_now
- changes["db_"+status_db] += 1
- changes["now_"+status_now] += 1
- changes["old_"+status_old] += 1
- changes[key] += 1
-
- if key == "regressed-noregression" or key == "noregression-regressed" or status_now == "regressed":
- print datetime.datetime.fromtimestamp(
- int(score.get("build").get("run").get("stamp"))
- ).strftime('%Y-%m-%d %H:%M:%S'),
- if regression.id == 0:
- print 0,
- else:
- print regression.regression().id,
- print score.get("build").get("run").get("machine_id"),
- print score.get("build").get("mode").get("name"),
- print score.change(),
- print score.noise(),
- print score.avg_change(),
- if score.__class__ == tables.Score:
- print score.get("suite_version").get("name"),
- else:
- print score.get("suite_test").get("name"),
- print key
-
-print "Lost detections!:", changes["regressed-noregression"]
-print "Over active detection: ", changes["noregression-regressed"]
-print "% less detections: ", 1.0*int(changes["marked noregression-noregression"])/(int(changes["marked noregression-regressed"])+int(changes["marked noregression-noregression"]))
-
-print "Db regressions: ", int(changes["db_marked noregression"]) + int(changes["db_regressed"]) + int(changes["db_unconfirmed"])
-print "Old regressions: ", int(changes["old_regressed"])
-print "Now regressions: ", int(changes["now_regressed"])
diff --git a/website/awfy.js b/website/awfy.js
index 8050bdc..2589312 100755
--- a/website/awfy.js
+++ b/website/awfy.js
@@ -20,7 +20,6 @@ AWFY.lastHash = null;
AWFY.lastRefresh = 0;
// Hide a view modes by default. Since they aren't active anymore
-//AWFYMaster.modes["30"].hidden = true
AWFYMaster.modes["35"].hidden = true
AWFYMaster.modes["27"].hidden = true
AWFYMaster.modes["29"].hidden = true
diff --git a/website/data-regression.php b/website/data-regression.php
index de391e3..081c352 100644
--- a/website/data-regression.php
+++ b/website/data-regression.php
@@ -13,13 +13,13 @@ if ($subtest) {
$query = mysql_query("SELECT awfy_regression.id, noise, status
FROM `awfy_regression_breakdown`
LEFT JOIN awfy_regression
- ON awfy_regression.build_id = awfy_regression_breakdown.build_id
+ ON awfy_regression.id = awfy_regression_breakdown.regression_id
WHERE breakdown_id = ".$id);
} else {
$query = mysql_query("SELECT awfy_regression.id, noise, status
FROM `awfy_regression_score`
LEFT JOIN awfy_regression
- ON awfy_regression.build_id = awfy_regression_score.build_id
+ ON awfy_regression.id = awfy_regression_score.regression_id
WHERE breakdown_id = ".$id);
}
diff --git a/website/index.html b/website/index.html
index 2c6a9b2..70917ab 100755
--- a/website/index.html
+++ b/website/index.html
@@ -143,6 +143,21 @@
ga('send', 'pageview');
+
+
+
+