Get to submit to treeherder

database/migration-9.php

@@ -0,0 +1,13 @@
+<?php
+
+// Add field 'treeherder' to the awfy_run database.
+// And mark all previous runs supposedly submitted to treeherder
+
+$migrate = function() {
+    mysql_query("ALTER TABLE `awfy_run` ADD `treeherder` BOOLEAN NOT NULL;") or die(mysql_error());
+    mysql_query("UPDATE awfy_run SET `treeherder` = 1 WHERE status != 0;") or die(mysql_error());
+};
+
+$rollback = function() {
+    mysql_query("ALTER TABLE `awfy_run` DROP `treeherder`;") or die(mysql_error());
+};

detector/awfy.py

@@ -1,73 +0,0 @@
-# vim: set ts=4 sw=4 tw=99 et:
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-try:
-  import MySQLdb as mdb
-except:
-  import mysqldb as mdb
-try:
-  import ConfigParser
-except:
-  import configparser as ConfigParser
-
-db = None
-version = None
-path = None
-
-
-queries = 0
-class DB:
-  def __init__(self, host, user, pw, name):
-    self.host = host
-    self.user = user
-    self.pw = pw
-    self.name = name
-    self.connect()
-  def connect(self):
-    if self.host[0] == '/':
-      self.db = mdb.connect(unix_socket=self.host, user=self.user, passwd=self.pw,
-                            db=self.name, use_unicode=True)
-    else:
-      self.db = mdb.connect(self.host, self.user, self.pw, self.name, use_unicode=True)
-  def cursor(self):
-    return DBCursor(self.db.cursor())
-  def commit(self):
-    return self.db.commit()
-class DBCursor:
-  def __init__(self, cursor):
-    self.cursor = cursor
-  def execute(self, sql, data=None):
-    global queries
-    queries+=1
-    exe = self.cursor.execute(sql, data);
-    self.description = self.cursor.description
-    self.lastrowid = self.cursor.lastrowid
-    self.rowcount = self.cursor.rowcount
-    return exe
-  def fetchone(self):
-    return self.cursor.fetchone();
-  def fetchall(self):
-    return self.cursor.fetchall();
-
-def Startup():
-    global db, version, path
-    config = ConfigParser.RawConfigParser()
-    config.read("/etc/awfy-server.config")
-
-    host = config.get('mysql', 'host')
-    user = config.get('mysql', 'user')
-    pw = config.get('mysql', 'pass')
-    name = config.get('mysql', 'db_name')
-
-    db = DB(host, user, pw, name)
-    c = db.cursor()
-    c.execute("SELECT `value` FROM awfy_config WHERE `key` = 'version'")
-    row = c.fetchone()
-    version = int(row[0])
-
-    path = config.get('general', 'data_folder')
-
-Startup()
-

detector/compute_noise.py

@@ -3,11 +3,14 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-import awfy
+from optparse import OptionParser
 import sys
 import time
+
+sys.path.append("../server") 
+import awfy
 import tables
-from optparse import OptionParser
+
 
 parser = OptionParser(usage="usage: %prog [options]")
 parser.add_option("-n", "--non-existing", dest="nonexistonly", action="store_true", default=False,

detector/improve_detection.py

@@ -3,6 +3,9 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import sys
+
+sys.path.append("../server") 
 import awfy
 import tables
 

detector/regression_detector.py

@@ -3,12 +3,13 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-import awfy
+from optparse import OptionParser
 import sys
 import time
-import tables
 
-from optparse import OptionParser
+sys.path.append("../server") 
+import awfy
+import tables
 
 parser = OptionParser(usage="usage: %prog [options]")
 parser.add_option( "--dry-run", dest="dryrun", action="store_true", default=False,

detector/remove_unconfirmed.py

@@ -3,6 +3,9 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import sys
+
+sys.path.append("../server") 
 import awfy
 
 c = awfy.db.cursor()

server/awfy-server.config

@@ -7,4 +7,9 @@ db_name = ???
 [general]
 data_folder = /home/awfy
 machine_timeout = 480 ; 8 hours (480 minutes)
+slack_webhook = ??? 
 
+[treeherder]
+host = ???
+user = ???
+secret = ???

server/awfy.py

@@ -15,6 +15,9 @@ except:
 db = None
 version = None
 path = None
+th_host = None
+th_user = None
+th_secret = None
 
 
 queries = 0
@@ -52,7 +55,7 @@ class DBCursor:
     return self.cursor.fetchall();
 
 def Startup():
-    global db, version, path
+    global db, version, path, th_host, th_user, th_secret
     config = ConfigParser.RawConfigParser()
     config.read("/etc/awfy-server.config")
 
@@ -69,5 +72,9 @@ def Startup():
 
     path = config.get('general', 'data_folder')
 
-Startup()
+    if config.has_section('treeherder'):
+		th_host = config.get('treeherder', 'host')
+		th_user = config.get('treeherder', 'user')
+		th_secret = config.get('treeherder', 'secret')
 
+Startup()

server/tables.py

@@ -28,6 +28,13 @@ def camelcase(string):
 class DBTable(object):
   globalcache = {}
 
+  @classmethod
+  def FromId(class_, id):
+    obj = class_(row[0])
+    if obj.exists():
+      return obj
+    return None
+    
   def __init__(self, id):
     self.id = int(id)
     self.initialized = False
@@ -142,6 +149,19 @@ class Run(DBTable):
   def table():
     return "awfy_run"
 
+  @staticmethod
+  def fromSortOrder(machine_id, sort_order):
+    c = awfy.db.cursor()
+    c.execute("SELECT id                                                              \
+               FROM awfy_run                                                          \
+               WHERE machine = %s AND                                                 \
+                     sort_order = %s", (machine_id, sort_order))
+    rows = c.fetchall()
+    if len(rows) == 0:
+        return None
+    assert len(rows) == 1
+    return Run(rows[0][0])
+
   def initialize(self):
     if self.initialized:
       return
@@ -348,6 +368,19 @@ class Build(DBTable):
   def table():
     return "awfy_build"
 
+  @staticmethod
+  def fromRunAndMode(run_id, mode_id):
+    c = awfy.db.cursor()
+    c.execute("SELECT id                                                             \
+               FROM awfy_build                                                       \
+               WHERE run_id = %s AND                                                 \
+                     mode_id = %s", (run_id, mode_id))
+    rows = c.fetchall()
+    if len(rows) == 0:
+        return None
+    assert len(rows) == 1
+    return Build(rows[0][0])
+
   def getScores(self):
     scores = []
     c = awfy.db.cursor()
@@ -591,6 +624,29 @@ class Score(RegressionTools):
   def table():
     return "awfy_score"
 
+  @staticmethod
+  def fromBuildAndSuite(build_id, suite_version_id):
+    c = awfy.db.cursor()
+    c.execute("SELECT id                                                              \
+               FROM awfy_score                                                        \
+               WHERE build_id = %s AND                                                \
+                     suite_version_id = %s", (build_id, suite_version_id))
+    rows = c.fetchall()
+    if len(rows) == 0:
+        return None
+    assert len(rows) == 1
+    return Score(rows[0][0])
+
+  def getBreakdowns(self):
+    c = awfy.db.cursor()
+    c.execute("SELECT awfy_breakdown.id                                               \
+               FROM awfy_breakdown                                                    \
+               WHERE score_id = %s", (self.id,))
+    breakdowns = []
+    for row in c.fetchall():
+      breakdowns.append(Breakdown(row[0]))
+    return breakdowns
+
   def sane(self):
     if self.get("suite_version_id") == -1:
         return False

treeherder/config.json

@@ -0,0 +1,12 @@
+[{
+    "machine": 28,
+    "modes": [{
+        "mode": "jmim",
+        "repo": "mozilla-inbound",
+        "platform": ["linux", "linux32", "x86"],
+        "tier": 2,
+        "job_symbol": "shell",
+        "job_name": "shell",
+		"enabled": true
+    }]
+}]

treeherder/config.schema

@@ -0,0 +1,55 @@
+{
+	"type": "array",
+	"items": {
+		"type": "object",
+		"properties": {
+            "machine": {
+				"description": "The id of the machine in the db",
+				"type": "integer"
+			},
+			"modes": {
+				"type": "array",
+				"items": {
+					"type": "object",
+					"required": ["mode","repo","tier","job_symbol","job_name","platform","enabled"],
+					"properties": {
+						"enabled": {
+							"type": "boolean"
+						},
+						"mode": {
+							"description": "the mode name in the db",
+							"type": "string"
+						},
+						"repo": {
+							"description": "The repo this dataset is testing. E.g. mozilla-inbound",
+							"type": "string"
+						},
+						"tier": {
+							"description": "On a scale of 1-3 how important the data is",
+							"type": "integer",
+							"minimum": 1,
+							"maximum": 3
+						},
+						"job_symbol": {
+							"description": "The symbol used to report on treeherder",
+							"type": "string"
+						},
+						"job_name": {
+							"description": "The name used to report on treeherder",
+							"type": "string"
+						},
+						"platform": {
+							"description": "An array explaining the platform. [OS, OS+bits, CPU]",
+							"type": "array",
+							"items": {
+								"type": "string"
+							},
+							"minItems": 3,
+							"maxItems": 3
+						}
+					}
+				}
+			}
+		}
+	}
+}

treeherder/submission.py

@@ -0,0 +1,226 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import logging
+import os
+import requests
+import socket
+import time
+from urlparse import urljoin, urlparse
+import uuid
+
+try:
+	from thclient import TreeherderClient, TreeherderJob, TreeherderJobCollection
+except:
+	print "run 'sudo pip install treeherder-client' to install the needed libraries"
+	exit()
+
+RESULTSET_FRAGMENT = 'api/project/{repository}/resultset/?revision={revision}'
+JOB_FRAGMENT = '/#/jobs?repo={repository}&revision={revision}'
+BUILD_STATES = ['running', 'completed']
+
+logging.basicConfig(format='%(asctime)s %(levelname)s | %(message)s', datefmt='%H:%M:%S')
+logger = logging.getLogger('mozmill-ci')
+logger.setLevel(logging.INFO)
+
+class Submission(object):
+    """Class for submitting reports to Treeherder."""
+
+    def __init__(self, repository, revision, settings,
+                 treeherder_url=None, treeherder_client_id=None, treeherder_secret=None):
+        """Creates new instance of the submission class.
+
+        :param repository: Name of the repository the build has been built from.
+        :param revision: Changeset of the repository the build has been built from.
+        :param settings: Settings for the Treeherder job as retrieved from the config file.
+        :param treeherder_url: URL of the Treeherder instance.
+        :param treeherder_client_id: The client ID necessary for the Hawk authentication.
+        :param treeherder_secret: The secret key necessary for the Hawk authentication.
+
+        """
+        self.repository = repository
+        self.revision = revision
+        self.settings = settings
+
+        self._job_details = []
+
+        self.url = treeherder_url
+        self.client_id = treeherder_client_id
+        self.secret = treeherder_secret
+
+        if not self.client_id or not self.secret and self.url != "mock":
+            raise ValueError('The client_id and secret for Treeherder must be set.')
+
+    def _get_treeherder_platform(self):
+        """Returns the Treeherder equivalent platform identifier of the current platform."""
+
+        #Todo
+        return ('linux', 'linux64', 'x86_64')
+
+    def create_job(self, data=None, **kwargs):
+        """Creates a new instance of a Treeherder job for submission.
+
+        :param data: Job data to use for initilization, e.g. from a previous submission, optional
+        :param kwargs: Dictionary of necessary values to build the job details. The
+            properties correlate to the placeholders in config.py.
+
+        """
+        data = data or {}
+
+        job = TreeherderJob(data=data)
+
+        # If no data is available we have to set all properties
+        if not data:
+            job.add_job_guid(str(uuid.uuid4()))
+            job.add_tier(self.settings['treeherder']['tier'])
+
+            job.add_product_name('firefox')
+
+            job.add_project(self.repository)
+            job.add_revision_hash(self.retrieve_revision_hash())
+
+            # Add platform and build information
+            job.add_machine(socket.getfqdn())
+            platform = self._get_treeherder_platform()
+            job.add_machine_info(*platform)
+            job.add_build_info(*platform)
+
+            # TODO debug or others?
+            job.add_option_collection({'opt': True})
+
+            # TODO: Add e10s group once we run those tests
+            job.add_group_name(self.settings['treeherder']['group_name'].format(**kwargs))
+            job.add_group_symbol(self.settings['treeherder']['group_symbol'].format(**kwargs))
+
+            # Bug 1174973 - for now we need unique job names even in different groups
+            job.add_job_name(self.settings['treeherder']['job_name'].format(**kwargs))
+            job.add_job_symbol(self.settings['treeherder']['job_symbol'].format(**kwargs))
+
+            job.add_start_timestamp(int(time.time()))
+
+            # Bug 1175559 - Workaround for HTTP Error
+            job.add_end_timestamp(0)
+
+        return job
+
+    def retrieve_revision_hash(self):
+        """Retrieves the unique hash for the current revision."""
+        if not self.url:
+            raise ValueError('URL for Treeherder is missing.')
+
+        lookup_url = urljoin(self.url,
+                             RESULTSET_FRAGMENT.format(repository=self.repository,
+                                                       revision=self.revision))
+
+        if self.url == "mock":
+            logger.info('Pretend to get revision hash from: {}'.format(lookup_url))
+            return None
+
+        logger.info('Getting revision hash from: {}'.format(lookup_url))
+
+        response = requests.get(lookup_url)
+        response.raise_for_status()
+
+        if not response.json():
+            raise ValueError('Unable to determine revision hash for {}. '
+                             'Perhaps it has not been ingested by '
+                             'Treeherder?'.format(self.revision))
+
+        return response.json()['results'][0]['revision_hash']
+
+    def submit(self, job):
+        """Submit the job to treeherder.
+
+        :param job: Treeherder job instance to use for submission.
+
+        """
+        job.add_submit_timestamp(int(time.time()))
+
+        # We can only submit job info once, so it has to be done in completed
+        if self._job_details:
+            job.add_artifact('Job Info', 'json', {'job_details': self._job_details})
+
+        job_collection = TreeherderJobCollection()
+        job_collection.add(job)
+
+        logger.info('Sending results to Treeherder: {}'.format(job_collection.to_json()))
+        if self.url == 'mock':
+            logger.info('Pretending to submit job')
+            return
+
+        url = urlparse(self.url)
+        client = TreeherderClient(protocol=url.scheme, host=url.hostname,
+                                  client_id=self.client_id, secret=self.secret)
+        client.post_collection(self.repository, job_collection)
+
+        logger.info('Results are available to view at: {}'.format(
+                    urljoin(self.url,
+                            JOB_FRAGMENT.format(repository=self.repository,
+                                                revision=self.revision))))
+
+    def submit_running_job(self, job):
+        """Submit job as state running.
+
+        :param job: Treeherder job instance to use for submission.
+
+        """
+        job.add_state('running')
+        self.submit(job)
+
+    def submit_completed_job(self, job, perfdata, state="success"):
+        """Submit job as state completed.
+
+        :param job: Treeherder job instance to use for submission.
+        :param state: success, testfailed, busted, skipped, exception, retry, usercancel 
+        :param uploaded_logs: List of uploaded logs to reference in the job.
+
+        """
+        job.add_state('completed')
+        job.add_result(state)
+
+        jsondata = json.dumps({'performance_data': perfdata})
+        job.add_artifact('performance_data', 'json', jsondata)
+
+        job.add_end_timestamp(int(time.time()))
+
+        self.submit(job)
+
+
+def upload_log_files(guid, logs,
+                     bucket_name=None, access_key_id=None, access_secret_key=None):
+    """Upload all specified logs to Amazon S3.
+
+    :param guid: Unique ID which is used as subfolder name for all log files.
+    :param logs: List of log files to upload.
+    :param bucket_name: Name of the S3 bucket.
+    :param access_key_id: Client ID used for authentication.
+    :param access_secret_key: Secret key for authentication.
+
+    """
+    # If no AWS credentials are given we don't upload anything.
+    if not bucket_name:
+        logger.info('No AWS Bucket name specified - skipping upload of artifacts.')
+        return {}
+
+    s3_bucket = S3Bucket(bucket_name, access_key_id=access_key_id,
+                         access_secret_key=access_secret_key)
+
+    uploaded_logs = {}
+
+    for log in logs:
+        try:
+            if os.path.isfile(logs[log]):
+                remote_path = '{dir}/{filename}'.format(dir=str(guid),
+                                                        filename=os.path.basename(log))
+                url = s3_bucket.upload(logs[log], remote_path)
+
+                uploaded_logs.update({log: {'path': logs[log], 'url': url}})
+                logger.info('Uploaded {path} to {url}'.format(path=logs[log], url=url))
+
+        except Exception:
+            logger.exception('Failure uploading "{path}" to S3'.format(path=logs[log]))
+
+    return uploaded_logs
+

treeherder/update.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from submission import Submission
+import json
+import sys
+
+sys.path.append("../server") 
+import awfy
+import tables
+
+def first(gen):
+    return list(gen)[0]
+
+class Submitter(object):
+    def __init__(self):
+        pass
+
+    """
+    Submit the given data
+    """
+    def submit(self, revision, data, mode_info):
+        repo = mode_info["repo"]
+        settings = {
+            "treeherder": {
+                'group_symbol': 'AWFY',
+                'group_name': 'AWFY',
+                'job_name': mode_info["job_name"],
+                'job_symbol': mode_info["job_symbol"],
+                "tier": mode_info["tier"]
+            }
+        }
+
+        th = Submission(repo, revision,
+                        treeherder_url = awfy.th_host,
+                        treeherder_client_id = awfy.th_user, 
+                        treeherder_secret = awfy.th_secret,
+                        settings = settings)
+
+        job = th.create_job(None)
+        th.submit_completed_job(job, data)
+
+    """
+    Takes all scores/subscores from a build and submit the data to treeherder.
+    """
+    def submitBuild(self, build):
+        revision = build.get("cset")
+        machine_id = build.get("run").get("machine_id")
+        mode_symbol = build.get("mode").get("mode")
+        perfdata = []
+
+        mode_info = config.mode_info(machine_id, mode_symbol)
+        if not mode_info:
+            print "Couldn't submit", revision, "with mode", mode_symbol
+            print "No data found in the config file about how to submit it" 
+            return
+
+        scores = build.getScores()
+        for score in scores:
+            if not score.get("suite_version") or not score.get("suite_version").exists():
+                continue
+            suite_version = score.get("suite_version")
+            perfdata.append({
+                "name": suite_version.get("name"),
+                "score": score.get("score"),
+                "lowerIsBetter": suite_version.get("suite").get("better_direction") == -1,
+                "subscores": {}
+            })
+            for breakdown in score.getBreakdowns():
+                if not breakdown.get("suite_test") or not breakdown.get("suite_test").exists():
+                    continue
+                suite_test = breakdown.get("suite_test")
+                perfdata[-1]["subscores"][suite_test.get("name")] = breakdown.get("score")
+
+        data = self.transform(perfdata)
+
+        self.submit(revision, data, mode_info)
+
+      
+    """
+    Takes all builds from a run and submit the enabled ones to treeherder.
+    """
+    def submitRun(self, run):
+        # Annonate run that it was forwared to treeherder.
+        run.update({"treeherder": 1})
+        awfy.db.commit()
+
+        # Send the data.
+        modes = config.modes(run.get("machine_id"))
+        for mode in modes:
+            mode = first(tables.Mode.where({"mode": mode}))
+            build = tables.Build.fromRunAndMode(run.id, mode.id)
+            self.submitBuild(build)
+
+    """
+    transforms the intermediate representation of benchmark results pulled from the DB
+    into the canonical format needed by treeherder/perfherder
+    """
+    def transform(self, tests):
+        data = {
+            "framework": {
+                "name": "awfy"
+            },
+            "suites": []
+        }
+        for test in tests:
+            testdata = {
+                "name": test["name"],
+                "value": float(test["score"]),
+                "subtests": [],
+                "lowerIsBetter": bool(test["lowerIsBetter"])
+            }
+            if "subscores" not in test:
+                test["subscores"] = []
+            for subtest in test["subscores"]:
+                subtestdata = {
+                    "lowerIsBetter": bool(test["lowerIsBetter"]),
+                    "name": subtest,
+                    "value": float(test["subscores"][subtest])
+                }
+                testdata["subtests"].append(subtestdata)
+            data["suites"].append(testdata)
+        return data 
+
+class Config(object):
+  def __init__(self, filename):
+    fp = open(filename)
+    self.data = json.load(fp)
+    fp.close()
+
+  def modes(self, machine_id):
+    for machine_data in self.data:
+        if str(machine_data["machine"]) != str(machine_id):
+            continue
+        return [mode["mode"] for mode in machine_data["modes"] if mode["enabled"]]
+    return []
+  def mode_info(self, machine_id, mode_symbol):
+    for machine_data in self.data:
+        if str(machine_data["machine"]) != str(machine_id):
+            continue
+        for mode in machine_data["modes"]:
+            if mode["mode"] != mode_symbol:
+                continue
+            return mode 
+    return None
+  def validate(self, schema):
+    from jsonschema import validate
+    fp = open(schema)
+    schema = json.load(fp)
+    fp.close()
+    validate(self.data, schema)
+
+if __name__ == '__main__':
+    config = Config("config.json")
+    config.validate("config.schema")
+    
+    submitter = Submitter()
+    for run in tables.Run.where({"status": 1, "treeherder": 0}):
+      submitter.submitRun(run)
+