Bug 930143 - Make it easier to generate a webpage of results using get-metric-for-build;r=davehunt

bin/get-metric-for-build.py

@@ -4,12 +4,14 @@ import datetime
 import eideticker
 import json
 import os
+import shutil
 import sys
 import time
 import videocapture
 
 CAPTURE_DIR = os.path.join(os.path.dirname(__file__), "../captures")
 PROFILE_DIR = os.path.join(os.path.dirname(__file__), "../profiles")
+DASHBOARD_DIR = os.path.join(os.path.dirname(__file__), "../src/dashboard")
 
 def runtest(device_prefs, testname, options, apk=None, appname = None,
             appdate = None):
@@ -67,18 +69,29 @@ def runtest(device_prefs, testname, options, apk=None, appname = None,
             capture_result['file'] = capture_file
 
             capture = videocapture.Capture(capture_file)
-            capture_result['capture_fps'] = capture.fps
+            capture_result['captureFPS'] = capture.fps
 
             if stableframecapture:
-                capture_result['stableframetime'] = eideticker.get_stable_frame_time(capture)
+                capture_result['timetostableframe'] = eideticker.get_stable_frame_time(capture)
             else:
                 capture_result.update(
                     eideticker.get_standard_metrics(capture, testlog.actions))
             if options.outputdir:
-                video_path = os.path.join('videos', 'video-%s.webm' % time.time())
+                # video
-                video_file = os.path.join(options.outputdir, video_path)
+                video_relpath = os.path.join('videos', 'video-%s.webm' % time.time())
-                open(video_file, 'w').write(capture.get_video().read())
+                video_path = os.path.join(options.outputdir, video_relpath)
-                capture_result['video'] = video_path
+                open(video_path, 'w').write(capture.get_video().read())
+                capture_result['video'] = video_relpath
+
+                # framediff
+                framediff_relpath = os.path.join('framediffs', 'framediff-%s.json' % time.time())
+                framediff_path = os.path.join(options.outputdir, framediff_relpath)
+                with open(framediff_path, 'w') as f:
+                    framediff = videocapture.get_framediff_sums(capture)
+                    f.write(json.dumps({ 'diffsums': framediff }))
+                capture_result['frameDiff'] = framediff_relpath
+
+
 
         if options.enable_profiling:
             capture_result['profile'] = profile_file
@@ -91,7 +104,7 @@ def runtest(device_prefs, testname, options, apk=None, appname = None,
     if options.devicetype == "b2g":
         # FIXME: get information from sources.xml and application.ini on
         # device, as we do in update-dashboard.py
-        display_key = "FirefoxOS"
+        display_key = appkey = "FirefoxOS"
     else:
         appkey = appname
         if appdate:
@@ -109,7 +122,7 @@ def runtest(device_prefs, testname, options, apk=None, appname = None,
     if not options.no_capture:
         if stableframecapture:
             print "  Times to first stable frame (seconds):"
-            print "  %s" % map(lambda c: c['stableframetime'], capture_results)
+            print "  %s" % map(lambda c: c['timetostableframe'], capture_results)
             print
         else:
             print "  Number of unique frames:"
@@ -143,7 +156,7 @@ def runtest(device_prefs, testname, options, apk=None, appname = None,
         print
 
     if options.outputdir:
-        outputfile = os.path.join(options.outputdir, "metric-test-%s.json" % time.time())
+        outputfile = os.path.join(options.outputdir, "metric.json")
         resultdict = { 'title': testname, 'data': {} }
         if os.path.isfile(outputfile):
             resultdict.update(json.loads(open(outputfile).read()))
@@ -164,7 +177,7 @@ def main(args=sys.argv[1:]):
                       help = "number of runs (default: 1)")
     parser.add_option("--output-dir", action="store",
                       type="string", dest="outputdir",
-                      help="output results to json file")
+                      help="output results to web site")
     parser.add_option("--no-capture", action="store_true",
                       dest = "no_capture",
                       help = "run through the test, but don't actually "
@@ -222,6 +235,35 @@ def main(args=sys.argv[1:]):
 
     device_prefs = eideticker.getDevicePrefs(options)
 
+    if options.outputdir:
+        for dirname in [ options.outputdir,
+                         os.path.join(options.outputdir, 'css'),
+                         os.path.join(options.outputdir, 'js'),
+                         os.path.join(options.outputdir, 'videos'),
+                         os.path.join(options.outputdir, 'framediffs'),
+                         ]:
+            if not os.path.exists(dirname):
+                os.makedirs(dirname)
+        for filename in [
+            'metric.html',
+            'framediff-view.html',
+            'css/bootstrap.min.css',
+            'js/ICanHaz.min.js',
+            'js/SS.min.js',
+            'js/common.js',
+            'js/framediff.js',
+            'js/jquery-1.7.1.min.js',
+            'js/jquery.flot.axislabels.js',
+            'js/jquery.flot.js',
+            'js/jquery.flot.stack.js',
+            'js/metric.js' ]:
+            if filename == 'metric.html':
+                outfilename = 'index.html'
+            else:
+                outfilename = filename
+            shutil.copyfile(os.path.join(DASHBOARD_DIR, filename),
+                            os.path.join(options.outputdir, outfilename))
+
     if options.devicetype == "b2g":
         runtest(device_prefs, testname, options)
     elif appnames:

src/dashboard/framediff-view.html

@@ -2,7 +2,7 @@
 <html lang="en">
   <head>
     <meta charset="utf-8">
-    <title>Eideticker Dashboard</title>
+    <title>Frame Difference View</title>
 
     <link href="css/bootstrap.min.css" rel="stylesheet">
 
@@ -70,6 +70,7 @@
     <script src="js/jquery.flot.stack.js"></script>
     <script src="js/jquery.flot.axislabels.js"></script>
     <script src="js/jquery.flot.navigate.js"></script>
+    <script src="js/common.js"></script>
     <script src="js/framediff.js"></script>
 
     <script id="pageHeader" type="text/html">

src/dashboard/index.html

@@ -112,6 +112,7 @@
     <script src="js/jquery.flot.axislabels.js"></script>
     <script src="js/jquery.flot.hiddengraphs.js"></script>
     <script src="js/jquery.flot.navigate.js"></script>
+    <script src="js/common.js"></script>
     <script src="js/index.js"></script>
 
   <script id="graph" type="text/html">

src/dashboard/js/bootstrap.min.js

@@ -0,0 +1 @@
+../../bootstrap/js/bootstrap.min.js

src/dashboard/js/common.js

@@ -0,0 +1,32 @@
+// various utility functions used by all eideticker web interface pages
+
+function getParameterByName(name) {
+  var match = RegExp('[?&]' + name + '=([^&]*)').exec(window.location.search);
+  if (!match)
+    return null;
+  
+  return decodeURIComponent(match[1].replace(/\+/g, ' ')).replace(
+      /[\"\']/g, ' ');
+}
+
+function parseDate(datestr) {
+  var parsed = datestr.split("-");
+  var year = parsed[0];
+  var month = parsed[1] - 1; //Javascript months index from 0 instead of 1
+  var day = parsed[2];
+
+  return Date.UTC(year, month, day);
+}
+
+var measures = {
+  'checkerboard': { 'shortDesc': 'Checkerboard',
+                    'longDesc': 'The measure is the sum of the percentages of frames that are checkerboarded over the entire capture. Lower values are better.' },
+  'uniqueframes': { 'shortDesc': 'Unique frames',
+                    'longDesc': 'The measure is a calculation of the average number of UNIQUE frames per second of capture. The theoretical maximum is 60 fps (which is what we are capturing), but note that if there periods where the page being captured is unchanging this number may be aritifically low.' },
+  'fps': { 'shortDesc': 'Frames per second',
+           'longDesc': 'The measure is a calculation of the average number of UNIQUE frames per second of capture. The theoretical maximum is 60 fps (which is what we are capturing), but note that if there periods where the page being captured is unchanging this number may be aritifically low.' },
+  'timetostableframe': { 'shortDesc': 'Time to first stable frame (seconds)',
+                         'longDesc': 'The time to the first frame of the capture where the image is stable (i.e. mostly unchanging). This is a startup metric that indicates roughly when things have stopped loading. Lower values are better.' },
+  'timetoresponse': { 'shortDesc': 'Time to visible response (seconds)',
+                      'longDesc': 'Time between event being first sent to device and an observable response. A long pause may indicate that the application is unresponsive.' }
+}

src/dashboard/js/framediff.js

@@ -1,15 +1,6 @@
 "use strict";
 
 $(function() {
-  function getParameterByName(name) {
-    var match = RegExp('[?&]' + name + '=([^&]*)').exec(window.location.search);
-    if (!match)
-      return null;
-
-    return decodeURIComponent(match[1].replace(/\+/g, ' ')).replace(
-        /[\"\']/g, ' ');
-  }
-
   function render(diffsums, actions) {
     var seriesList = [];
     var currentSeries = null;
@@ -20,6 +11,9 @@ $(function() {
     var i = 0.0;
     var actionCount = {};
 
+    var fps = getParameterByName('fps');
+    if (!fps) fps = 60;
+
     diffsums.forEach(function(diffsum) {
       // if we have a current action, check to make sure
       // we're still within it
@@ -70,7 +64,7 @@ $(function() {
         lastSeries = null;
       }
       currentSeries.data.push([ i, diffsum ]);
-      i+=(1.0/60.0);
+      i+=(1.0/fps);
     });
 
     if (currentSeries)

src/dashboard/js/index.js

@@ -8,29 +8,6 @@ function getResourceURL(path) {
   return serverPrefix + path;
 }
 
-
-function parseDate(datestr) {
-  var parsed = datestr.split("-");
-  var year = parsed[0];
-  var month = parsed[1] - 1; //Javascript months index from 0 instead of 1
-  var day = parsed[2];
-
-  return Date.UTC(year, month, day);
-}
-
-var measures = {
-  'checkerboard': { 'shortDesc': 'Checkerboard',
-                    'longDesc': 'The measure is the sum of the percentages of frames that are checkerboarded over the entire capture. Lower values are better.' },
-  'uniqueframes': { 'shortDesc': 'Unique frames',
-                    'longDesc': 'The measure is a calculation of the average number of UNIQUE frames per second of capture. The theoretical maximum is 60 fps (which is what we are capturing), but note that if there periods where the page being captured is unchanging this number may be aritifically low.' },
-  'fps': { 'shortDesc': 'Frames per second',
-           'longDesc': 'The measure is a calculation of the average number of UNIQUE frames per second of capture. The theoretical maximum is 60 fps (which is what we are capturing), but note that if there periods where the page being captured is unchanging this number may be aritifically low.' },
-  'timetostableframe': { 'shortDesc': 'Time to first stable frame (seconds)',
-                         'longDesc': 'The time to the first frame of the capture where the image is stable (i.e. mostly unchanging). This is a startup metric that indicates roughly when things have stopped loading. Lower values are better.' },
-  'timetoresponse': { 'shortDesc': 'Time to visible response (seconds)',
-                      'longDesc': 'Time between event being first sent to device and an observable response. A long pause may indicate that the application is unresponsive.' }
-}
-
 function updateContent(testInfo, deviceId, testId, measureId) {
   $.getJSON(getResourceURL(deviceId + '/' + testId + '.json'), function(dict) {
     if (!dict || !dict['testdata']) {

src/dashboard/js/metric.js

@@ -1,24 +1,19 @@
 "use strict";
 
-function getParameterByName(name) {
+var availableMeasures = [];
-  var match = RegExp('[?&]' + name + '=([^&]*)').exec(window.location.search);
-
-  return match && decodeURIComponent(match[1].replace(/\+/g, ' '));
-}
 
 function updateContent(title, measure) {
 
-  var measureDescription;
+  var measureDescription = measures[measure].longDesc;
-  if (measure === "checkerboard") {
-    measureDescription = 'The measure is the sum of the percentages of frames that are checkerboarded over the entire capture. Lower values are better.';
-  } else if (measure === "fps") {
-    measureDescription = 'The measure is a calculation of the average number of UNIQUE frames per second of capture. The theoretical maximum is 60 fps (which is what we are capturing), but note that if there periods where the page being captured is unchanging this number may be artifically low.';
-  } else {
-    measureDescription = 'The measure is a calculation of the total number of UNIQUE frames in the capture. Higher values generally indicate more fluid animations.';
-  }
 
   $('#content').html(ich.graph({'title': title,
-                                'measureDescription': measureDescription
+                                'measureDescription': measureDescription,
+                                'measures': availableMeasures.map(
+                                  function(measureId) {
+                                    return { 'id': measureId,
+                                             'desc': measures[measureId].shortDesc
+                                           };
+                                  })
                                }));
   $('#measure-'+measure).attr("selected", "true");
   $('#measure').change(function() {
@@ -66,7 +61,7 @@ function updateGraph(rawdata, measure) {
     rawdata[appname].forEach(function(sample) {
       series.data.push([ barPosition, sample[measure] ]);
 
-      metadataHash[seriesIndex].push({'videoURL': sample.video, 'appDate': sample.appdate, 'revision': sample.revision, 'buildId': sample.buildid });
+      metadataHash[seriesIndex].push({'videoURL': sample.video, 'appDate': sample.appdate, 'revision': sample.revision, 'buildId': sample.buildid, 'frameDiff': sample.frameDiff, 'fps': sample.captureFPS });
       barPosition++;
     });
     graphdata.push(series);
@@ -106,7 +101,9 @@ function updateGraph(rawdata, measure) {
                                                      'appDate': metadata.appDate,
                                                      'revision': metadata.revision,
                                                      'buildId': metadata.buildId,
-                                                     'measureValue': Math.round(100.0*item.datapoint[1])/100.0
+                                                     'measureValue': Math.round(100.0*item.datapoint[1])/100.0,
+                                                     'frameDiff': metadata.frameDiff,
+                                                     'fps': metadata.fps
                                                    }));
       $('#video').css('width', $('#video').parent().width());
       $('#video').css('max-height', $('#graph-container').height());
@@ -119,16 +116,31 @@ function updateGraph(rawdata, measure) {
 }
 
 $(function() {
-  var router = Router({ '/(checkerboard|fps|uniqueframes)': {
-    on: function(measure) {
 
-      var jsonFile = getParameterByName('data');
+  var dataFilename = getParameterByName('data');
-      $.getJSON(jsonFile, function(data) {
+  if (!dataFilename)
-        console.log(data);
+    dataFilename = 'metric.json';
-        updateContent(data['title'], measure);
+
-        updateGraph(data['data'], measure);
+  $.getJSON(dataFilename, function(data) {
+    // figure out which measures could apply to this graph
+    Object.keys(data['data']).forEach(function(appname) {
+      data['data'][appname].forEach(function(sample) {
+        Object.keys(sample).forEach(function(potentialMeasure) {
+          if (jQuery.inArray(potentialMeasure, Object.keys(measures)) !== -1 &&
+              jQuery.inArray(potentialMeasure, availableMeasures) === -1) {
+            availableMeasures.push(potentialMeasure);
+          }
+        });
       });
-    }
+    });
-  } }).init('/fps');
 
+    var defaultMeasure = availableMeasures[0];
+
+    var router = Router({ '/:measureId': {
+      on: function(measureId) {
+        updateContent(data['title'], measureId);
+        updateGraph(data['data'], measureId);
+      }
+    } }).init('/' + defaultMeasure);
+  });
 });

src/dashboard/metric.html

@@ -37,12 +37,12 @@
     </div>
 
     <script src="js/jquery-1.7.1.min.js"></script>
-    <script src="js/bootstrap.min.js"></script>
     <script src="js/ICanHaz.min.js"></script>
     <script src="js/SS.min.js"></script>
     <script src="js/jquery.flot.js"></script>
     <script src="js/jquery.flot.stack.js"></script>
     <script src="js/jquery.flot.axislabels.js"></script>
+    <script src="js/common.js"></script>
     <script src="js/metric.js"></script>
 
   <script id="graph" type="text/html">
@@ -58,13 +58,15 @@
         <div id="measure-form">
           <form>
             <select id="measure" type="text">
-              <option id="measure-fps" value="fps">Frames Per Second</option>
+              {{#measures}}
-              <option id="measure-uniqueframes" value="uniqueframes">Unique Frames</option>
+              <option id="measure-{{id}}" value="{{id}}">{{desc}}</option>
-              <option id="measure-checkerboard" value="checkerboard">Checkerboard</option>
+              {{/measures}}
             </select>
-            <span class="help-inline">
+            <p>
-              {{measureDescription}}
+              <span class="help-inline">
-            </span>
+                {{measureDescription}}
+              </span>
+            </p>
           </form>
         </div>
         <p class="help-block"><strong>Hint:</strong> Click on the bars to see a video of the testrun!</p>
@@ -107,6 +109,9 @@
         <dt>{{measureName}}</dt>
         <dd>{{measureValue}}</dd>
       </dl>
+      {{#frameDiff}}
+      <p><a href="framediff-view.html?title=Frame Difference&video={{videoURL}}&framediff={{frameDiff}}&fps={{fps}}" target="_blank">Frame difference view</a></p>
+      {{/frameDiff}}
     </div>
   </script>