bug 390845 - integrate talos with new pageloader, also covers additions/fixes to

make talos easier to set up and configure.  p=anodelman r=bhearsum

testing/performance/talos/base_profile/prefs.js

@@ -5,4 +5,7 @@
 user_pref("capability.principal.codebase.p0.granted", "UniversalPreferencesWrite UniversalXPConnect UniversalPreferencesRead");
 user_pref("capability.principal.codebase.p0.id", "file://");
 user_pref("capability.principal.codebase.p0.subjectName", "");
+user_pref("capability.principal.codebase.p1.granted", "UniversalPreferencesWrite UniversalXPConnect UniversalPreferencesRead");
+user_pref("capability.principal.codebase.p1.id", "http://localhost");
+user_pref("capability.principal.codebase.p1.subjectName", "");
 user_pref("signed.applets.codebase_principal_support", true);

testing/performance/talos/cmanager_linux.py

@@ -0,0 +1,220 @@
+#!/usr/bin/env python
+#
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is standalone Firefox Windows performance test.
+#
+# The Initial Developer of the Original Code is Google Inc.
+# Portions created by the Initial Developer are Copyright (C) 2006
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Annie Sullivan <annie.sullivan@gmail.com> (original author)
+#   Ben Hearsum    <bhearsum@wittydomain.com> (ported to linux)
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+"""A set of functions to run the Tp test.
+
+   The Tp test measures page load times in Firefox.  It does this with a
+   JavaScript script that opens a new window and cycles through page loads
+   from the local disk, timing each one.  The script dumps the sum of the
+   mean times to open each page, and the standard deviation, to standard out.
+   We can also measure performance attributes during the test.  See below for
+   what can be monitored
+"""
+
+__author__ = 'annie.sullivan@gmail.com (Annie Sullivan)'
+
+
+import os
+import time
+import threading
+
+import ffprocess
+
+def GetPrivateBytes(pid):
+  """Calculate the amount of private, writeable memory allocated to a process.
+     This code was adapted from 'pmap.c', part of the procps project.
+  """
+  mapfile = '/proc/%s/maps' % pid
+  maps = open(mapfile)
+
+  private = 0
+
+  for line in maps:
+    # split up
+    (range,line) = line.split(" ", 1)
+
+    (start,end) = range.split("-")
+    flags = line.split(" ", 1)[0]
+
+    size = int(end, 16) - int(start, 16)
+
+    if flags.find("p") >= 0:
+      if flags.find("w") >= 0:
+        private += size
+
+  return private
+
+
+def GetResidentSize(pid):
+  """Retrieve the current resident memory for a given process"""
+  # for some reason /proc/PID/stat doesn't give accurate information
+  # so we use status instead
+  file = '/proc/%s/status' % pid
+
+  status = open(file)
+
+  for line in status:
+    if line.find("VmRSS") >= 0:
+      return int(line.split()[1]) * 1024
+
+
+def GetCpuTime(pid, sampleTime=1):
+  # return all zeros on this platform as per the 7/18/07 perf meeting
+  return 0
+
+counterDict = {}
+counterDict["Private Bytes"] = GetPrivateBytes
+counterDict["RSS"] = GetResidentSize
+counterDict["% Processor Time"] = GetCpuTime
+
+class CounterManager(threading.Thread):
+  """This class manages the monitoring of a process with any number of
+     counters.
+
+     A counter can be any function that takes an argument of one pid and
+     returns a piece of data about that process.
+     Some examples are: CalcCPUTime, GetResidentSize, and GetPrivateBytes
+  """
+  
+  pollInterval = .25
+
+  def __init__(self, process, counters=None):
+    """Args:
+         counters: A list of counters to monitor. Any counters whose name does
+         not match a key in 'counterDict' will be ignored.
+    """
+    self.allCounters = {}
+    self.registeredCounters = {}
+    self.process = process
+    self.runThread = False
+    self.pid = -1
+
+    self._loadCounters()
+    self.registerCounters(counters)
+
+    threading.Thread.__init__(self)
+
+  def _loadCounters(self):
+    """Loads all of the counters defined in the counterDict"""
+    for counter in counterDict.keys():
+      self.allCounters[counter] = counterDict[counter]
+
+  def registerCounters(self, counters):
+    """Registers a list of counters that will be monitoring.
+       Only counters whose names are found in allCounters will be added
+    """
+    for counter in counters:
+      if counter in self.allCounters:
+        self.registeredCounters[counter] = \
+          [self.allCounters[counter], []]
+
+  def unregisterCounters(self, counters):
+    """Unregister a list of counters.
+       Only counters whose names are found in registeredCounters will be
+       paid attention to
+    """
+    for counter in counters:
+      if counter in self.registeredCounters:
+        del self.registeredCounters[counter]
+
+  def getRegisteredCounters(self):
+    """Returns a list of the registered counters."""
+    return keys(self.registeredCounters)
+
+  def getCounterValue(self, counterName):
+    """Returns the last value of the counter 'counterName'"""
+    try:
+      if counterName is "% Processor Time":
+        return self._getCounterAverage(counterName)
+      else:
+        return self.registeredCounters[counterName][1][-1]
+    except:
+      return None
+
+  def _getCounterAverage(self, counterName):
+    """Returns the average value of the counter 'counterName'"""
+    try:
+      total = 0
+      for v in self.registeredCounters[counterName][1]:
+        total += v
+      return total / len(self.registeredCounters[counterName][1])
+    except:
+      return None
+
+  def getProcess(self):
+    """Returns the process currently associated with this CounterManager"""
+    return self.process
+
+  def startMonitor(self):
+    """Starts the monitoring process.
+       Throws an exception if any error occurs
+    """
+    # TODO: make this function less ugly
+    try:
+      # the last process is the useful one
+      self.pid = ffprocess.GetPidsByName(self.process)[-1]
+      os.stat('/proc/%s' % self.pid)
+      self.runThread = True
+      self.start()
+    except:
+      print 'WARNING: problem starting counter monitor'
+
+  def stopMonitor(self):
+    """Stops the monitor"""
+    # TODO: should probably wait until we know run() is completely stopped
+    # before setting self.pid to None. Use a lock?
+    self.runThread = False
+
+  def run(self):
+    """Performs the actual monitoring of the process. Will keep running
+       until stopMonitor() is called
+    """
+    while self.runThread:
+      for counter in self.registeredCounters.keys():
+        # counter[0] is a function that gets the current value for
+        # a counter
+        # counter[1] is a list of recorded values
+        try:
+          self.registeredCounters[counter][1].append(
+            self.registeredCounters[counter][0](self.pid))
+        except:
+          # if a counter throws an exception, remove it
+          self.unregisterCounters([counter])
+
+      time.sleep(self.pollInterval)

testing/performance/talos/cmanager_mac.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python
+#
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is standalone Firefox Windows performance test.
+#
+# The Initial Developer of the Original Code is Google Inc.
+# Portions created by the Initial Developer are Copyright (C) 2006
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Annie Sullivan <annie.sullivan@gmail.com> (original author)
+#   Ben Hearsum    <bhearsum@wittydomain.com> (ported to linux)
+#   Zach Lipton    <zach@zachlipton.com>  (Mac port)
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+"""A set of functions to run the Tp test.
+
+   The Tp test measures page load times in Firefox.  It does this with a
+   JavaScript script that opens a new window and cycles through page loads
+   from the local disk, timing each one.  The script dumps the sum of the
+   mean times to open each page, and the standard deviation, to standard out.
+   We can also measure performance attributes during the test.  See below for
+   what can be monitored
+"""
+
+__author__ = 'annie.sullivan@gmail.com (Annie Sullivan)'
+
+
+import os
+import time
+import threading
+import subprocess
+
+import ffprocess
+
+def GetProcessData(pid):
+  """Runs a ps on the process identified by pid and returns the output line
+    as a list (uid, pid, ppid, cpu, pri, ni, vsz, rss, wchan, stat, tt, time, command)
+  """
+  command = ['ps -Acup'+str(pid)]
+  handle = subprocess.Popen(command, stdout=subprocess.PIPE, universal_newlines=True, shell=True)
+  handle.wait()
+  data = handle.stdout.readlines()
+  
+  # find all matching processes and add them to the list
+  for line in data:
+    if line.find(str(pid)) >= 0:
+      # splits by whitespace
+      line = line.split()
+      if (line[1] == str(pid)):
+        return line
+
+def GetPrivateBytes(pid):
+  """Calculate the amount of private, writeable memory allocated to a process.
+  """
+  psData = GetProcessData(pid)
+  return psData[5]
+
+
+def GetResidentSize(pid):
+  """Retrieve the current resident memory for a given process"""
+  psData = GetProcessData(pid)
+  return psData[4]
+
+def GetCpuTime(pid):
+  # return all zeros for now on this platform as per 7/18/07 perf meeting
+  return 0
+
+counterDict = {}
+counterDict["Private Bytes"] = GetPrivateBytes
+counterDict["RSS"] = GetResidentSize
+counterDict["% Processor Time"] = GetCpuTime
+
+class CounterManager(threading.Thread):
+  """This class manages the monitoring of a process with any number of
+     counters.
+
+     A counter can be any function that takes an argument of one pid and
+     returns a piece of data about that process.
+     Some examples are: CalcCPUTime, GetResidentSize, and GetPrivateBytes
+  """
+  
+  pollInterval = .25
+
+  def __init__(self, process, counters=None):
+    """Args:
+         counters: A list of counters to monitor. Any counters whose name does
+         not match a key in 'counterDict' will be ignored.
+    """
+    self.allCounters = {}
+    self.registeredCounters = {}
+    self.process = process
+    self.runThread = False
+    self.pid = -1
+
+    self._loadCounters()
+    self.registerCounters(counters)
+
+    threading.Thread.__init__(self)
+
+  def _loadCounters(self):
+    """Loads all of the counters defined in the counterDict"""
+    for counter in counterDict.keys():
+      self.allCounters[counter] = counterDict[counter]
+
+  def registerCounters(self, counters):
+    """Registers a list of counters that will be monitoring.
+       Only counters whose names are found in allCounters will be added
+    """
+    for counter in counters:
+      if counter in self.allCounters:
+        self.registeredCounters[counter] = \
+          [self.allCounters[counter], []]
+
+  def unregisterCounters(self, counters):
+    """Unregister a list of counters.
+       Only counters whose names are found in registeredCounters will be
+       paid attention to
+    """
+    for counter in counters:
+      if counter in self.registeredCounters:
+        del self.registeredCounters[counter]
+
+  def getRegisteredCounters(self):
+    """Returns a list of the registered counters."""
+    return keys(self.registeredCounters)
+
+  def getCounterValue(self, counterName):
+    """Returns the last value of the counter 'counterName'"""
+    try:
+      if counterName is "% Processor Time":
+        return self._getCounterAverage(counterName)
+      else:
+        return self.registeredCounters[counterName][1][-1]
+    except:
+      return None
+
+  def _getCounterAverage(self, counterName):
+    """Returns the average value of the counter 'counterName'"""
+    try:
+      total = 0
+      for v in self.registeredCounters[counterName][1]:
+        total += v
+      return total / len(self.registeredCounters[counterName][1])
+    except:
+      return None
+
+  def getProcess(self):
+    """Returns the process currently associated with this CounterManager"""
+    return self.process
+
+  def startMonitor(self):
+    """Starts the monitoring process.
+       Throws an exception if any error occurs
+    """
+    # TODO: make this function less ugly
+    try:
+      # the last process is the useful one
+      self.pid = ffprocess.GetPidsByName(self.process)[-1]
+      self.runThread = True
+      self.start()
+    except:
+      print 'WARNING: problem starting counter monitor'
+
+  def stopMonitor(self):
+    """Stops the monitor"""
+    # TODO: should probably wait until we know run() is completely stopped
+    # before setting self.pid to None. Use a lock?
+    self.runThread = False
+
+  def run(self):
+    """Performs the actual monitoring of the process. Will keep running
+       until stopMonitor() is called
+    """
+    while self.runThread:
+      for counter in self.registeredCounters.keys():
+        # counter[0] is a function that gets the current value for
+        # a counter
+        # counter[1] is a list of recorded values
+        try:
+          self.registeredCounters[counter][1].append(
+            self.registeredCounters[counter][0](self.pid))
+        except:
+          # if a counter throws an exception, remove it
+          self.unregisterCounters([counter])
+
+      time.sleep(self.pollInterval)

testing/performance/talos/cmanager_win32.py

@@ -0,0 +1,98 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is standalone Firefox Windows performance test.
+#
+# The Initial Developer of the Original Code is Google Inc.
+# Portions created by the Initial Developer are Copyright (C) 2006
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Annie Sullivan <annie.sullivan@gmail.com> (original author)
+#   Ben Hearsum    <bhearsum@wittydomain.com> (OS independence)
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+
+import win32pdh
+import win32pdhutil
+
+
+class CounterManager:
+
+  def __init__(self, process, counters=None):
+    self.process = process
+    self.registeredCounters = {}
+    self.registerCounters(counters)
+    # PDH might need to be "refreshed" if it has been queried while Firefox
+    # is closed
+    win32pdh.EnumObjects(None, None, 0, 1)
+
+  def registerCounters(self, counters):
+    for counter in counters:
+      self.registeredCounters[counter] = []
+            
+  def unregisterCounters(self, counters):
+    for counter in counters:
+      if counter in self.registeredCounters:
+        del self.registeredCounters[counter]
+
+  def getRegisteredCounters(self):
+    return keys(self.registeredCounters)
+
+  def getCounterValue(self, counter):
+    hq = self.registeredCounters[counter][0]
+    hc = self.registeredCounters[counter][1]
+    try:
+      win32pdh.CollectQueryData(hq)
+      type, val = win32pdh.GetFormattedCounterValue(hc, win32pdh.PDH_FMT_LONG)
+      return val
+    except:
+      return None
+
+  def getProcess(self):
+    return self.process
+
+  def startMonitor(self):
+    # PDH might need to be "refreshed" if it has been queried while Firefox
+    # is closed
+    win32pdh.EnumObjects(None, None, 0, 1)
+
+    for counter in self.registeredCounters:
+      path = win32pdh.MakeCounterPath((None, 'process', self.process,
+                                       None, -1, counter))
+      hq = win32pdh.OpenQuery()
+      try:
+        hc = win32pdh.AddCounter(hq, path)
+      except:
+        win32pdh.CloseQuery(hq)
+
+      self.registeredCounters[counter] = [hq, hc]
+
+  def stopMonitor(self):
+    for counter in self.registeredCounters:
+      win32pdh.RemoveCounter(self.registeredCounters[counter][1])
+      win32pdh.CloseQuery(self.registeredCounters[counter][0])
+    self.registeredCounters.clear()

testing/performance/talos/config.py

@@ -1,97 +0,0 @@
-# ***** BEGIN LICENSE BLOCK *****
-# Version: MPL 1.1/GPL 2.0/LGPL 2.1
-#
-# The contents of this file are subject to the Mozilla Public License Version
-# 1.1 (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-# http://www.mozilla.org/MPL/
-#
-# Software distributed under the License is distributed on an "AS IS" basis,
-# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
-# for the specific language governing rights and limitations under the
-# License.
-#
-# The Original Code is standalone Firefox Windows performance test.
-#
-# The Initial Developer of the Original Code is Google Inc.
-# Portions created by the Initial Developer are Copyright (C) 2006
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-#   Annie Sullivan <annie.sullivan@gmail.com> (original author)
-#   Alice Nodelman <anodelman@mozilla.com>
-#
-# Alternatively, the contents of this file may be used under the terms of
-# either the GNU General Public License Version 2 or later (the "GPL"), or
-# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
-# in which case the provisions of the GPL or the LGPL are applicable instead
-# of those above. If you wish to allow use of your version of this file only
-# under the terms of either the GPL or the LGPL, and not to allow others to
-# use your version of this file under the terms of the MPL, indicate your
-# decision by deleting the provisions above and replace them with the notice
-# and other provisions required by the GPL or the LGPL. If you do not delete
-# the provisions above, a recipient may use your version of this file under
-# the terms of any one of the MPL, the GPL or the LGPL.
-#
-# ***** END LICENSE BLOCK *****
-
-"""A list of constants containing the paths to programs and files
-   needed by the performance testing scripts.
-"""
-
-__author__ = 'annie.sullivan@gmail.com (Annie Sullivan)'
-
-BROWSER_HEIGHT = 768
-BROWSER_WIDTH = 1024
-
-"""For some reason, can only get output from dump() in Firefox if
-   it's run through cygwin bash.  So here's the path to cygwin.
-"""
-CYGWIN = r'c:\cygwin\bin\bash.exe -c'
-
-"""The tinderbox scripts run sync between Ts runs, so we do, too."""
-SYNC = r'c:\cygwin\bin\sync'
-
-"""The path to the base profile directory to use for testing.  For the page
-   load test to work, this profile should have its hostperm.1 file set to allow
-   urls with scheme:file to open in new windows, and the preference to open
-   new windows in a tab should be off.
-"""
-BASE_PROFILE_DIR = r'C:\talos\base_profile'
-
-"""The path to the file url to load when initializing a new profile"""
-INIT_URL = 'file:///c:/talos/initialize.html'
-
-"""The path to the file url to load when collecting information from the browser"""
-INFO_URL = 'file:///c:/talos/getInfo.html'
-
-"""The path to the file url to load for startup test (Ts)"""
-TS_URL = 'file:///c:/talos/startup_test/startup_test.html?begin='
-
-"""Number of times to run startup test (Ts)"""
-TS_NUM_RUNS = 20
-
-"""The path to the file url to load for page load test (Tp)"""
-TP_URL = 'http://localhost/page_load_test/framecycler.html'
-
-"""Number of times the page load test (Tp) loads each page in the test."""
-TP_NUM_CYCLES = 5
-
-"""Resolution of counter sample data for page load test (Tp), in seconds
-   (For example, if TP_RESOLUTION=1, sample counters every 1 second"""
-TP_RESOLUTION = 1
-  
-"""Run page load test.
-   For possible values of counters argument on Windows, see
-   http://technet2.microsoft.com/WindowsServer/en/Library/86b5d116-6fb3-427b-af8c-9077162125fe1033.mspx?mfr=true
-   Possible values on Linux and Mac:
-     'Private Bytes', '% Processor Time', 'RSS'
-"""
-COUNTERS = ['Private Bytes', 'Working Set', '% Processor Time']
-
-"""URL for the results server"""
-RESULTS_SERVER = 'graphserver.url.here'
-RESULTS_LINK = '/bulk.cgi'
-
-"""Enable/disable debugging output"""
-DEBUG = 0

testing/performance/talos/ffprocess.py

@@ -58,12 +58,10 @@ elif platform.system() == "Darwin":
 
 
 
-def SyncAndSleep():
+def Sleep():
   """Runs sync and sleeps for a few seconds between Firefox runs.
      Otherwise "Firefox is already running.." errors occur
   """
-
-  os.spawnl(os.P_WAIT, config.SYNC)
   time.sleep(5)
 
 

testing/performance/talos/ffprocess_linux.py

@@ -44,8 +44,7 @@ import config
 
 
 def GenerateFirefoxCommandLine(firefox_path, profile_dir, url):
-  """Generates the command line for a process to run Firefox, wrapped
-     by cygwin so that we can read the output from dump() statements.
+  """Generates the command line for a process to run Firefox
 
   Args:
     firefox_path: String containing the path to the firefox exe to use
@@ -57,15 +56,9 @@ def GenerateFirefoxCommandLine(firefox_path, profile_dir, url):
   if profile_dir:
     profile_arg = '-profile %s' % profile_dir
 
-  url_arg = ''
-  if url:
-    url_arg = '-url %s' % url
-
-  cmd = '%s %s %s -width %d -height %d' % (firefox_path,
+  cmd = '%s %s %s' % (firefox_path,
                       profile_arg,
-                      url_arg,
-                      config.BROWSER_WIDTH,
-                      config.BROWSER_HEIGHT)
+                      url)
   return cmd
 
 
@@ -121,8 +114,10 @@ def TerminateProcess(pid):
   Args:
     pid: integer process id of the process to terminate.
   """
-
-  os.kill(pid, signal.SIGTERM)
+  try:
+    os.kill(pid, signal.SIGTERM)
+  except OSError, (errno, strerror):
+        print 'WARNING: failed os.kill: %s : %s' % (errno, strerror)
 
 
 def TerminateAllProcesses(process_name):

testing/performance/talos/ffprocess_mac.py

@@ -57,15 +57,9 @@ def GenerateFirefoxCommandLine(firefox_path, profile_dir, url):
   if profile_dir:
     profile_arg = '-profile %s' % profile_dir
 
-  url_arg = ''
-  if url:
-    url_arg = '-url %s' % url
-
-  cmd = '%s %s %s -width %d -height %d' % (firefox_path,
+  cmd = '%s %s %s' % (firefox_path,
                       profile_arg,
-                      url_arg,
-                      config.BROWSER_WIDTH,
-                      config.BROWSER_HEIGHT)
+                      url)
   return cmd
 
 
@@ -120,7 +114,10 @@ def TerminateProcess(pid):
   Args:
     pid: integer process id of the process to terminate.
   """
-  os.kill(pid, signal.SIGTERM)
+  try:
+    os.kill(pid, signal.SIGTERM)
+  except OSError, (errno, strerror):
+    print 'WARNING: failed os.kill: %s : %s' % (errno, strerror)
 
 def TerminateAllProcesses(process_name):
   """Helper function to terminate all processes with the given process name

testing/performance/talos/ffprocess_win32.py

@@ -44,30 +44,8 @@ import msvcrt
 import config
 
 
-def GetCygwinPath(dos_path):
-  """Helper function to get the Cygwin path from a dos path.
-     Used to generate a Firefox command line piped through the
-     Cygwin bash shell
-
-  Args:
-    dos_path: String containing the dos path
-
-  Returns:
-    String containing the cygwin path
-  """
-
-  # Convert the path to a cygwin path. Assumes the path starts with
-  # /cygdrive/driveletter
-  cygwin_path = '/' + dos_path[3:]                       # Remove 'C:\'
-  cygwin_path = cygwin_path.replace('\\', '/')           # Backslashes->slashes
-  cygwin_path = cygwin_path.replace(' ', '\\ ')          # Escape spaces
-  cygwin_path = '/cygdrive/' + dos_path[0] + cygwin_path # Add drive letter
-  return cygwin_path
-
-
 def GenerateFirefoxCommandLine(firefox_path, profile_dir, url):
-  """Generates the command line for a process to run Firefox, wrapped
-     by cygwin so that we can read the output from dump() statements.
+  """Generates the command line for a process to run Firefox
 
   Args:
     firefox_path: String containing the path to the firefox exe to use
@@ -80,16 +58,9 @@ def GenerateFirefoxCommandLine(firefox_path, profile_dir, url):
     profile_dir = profile_dir.replace('\\', '\\\\\\')
     profile_arg = '-profile %s' % profile_dir
 
-  url_arg = ''
-  if url:
-    url_arg = '-url %s' % url
-
-  cmd = '%s "%s %s %s -width %d -height %d"' % (config.CYGWIN,
-                           GetCygwinPath(firefox_path),
+  cmd = '%s %s %s' % (firefox_path,
                            profile_arg,
-                           url_arg,
-                           config.BROWSER_WIDTH,
-                           config.BROWSER_HEIGHT)
+                           url)
   return cmd
 
 

testing/performance/talos/ffsetup.py

@@ -0,0 +1,166 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is standalone Firefox Windows performance test.
+#
+# The Initial Developer of the Original Code is Google Inc.
+# Portions created by the Initial Developer are Copyright (C) 2006
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Annie Sullivan <annie.sullivan@gmail.com> (original author)
+#   Alice Nodelman <anodelman@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+"""A set of functions to set up a Firefox browser with the correct
+   preferences and extensions in the given directory.
+
+"""
+
+__author__ = 'annie.sullivan@gmail.com (Annie Sullivan)'
+
+
+import platform
+import os
+import re
+import shutil
+import tempfile
+import time
+import glob
+
+import utils
+import ffprocess
+import config
+
+if platform.system() == "Linux":
+    from ffprofile_unix import *
+elif platform.system() == "Windows":
+    from ffprofile_win32 import *
+elif platform.system() == "Darwin":
+    from ffprofile_unix import *
+
+def PrefString(name, value, newline):
+  """Helper function to create a pref string for Firefox profile prefs.js
+     in the form 'user_pref("name", value);<newline>'
+
+  Args:
+    name: String containing name of pref
+    value: String containing value of pref
+    newline: Line ending to use, i.e. '\n' or '\r\n'
+
+  Returns:
+    String containing 'user_pref("name", value);<newline>'
+  """
+
+  out_value = str(value)
+  if type(value) == bool:
+    # Write bools as "true"/"false", not "True"/"False".
+    out_value = out_value.lower()
+  if type(value) == str:
+    # Write strings with quotes around them.
+    out_value = '"%s"' % value
+  return 'user_pref("%s", %s);%s' % (name, out_value, newline)
+
+
+def CreateTempProfileDir(source_profile, prefs, extensions):
+  """Creates a temporary profile directory from the source profile directory
+     and adds the given prefs and links to extensions.
+
+  Args:
+    source_profile: String containing the absolute path of the source profile
+                    directory to copy from.
+    prefs: Preferences to set in the prefs.js file of the new profile.  Format:
+           {"PrefName1" : "PrefValue1", "PrefName2" : "PrefValue2"}
+    extensions: Guids and paths of extensions to link to.  Format:
+                {"{GUID1}" : "c:\\Path\\to\\ext1", "{GUID2}", "c:\\Path\\to\\ext2"}
+
+  Returns:
+    String containing the absolute path of the profile directory.
+  """
+
+  # Create a temporary directory for the profile, and copy the
+  # source profile to it.
+  profile_dir = tempfile.mkdtemp()
+  profile_dir = os.path.join(profile_dir, 'profile')
+  shutil.copytree(source_profile, profile_dir)
+  MakeDirectoryContentsWritable(profile_dir)
+
+  # Copy the user-set prefs to user.js
+  user_js_filename = os.path.join(profile_dir, 'user.js')
+  user_js_file = open(user_js_filename, 'w')
+  for pref in prefs:
+    user_js_file.write(PrefString(pref, prefs[pref], '\n'))
+  user_js_file.close()
+
+  # Add links to all the extensions.
+  extension_dir = os.path.join(profile_dir, "extensions")
+  if not os.path.exists(extension_dir):
+    os.makedirs(extension_dir)
+  for extension in extensions:
+    link_file = open(os.path.join(extension_dir, extension), 'w')
+    link_file.write(extensions[extension])
+    link_file.close()
+
+  return profile_dir
+
+def InstallInBrowser(firefox_path, dir_path):
+  """
+    Take the given directory and copies it to appropriate location in the given
+    firefox install
+  """
+  # add the provided directory to the given firefox install
+  fromfiles = glob.glob(os.path.join(dir_path, '*'))
+  todir = os.path.join(os.path.dirname(firefox_path), os.path.basename(os.path.normpath(dir_path)))
+  for fromfile in fromfiles:
+      if not os.path.isfile(os.path.join(todir, os.path.basename(fromfile))):
+          shutil.copy(fromfile, todir)
+          utils.debug("installed " + fromfile)
+      else:
+          utils.debug("WARNING: file already installed (" + fromfile + ")")
+
+def InitializeNewProfile(firefox_path, profile_dir):
+  """Runs Firefox with the new profile directory, to negate any performance
+     hit that could occur as a result of starting up with a new profile.  
+     Also kills the "extra" Firefox that gets spawned the first time Firefox
+     is run with a new profile.
+
+  Args:
+    firefox_path: String containing the path to the Firefox exe
+    profile_dir: The full path to the profile directory to load
+  """
+  PROFILE_REGEX = re.compile('__metrics(.*)__metrics', re.DOTALL|re.MULTILINE)
+  res = 1
+  cmd = ffprocess.GenerateFirefoxCommandLine(firefox_path, profile_dir, config.INIT_URL)
+  (match, timed_out) = ffprocess.RunProcessAndWaitForOutput(cmd,
+                                                              'firefox',
+                                                              PROFILE_REGEX,
+                                                              30)
+  if (not timed_out):
+    print match
+  else:
+    res = 0
+    print "ERROR: no metrics"
+  return res

testing/performance/talos/getInfo.html

@@ -45,7 +45,7 @@
 
   <body onload="
 	if (window.dump) {
-		dump('__metricsScreen width:' + screen.width + ' Screen height:' + screen.height + ' colorDepth:' + screen.colorDepth + '\n\n');
+            dump('__metrics\tScreen width/height:' + screen.width + '/' + screen.height + '\n\tcolorDepth:' + screen.colorDepth + '\n\tBrowser inner width/height: ' + window.innerWidth + '/' + window.innerHeight + '\n\tBrowser outer width/height: ' + window.outerWidth + '/' + window.outerHeight + '__metrics')
 	}
 	goQuitApplication();
 	window.close();

testing/performance/talos/page_load_test/framecycler.html

@@ -5,7 +5,7 @@
 
   var NUM_PAGES;
   var NUM_CYCLES;
-  var DEFAULT_TIMEOUT = 25000; //how long any given page can take to load
+  var DEFAULT_TIMEOUT = 55000; //how long any given page can take to load
   var QUIT; // whether to quit the app (with quit.js) after the tests
   var t; 
 
@@ -134,9 +134,9 @@
 
     var r = getArrayStats(all);
     dump(
-      "__start_tp_report\n" + 
+      "__start_report\n" + 
       "_x_x_mozilla_page_load,"+avgmed+","+r.max+","+r.min+"\n"+
-      "_x_x_mozilla_page_load_details,avgmedian|"+avgmed+"|average|"+avg.toFixed(2)+"|minimum|"+r.min+"|maximum|"+r.max+"|stddev|"+r.stdd.toFixed(2)+"\n"
+      "_x_x_mozilla_page_load_details,avgmedian|"+avgmed+"|average|"+avg.toFixed(2)+"|minimum|"+r.min+"|maximum|"+r.max+"|stddev|"+r.stdd.toFixed(2)
     );
 
     for (var i = 0; i < timeVals.length; ++i) {
@@ -160,7 +160,7 @@
       }
       dump("\n")
     } 
-    dump("__end_tp_report\n\n");
+    dump("__end_report\n\n");
     //rstring += "__end_page_load_report";
     //alert(rstring);
     //dump(rstring);
@@ -194,6 +194,9 @@
 
   function loadFail() {
     dump("__FAILtimeout:" + frames["content"].document.location.href + "__FAIL");
+    if (QUIT) {
+      goQuitApplication();
+    }
     window.close();
   }
 
@@ -211,7 +214,7 @@
       
       // use quit.js to quit the app if the quit param is true and 
       // we have universalxpconnect privs:
-      if (QUIT && canQuitApplication()) {
+      if (QUIT) {
         goQuitApplication();
       }
       

testing/performance/talos/page_load_test/manifest.txt

@@ -0,0 +1,395 @@
+http://localhost/page_load_test/pages/www.yahoo.com/www.yahoo.com/index.html
+http://localhost/page_load_test/pages/www.msn.com/www.msn.com/index.html
+http://localhost/page_load_test/pages/www.google.com/www.google.com/index.html
+http://localhost/page_load_test/pages/www.baidu.com/www.baidu.com/index.html
+http://localhost/page_load_test/pages/www.myspace.com/www.myspace.com/index.html
+http://localhost/page_load_test/pages/www.qq.com/www.qq.com/index.html
+http://localhost/page_load_test/pages/www.live.com/www.live.com/index.html
+http://localhost/page_load_test/pages/www.ebay.com/www.ebay.com/index.html
+http://localhost/page_load_test/pages/www.wikipedia.org/www.wikipedia.org/index.html
+http://localhost/page_load_test/pages/www.163.com/www.163.com/index.html
+http://localhost/page_load_test/pages/www.microsoft.com/www.microsoft.com/index.html
+http://localhost/page_load_test/pages/www.blogger.com/www.blogger.com/start.html
+http://localhost/page_load_test/pages/www.yahoo.com.cn/cn.yahoo.com/index.html
+http://localhost/page_load_test/pages/www.amazon.com/www.amazon.com/index.html
+http://localhost/page_load_test/pages/www.google.co.uk/www.google.co.uk/index.html
+http://localhost/page_load_test/pages/www.taobao.com/www.taobao.com/index.html
+http://localhost/page_load_test/pages/www.google.de/www.google.de/index.html
+http://localhost/page_load_test/pages/www.google.co.jp/www.google.co.jp/index.html
+http://localhost/page_load_test/pages/www.wretch.cc/www.wretch.cc/index.html
+http://localhost/page_load_test/pages/www.google.com.br/www.google.com.br/index.html
+http://localhost/page_load_test/pages/www.bbc.co.uk/www.bbc.co.uk/index.html
+http://localhost/page_load_test/pages/www.uol.com.br/www.uol.com.br/index.html
+http://localhost/page_load_test/pages/www.fotolog.net/www.fotolog.com/index.html
+http://localhost/page_load_test/pages/www.passport.net/accountservices.passport.net/ppnetworkhome.srf@vv=450&lc=1033.html
+http://localhost/page_load_test/pages/www.craigslist.org/sfbay.craigslist.org/index.html
+http://localhost/page_load_test/pages/www.cnn.com/www.cnn.com/index.html
+http://localhost/page_load_test/pages/www.google.com.mx/www.google.com.mx/index.html
+http://localhost/page_load_test/pages/www.imdb.com/www.imdb.com/index.html
+http://localhost/page_load_test/pages/www.flickr.com/www.flickr.com/index.html
+http://localhost/page_load_test/pages/www.mail.ru/www.mail.ru/index.html
+http://localhost/page_load_test/pages/www.xanga.com/www.xanga.com/index.html
+http://localhost/page_load_test/pages/www.aol.com/www.aol.com/index.html
+http://localhost/page_load_test/pages/www.google.es/www.google.es/index.html
+http://localhost/page_load_test/pages/www.yandex.ru/www.yandex.ru/index.html
+http://localhost/page_load_test/pages/www.google.co.in/www.google.co.in/index.html
+#//"http://localhost/page_load_test/pages/www.discuss.com.hk/www.discuss.com.hk/index.html
+http://localhost/page_load_test/pages/www.ebay.co.uk/www.ebay.co.uk/index.html
+http://localhost/page_load_test/pages/www.mixi.jp/mixi.jp/index.html
+http://localhost/page_load_test/pages/www.naver.com/www.naver.com/index.html
+http://localhost/page_load_test/pages/www.friendster.com/www.friendster.com/index.html
+http://localhost/page_load_test/pages/www.google.fr/www.google.fr/index.html
+http://localhost/page_load_test/pages/www.facebook.com/www.facebook.com/index.html
+http://localhost/page_load_test/pages/www.google.pl/www.google.pl/index.html
+http://localhost/page_load_test/pages/www.google.ca/www.google.ca/index.html
+http://localhost/page_load_test/pages/www.google.com.tr/www.google.com.tr/index.html
+http://localhost/page_load_test/pages/www.onet.pl/www.onet.pl/index.html
+http://localhost/page_load_test/pages/www.google.cl/www.google.cl/index.html
+http://localhost/page_load_test/pages/www.pchome.com.tw/www.pchome.com.tw/index.html
+http://localhost/page_load_test/pages/www.terra.com.br/www.terra.com.br/capa/index.html
+http://localhost/page_load_test/pages/www.imageshack.us/www.imageshack.us/index.html
+http://localhost/page_load_test/pages/www.google.com.sa/www.google.com.sa/index.html
+http://localhost/page_load_test/pages/www.rakuten.co.jp/www.rakuten.co.jp/index.html
+http://localhost/page_load_test/pages/www.geocities.com/geocities.yahoo.com/index.html
+http://localhost/page_load_test/pages/www.vnet.cn/www.vnet.cn/default.aspx.html
+http://localhost/page_load_test/pages/www.ebay.de/www.ebay.de/index.html
+http://localhost/page_load_test/pages/www.uwants.com/www.uwants.com/index.html
+http://localhost/page_load_test/pages/www.rediff.com/www.rediff.com/index.html
+http://localhost/page_load_test/pages/www.photobucket.com/photobucket.com/index.html
+http://localhost/page_load_test/pages/www.soso.com/www.soso.com/index.html
+http://localhost/page_load_test/pages/www.google.com.ar/www.google.com.ar/index.html
+http://localhost/page_load_test/pages/www.adultfriendfinder.com/www.adultfriendfinder.com/index.html
+http://localhost/page_load_test/pages/www.apple.com/www.apple.com/index.html
+http://localhost/page_load_test/pages/www.livedoor.com/www.livedoor.com/index.html
+http://localhost/page_load_test/pages/www.sogou.com/www.sogou.com/index.html
+http://localhost/page_load_test/pages/www.google.cn/www.google.cn/index.html
+http://localhost/page_load_test/pages/www.newsgroup.la/www.newsgroup.la/index.html
+http://localhost/page_load_test/pages/www.chinaren.com/www.chinaren.com/index.html
+http://localhost/page_load_test/pages/www.sourceforge.net/sourceforge.net/index.php.html
+http://localhost/page_load_test/pages/www.digg.com/www.digg.com/index.html
+http://localhost/page_load_test/pages/www.126.com/www.126.com/index.html
+http://localhost/page_load_test/pages/www.daum.net/www.daum.net/index.html
+http://localhost/page_load_test/pages/www.xinhuanet.com/www.xinhuanet.com/index.html
+http://localhost/page_load_test/pages/www.about.com/www.about.com/index.html
+http://localhost/page_load_test/pages/www.nate.com/www.nate.com/index.html
+http://localhost/page_load_test/pages/www.rambler.ru/www.rambler.ru/index.html
+http://localhost/page_load_test/pages/www.google.it/www.google.it/index.html
+http://localhost/page_load_test/pages/www.comcast.net/www.comcast.net/index.html
+http://localhost/page_load_test/pages/www.badongo.com/www.badongo.com/index.html
+http://localhost/page_load_test/pages/www.free.fr/www.free.fr/index.html
+http://localhost/page_load_test/pages/www.3721.com/www.3721.com/wlsm/index.htm.html
+http://localhost/page_load_test/pages/www.ebay.com.cn/www.ebay.com.cn/index.html
+http://localhost/page_load_test/pages/www.hinet.net/www.hinet.net/index.html
+http://localhost/page_load_test/pages/www.statcounter.com/www.statcounter.com/index.html
+http://localhost/page_load_test/pages/www.adobe.com/www.adobe.com/index.html
+http://localhost/page_load_test/pages/www.google.com.au/www.google.com.au/index.html
+http://localhost/page_load_test/pages/www.mop.com/www.mop.com/index.html
+http://localhost/page_load_test/pages/www.ig.com.br/www.ig.com.br/index.html
+http://localhost/page_load_test/pages/www.starware.com/www.starware.com/2.0.0.0/index.html
+http://localhost/page_load_test/pages/www.google.co.il/www.google.co.il/index.html
+#//"http://localhost/page_load_test/pages/www.hkjc.com/www.hkjc.com/index.html
+http://localhost/page_load_test/pages/www.china.com/www.china.com/index.html
+http://localhost/page_load_test/pages/www.dell.com/www.dell.com/index.html
+http://localhost/page_load_test/pages/www.51.com/www.51.com/index.html
+http://localhost/page_load_test/pages/www.digitalpoint.com/www.digitalpoint.com/index.html
+http://localhost/page_load_test/pages/www.flurl.com/www.flurl.com/index.html
+http://localhost/page_load_test/pages/www.goo.ne.jp/www.goo.ne.jp/index.html
+http://localhost/page_load_test/pages/www.atnext.com/www.atnext.com/index.html
+http://localhost/page_load_test/pages/www.download.com/www.download.com/index.html
+http://localhost/page_load_test/pages/www.cnnic.cn/www.cnnic.cn/index.html
+http://localhost/page_load_test/pages/www.cmfu.com/www.cmfu.com/index.html
+http://localhost/page_load_test/pages/www.mediaplex.com/www.mediaplex.com/index.html
+http://localhost/page_load_test/pages/www.googlesyndication.com/www.googlesyndication.com/index.html
+http://localhost/page_load_test/pages/www.mapquest.com/www.mapquest.com/index.html
+http://localhost/page_load_test/pages/www.globo.com/www.globo.com/index.html
+http://localhost/page_load_test/pages/www.weather.com/www.weather.com/index.html
+http://localhost/page_load_test/pages/www.imagevenue.com/www.imagevenue.com/index.html
+http://localhost/page_load_test/pages/www.overture.com/www.content.overture.com/d/index.html
+http://localhost/page_load_test/pages/www.theplanet.com/www.theplanet.com/index.html
+http://localhost/page_load_test/pages/www.icio.us/
+http://localhost/page_load_test/pages/www.pconline.com.cn/www.pconline.com.cn/index.html
+http://localhost/page_load_test/pages/www.mywebsearch.com/search.mywebsearch.com/mywebsearch/default.jhtml.html
+http://localhost/page_load_test/pages/www.sendspace.com/www.sendspace.com/index.html
+http://localhost/page_load_test/pages/www.typepad.com/www.typepad.com/index.html
+http://localhost/page_load_test/pages/www.amazon.co.jp/www.amazon.co.jp/index.html
+http://localhost/page_load_test/pages/www.infoseek.co.jp/www.infoseek.co.jp/index.html
+http://localhost/page_load_test/pages/www.21cn.com/www.21cn.com/index.html
+http://localhost/page_load_test/pages/www.gamer.com.tw/www.gamer.com.tw/index.html
+http://localhost/page_load_test/pages/www.tianya.cn/www.tianya.cn/index.html
+http://localhost/page_load_test/pages/www.google.com.eg/www.google.com.eg/index.html
+http://localhost/page_load_test/pages/www.deviantart.com/www.deviantart.com/index.html
+http://localhost/page_load_test/pages/www.metacafe.com/www.metacafe.com/index.html
+http://localhost/page_load_test/pages/www.265.com/www.265.com/index.html
+http://localhost/page_load_test/pages/www.google.com.tw/www.google.com.tw/index.html
+http://localhost/page_load_test/pages/www.match.com/www.match.com/index.html
+http://localhost/page_load_test/pages/www.php.net/www.php.net/index.html
+http://localhost/page_load_test/pages/www.spiegel.de/www.spiegel.de/index.html
+http://localhost/page_load_test/pages/www.neopets.com/www.neopets.com/index.html
+http://localhost/page_load_test/pages/www.phoenixtv.com/www.phoenixtv.com/index.html
+http://localhost/page_load_test/pages/www.hp.com/www.hp.com/index.html
+http://localhost/page_load_test/pages/www.google.nl/www.google.nl/index.html
+http://localhost/page_load_test/pages/www.nifty.com/www.nifty.com/index.html
+http://localhost/page_load_test/pages/www.sina.com.hk/www.sina.com.hk/index.html
+http://localhost/page_load_test/pages/www.sexyono.com/www.sexyono.com/index.html
+http://localhost/page_load_test/pages/www.paipai.com/www.paipai.com/index.html
+http://localhost/page_load_test/pages/www.amazon.co.uk/www.amazon.co.uk/index.html
+http://localhost/page_load_test/pages/www.chinahr.com/www.chinahr.com/index.html
+http://localhost/page_load_test/pages/www.tripod.com/www.tripod.lycos.com/index.html
+http://localhost/page_load_test/pages/www.google.co.ve/www.google.co.ve/index.html
+http://localhost/page_load_test/pages/www.yam.com/www.yam.com/index.html
+http://localhost/page_load_test/pages/www.google.com.pe/www.google.com.pe/index.html
+http://localhost/page_load_test/pages/www.reference.com/www.reference.com/index.html
+http://localhost/page_load_test/pages/www.maktoob.com/www.maktoob.com/index.html
+http://localhost/page_load_test/pages/www.wordpress.com/wordpress.com/index.html
+http://localhost/page_load_test/pages/www.gmx.net/www.gmx.net/index.html
+http://localhost/page_load_test/pages/www.google.com.co/www.google.com.co/index.html
+http://localhost/page_load_test/pages/www.sexuploader.com/www.megarotic.com/index.html
+http://localhost/page_load_test/pages/www.webs-tv.net/www.webs-tv.net/index.html
+http://localhost/page_load_test/pages/www.narod.ru/narod.yandex.ru/index.html
+http://localhost/page_load_test/pages/www.aebn.net/www.aebn.net/index.html
+http://localhost/page_load_test/pages/www.mozilla.com/www.mozilla.com/en-US/index.html
+http://localhost/page_load_test/pages/www.invisionfree.com/www.invisionfree.com/index.html
+http://localhost/page_load_test/pages/www.gamespot.com/www.gamespot.com/index.html
+http://localhost/page_load_test/pages/www.nastydollars.com/www.nastydollars.com/index.html
+http://localhost/page_load_test/pages/www.wefong.com/www.wefong.com/index.html
+http://localhost/page_load_test/pages/www.homeway.com.cn/www.hexun.com/index.html
+http://localhost/page_load_test/pages/www.rapidshare.com/www.rapidshare.com/index.html
+http://localhost/page_load_test/pages/www.vnexpress.net/www.vnexpress.net/Vietnam/Home/index.html
+http://localhost/page_load_test/pages/www.ask.com/www.ask.com/index.html
+http://localhost/page_load_test/pages/www.uusee.com/www.uusee.com/index.html
+http://localhost/page_load_test/pages/www.linkedin.com/www.linkedin.com/index.html
+http://localhost/page_load_test/pages/www.yesky.com/www.yesky.com/index.html
+http://localhost/page_load_test/pages/www.information.com/www.information.com/index.html
+http://localhost/page_load_test/pages/www.msn.com.br/br.msn.com/index.html
+http://localhost/page_load_test/pages/www.51job.com/www.51job.com/index.html
+http://localhost/page_load_test/pages/www.people.com.cn/www.people.com.cn/index.html
+http://localhost/page_load_test/pages/www.verycd.com/www.verycd.com/index.html
+http://localhost/page_load_test/pages/www.sportsline.com/www.sportsline.com/index.html
+http://localhost/page_load_test/pages/www.youthwant.com.tw/www.youthwant.com.tw/index.html
+http://localhost/page_load_test/pages/www.skyblog.com/www.skyblog.com/index.html
+http://localhost/page_load_test/pages/www.technorati.com/www.technorati.com/index.html
+http://localhost/page_load_test/pages/www.google.co.th/www.google.co.th/index.html
+http://localhost/page_load_test/pages/www.google.com.vn/www.google.com.vn/index.html
+http://localhost/page_load_test/pages/www.hatena.ne.jp/www.hatena.ne.jp/index.html
+http://localhost/page_load_test/pages/www.yourfilehost.com/www.yourfilehost.com/index.html
+http://localhost/page_load_test/pages/www.amazon.de/www.amazon.de/index.html
+http://localhost/page_load_test/pages/www.chinamobile.com/www.chinamobile.com/index.html
+http://localhost/page_load_test/pages/www.centrum.cz/www.centrum.cz/index.html
+http://localhost/page_load_test/pages/www.flogao.com.br/www.flogao.com/index.html
+http://localhost/page_load_test/pages/www.myway.com/www.myway.com/index.html
+http://localhost/page_load_test/pages/www.xuite.net/www.xuite.net/index.html
+http://localhost/page_load_test/pages/www.msn.com.cn/cn.msn.com/index.html
+http://localhost/page_load_test/pages/www.kooora.com/www.kooora.com/index.html
+http://localhost/page_load_test/pages/www.godaddy.com/www.godaddy.com/gdshop/default.asp.html
+http://localhost/page_load_test/pages/www.google.com.sg/www.google.com.sg/index.html
+http://localhost/page_load_test/pages/www.4399.com/www.4399.com/index.html
+http://localhost/page_load_test/pages/www.earthlink.net/www.earthlink.net/index.html
+http://localhost/page_load_test/pages/www.torrentspy.com/www.torrentspy.com/index.html
+http://localhost/page_load_test/pages/www.pornotube.com/www.pornotube.com/index.html
+http://localhost/page_load_test/pages/www.aweber.com/www.aweber.com/index.html
+http://localhost/page_load_test/pages/www.slashdot.org/slashdot.org/index.html
+http://localhost/page_load_test/pages/www.2ch.net/www.2ch.net/index.html
+http://localhost/page_load_test/pages/www.ev1servers.net/www.ev1servers.net/index.html
+http://localhost/page_load_test/pages/www.webshots.com/www.webshots.com/index.html
+http://localhost/page_load_test/pages/www.webmasterworld.com/www.webmasterworld.com/index.html
+http://localhost/page_load_test/pages/www.google.se/www.google.se/index.html
+http://localhost/page_load_test/pages/www.biglobe.ne.jp/www.biglobe.ne.jp/index.html
+http://localhost/page_load_test/pages/www.domaintools.com/www.domaintools.com/index.html
+http://localhost/page_load_test/pages/www.mininova.org/www.mininova.org/index.html
+http://localhost/page_load_test/pages/www.elmundo.es/www.elmundo.es/index.html
+http://localhost/page_load_test/pages/www.google.ro/www.google.ro/index.html
+http://localhost/page_load_test/pages/www.google.ae/www.google.ae/index.html
+http://localhost/page_load_test/pages/www.clubbox.co.kr/www.clubbox.co.kr/index.html
+http://localhost/page_load_test/pages/www.w3.org/www.w3.org/index.html
+http://localhost/page_load_test/pages/www.qihoo.com/www.qihoo.com/index.html
+http://localhost/page_load_test/pages/www.google.ru/www.google.ru/index.html
+http://localhost/page_load_test/pages/www.miniclip.com/www.miniclip.com/games/en/index.html
+http://localhost/page_load_test/pages/www.milliyet.com.tr/www.milliyet.com.tr/2006/12/05/index.html
+http://localhost/page_load_test/pages/www.google.com.my/www.google.com.my/index.html
+http://localhost/page_load_test/pages/www.bebo.com/www.bebo.com/index.html
+http://localhost/page_load_test/pages/www.dmm.co.jp/www.dmm.co.jp/index.html
+http://localhost/page_load_test/pages/www.orange.fr/www.orange.fr/index.html
+http://localhost/page_load_test/pages/www.bta.net.cn/www.bta.net.cn/index.html
+http://localhost/page_load_test/pages/www.istockphoto.com/www.istockphoto.com/index.php.html
+http://localhost/page_load_test/pages/www.qianlong.com/www.qianlong.com/index.html
+http://localhost/page_load_test/pages/www.ebay.com.au/www.ebay.com.au/index.html
+http://localhost/page_load_test/pages/www.chinaz.com/www.chinaz.com/index.html
+http://localhost/page_load_test/pages/www.adbrite.com/www.adbrite.com/index.html
+http://localhost/page_load_test/pages/www.sitepoint.com/www.sitepoint.com/index.html
+http://localhost/page_load_test/pages/www.zhongsou.com/www.zhongsou.com/index.html
+http://localhost/page_load_test/pages/www.ups.com/www.ups.com/index.html
+http://localhost/page_load_test/pages/www.wwe.com/www.wwe.com/index.html
+http://localhost/page_load_test/pages/www.netflix.com/www.netflix.com/Register.html
+http://localhost/page_load_test/pages/www.target.com/www.target.com/gp/homepage.html
+http://localhost/page_load_test/pages/www.it.com.cn/www.it.com.cn/index.html
+http://localhost/page_load_test/pages/www.washingtonpost.com/www.washingtonpost.com/index.html
+http://localhost/page_load_test/pages/www.usps.com/www.usps.com/index.html
+http://localhost/page_load_test/pages/www.iask.com/www.iask.com/index.html
+http://localhost/page_load_test/pages/www.google.com.hk/www.google.com.hk/index.html
+http://localhost/page_load_test/pages/www.ibm.com/www.ibm.com/us/index.html
+http://localhost/page_load_test/pages/www.google.gr/www.google.gr/index.html
+http://localhost/page_load_test/pages/www.6park.com/www.6park.com/index.html
+http://localhost/page_load_test/pages/www.sex141.com/www.sex141.com/index.html
+http://localhost/page_load_test/pages/www.excite.co.jp/www.excite.co.jp/index.html
+http://localhost/page_load_test/pages/www.sakura.ne.jp/www.sakura.ne.jp/index.html
+http://localhost/page_load_test/pages/www.icq.com/www.icq.com/index.html
+http://localhost/page_load_test/pages/www.bangbros1.com/www.bangbros1.com/index.html
+http://localhost/page_load_test/pages/www.answers.com/www.answers.com/index.html
+http://localhost/page_load_test/pages/www.foxsports.com/msn.foxsports.com/index.html
+http://localhost/page_load_test/pages/www.clickbank.com/www.clickbank.com/index.html
+http://localhost/page_load_test/pages/www.skype.com/www.skype.com/index.html
+http://localhost/page_load_test/pages/www.mofile.com/tv.mofile.com/cn/index/main.do.html
+http://localhost/page_load_test/pages/www.mlb.com/mlb.mlb.com/NASApp/mlb/index.jsp.html
+http://localhost/page_load_test/pages/www.89.com/www.89.com/index.html
+http://localhost/page_load_test/pages/www.nba.com/www.nba.com/index.html
+http://localhost/page_load_test/pages/www.pornaccess.com/www.pornaccess.com/index.html
+http://localhost/page_load_test/pages/www.imagefap.com/www.imagefap.com/index.html
+http://localhost/page_load_test/pages/www.pcpop.com/www.pcpop.com/index.html
+http://localhost/page_load_test/pages/www.hurriyet.com.tr/www.hurriyet.com.tr/anasayfa/index.html
+http://localhost/page_load_test/pages/www.t-online.de/www.t-online.de/index.html
+http://localhost/page_load_test/pages/www.google.pt/www.google.pt/index.html
+http://localhost/page_load_test/pages/www.no-ip.com/www.no-ip.com/index.html
+http://localhost/page_load_test/pages/www.ocn.ne.jp/www.ocn.ne.jp/index.html
+http://localhost/page_load_test/pages/www.it168.com/www.it168.com/index.html
+http://localhost/page_load_test/pages/www.seesaa.net/tag.seesaa.jp/index.html
+http://localhost/page_load_test/pages/www.nih.gov/www.nih.gov/index.html
+http://localhost/page_load_test/pages/www.raaga.com/www.raaga.com/index.html
+http://localhost/page_load_test/pages/www.lide.cz/www.lide.cz/index.html
+http://localhost/page_load_test/pages/www.indiatimes.com/in.indiatimes.com/usdefault.cms.html
+http://localhost/page_load_test/pages/www.doubleclick.com/www.doubleclick.com/us/index.html
+http://localhost/page_load_test/pages/www.reuters.com/today.reuters.com/news/home.aspx.html
+http://localhost/page_load_test/pages/www.dantri.com.vn/www19.dantri.com.vn/news/index.html
+http://localhost/page_load_test/pages/www.fotka.pl/www.fotka.pl/index.html
+http://localhost/page_load_test/pages/www.miarroba.com/miarroba.com/index.html
+http://localhost/page_load_test/pages/www.readnovel.com/www.readnovel.com/index.html
+http://localhost/page_load_test/pages/www.hawaaworld.com/www.hawaaworld.com/index.html
+http://localhost/page_load_test/pages/www.expedia.com/www.expedia.com/Default.asp@CCheck=1&.html
+http://localhost/page_load_test/pages/www.msn.co.uk/uk.msn.com/index.html
+http://localhost/page_load_test/pages/www.104.com.tw/www.104.com.tw/index.html
+http://localhost/page_load_test/pages/www.eastmoney.com/www.eastmoney.com/index.html
+http://localhost/page_load_test/pages/www.fares.net/www.fares.net/index.html
+http://localhost/page_load_test/pages/www.zhaopin.com/www.zhaopin.com/index.html
+http://localhost/page_load_test/pages/www.clarin.com/www.clarin.com/index.html
+http://localhost/page_load_test/pages/www.dreamwiz.com/www.dreamwiz.com/index.html
+http://localhost/page_load_test/pages/www.ameblo.jp/www.ameba.jp/index.html
+http://localhost/page_load_test/pages/www.dailymotion.com/www.dailymotion.com/index.html
+http://localhost/page_load_test/pages/www.ikea.com/www.ikea.com/index.html
+http://localhost/page_load_test/pages/www.constantcontact.com/www.constantcontact.com/index.jsp.html
+http://localhost/page_load_test/pages/www.mainichi-msn.co.jp/www.mainichi-msn.co.jp/index.html
+http://localhost/page_load_test/pages/www.xnxx.com/www.xnxx.com/index.html
+http://localhost/page_load_test/pages/www.dnsstuff.com/www.dnsstuff.com/index.html
+http://localhost/page_load_test/pages/www.tigerdirect.com/www.tigerdirect.com/index.html
+http://localhost/page_load_test/pages/www.nikkei.co.jp/www.nikkei.co.jp/index.html
+http://localhost/page_load_test/pages/www.liveinternet.ru/www.liveinternet.ru/index.html
+http://localhost/page_load_test/pages/www.forbes.com/www.forbes.com/index.html
+http://localhost/page_load_test/pages/www.linksynergy.com/
+http://localhost/page_load_test/pages/www.yousendit.com/www.yousendit.com/index.html
+http://localhost/page_load_test/pages/www.6rb.com/www.6rb.com/index.html
+http://localhost/page_load_test/pages/www.nfl.com/www.nfl.com/index.html
+http://localhost/page_load_test/pages/www.bestbuy.com/www.bestbuy.com/index.html
+http://localhost/page_load_test/pages/www.iwiw.hu/www.iwiw.hu/pages/user/login.jsp.html
+http://localhost/page_load_test/pages/www.aim.com/www.aim.com/index.html
+http://localhost/page_load_test/pages/www.zaobao.com/www.zaobao.com/index.html
+http://localhost/page_load_test/pages/www.gamefaqs.com/www.gamefaqs.com/index.html
+http://localhost/page_load_test/pages/www.whenu.com/www.whenu.com/index.html
+http://localhost/page_load_test/pages/www.pogo.com/www.pogo.com/home/home.do@sls=2&site=pogo.html
+http://localhost/page_load_test/pages/www.online.sh.cn/www.online.sh.cn/index.html
+http://localhost/page_load_test/pages/www.sanook.com/www.sanook.com/index.html
+http://localhost/page_load_test/pages/www.blog.cz/blog.cz/index.html
+http://localhost/page_load_test/pages/www.feedburner.com/www.feedburner.com/fb/a/home.html
+http://localhost/page_load_test/pages/www.msn.ca/sympatico.msn.ca/index.html
+http://localhost/page_load_test/pages/www.libero.it/www.libero.it/index.html
+http://localhost/page_load_test/pages/72.14.235.104/72.14.235.104/index.html
+http://localhost/page_load_test/pages/www.excite.com/www.excite.com/index.html
+http://localhost/page_load_test/pages/www.leo.org/www.leo.org/index.html
+http://localhost/page_load_test/pages/www.ebay.fr/www.ebay.fr/index.html
+http://localhost/page_load_test/pages/www.ctrip.com/www.ctrip.com/index.html
+http://localhost/page_load_test/pages/www.last.fm/www.last.fm/index.html
+http://localhost/page_load_test/pages/www.gamebase.com.tw/www.gamebase.com.tw/index.html
+http://localhost/page_load_test/pages/www.ebay.ca/www.ebay.ca/index.html
+http://localhost/page_load_test/pages/www.yimg.com/
+http://localhost/page_load_test/pages/www.vietnamnet.vn/www.vietnamnet.vn/index.html
+http://localhost/page_load_test/pages/www.uploading.com/www.uploading.com/index.html
+http://localhost/page_load_test/pages/www.sapo.pt/www.sapo.pt/index.html
+http://localhost/page_load_test/pages/www.usatoday.com/www.usatoday.com/index.html
+http://localhost/page_load_test/pages/www.pplive.com/www.pplive.com/zh-cn/index.html
+http://localhost/page_load_test/pages/www.multiply.com/multiply.com/index.html
+http://localhost/page_load_test/pages/www.jobsdb.com/www.jobsdb.com/default.htm@58518.html
+http://localhost/page_load_test/pages/www.4399.net/www.4399.net/index.html
+http://localhost/page_load_test/pages/www.ynet.com/www.ynet.com/index.html
+http://localhost/page_load_test/pages/www.google.ch/www.google.ch/index.html
+http://localhost/page_load_test/pages/www.mac.com/www.apple.com/dotmac/index.html
+http://localhost/page_load_test/pages/www.joomla.org/www.joomla.org/index.html
+http://localhost/page_load_test/pages/www.dyndns.org/www.dyndns.com/index.html
+http://localhost/page_load_test/pages/www.voyeurweb.com/www.voyeurweb.com/index.html
+http://localhost/page_load_test/pages/www.wuhan.net.cn/www.wuhan.net.cn/index.html
+http://localhost/page_load_test/pages/www.piczo.com/www.piczo.com/index.html@cr=4&rfm=y.html
+http://localhost/page_load_test/pages/www.google.be/www.google.be/index.html
+http://localhost/page_load_test/pages/www.panet.co.il/www.panet.co.il/index.html
+http://localhost/page_load_test/pages/www.google.co.ma/www.google.co.ma/index.html
+http://localhost/page_load_test/pages/72.14.221.104/72.14.221.104/index.html
+http://localhost/page_load_test/pages/www.msn.com.tw/tw.msn.com/index.html
+http://localhost/page_load_test/pages/www.wangyou.com/www.wangyou.com/index.html
+http://localhost/page_load_test/pages/www.6arab.com/www.6arab.com/index.html
+http://localhost/page_load_test/pages/www.wordpress.org/wordpress.org/index.html
+http://localhost/page_load_test/pages/www.onlinedown.net/www.onlinedown.net/index.html
+http://localhost/page_load_test/pages/www.drudgereport.com/www.drudgereport.com/index.html
+http://localhost/page_load_test/pages/www.joyo.com/www.joyo.com/index.html
+http://localhost/page_load_test/pages/www.skycn.com/www.skycn.com/index.html
+http://localhost/page_load_test/pages/www.zedo.com/www.zedo.com/index.html
+http://localhost/page_load_test/pages/www.i-part.com.cn/www.i-part.com.cn/index.html
+http://localhost/page_load_test/pages/www.w3schools.com/www.w3schools.com/index.html
+http://localhost/page_load_test/pages/www.payserve.com/www.payserve.com/index.html
+http://localhost/page_load_test/pages/www.macromedia.com/www.adobe.com/index.html
+http://localhost/page_load_test/pages/www.usercash.com/www.usercash.com/index.html
+http://localhost/page_load_test/pages/www.51.la/www.51.la/index.html
+http://localhost/page_load_test/pages/www.chinacars.com/www.chinacars.com/index.html
+http://localhost/page_load_test/pages/www.cj.com/www.cj.com/index.html
+http://localhost/page_load_test/pages/www.isohunt.com/www.isohunt.com/index.html
+http://localhost/page_load_test/pages/www.engadget.com/www.engadget.com/index.html
+http://localhost/page_load_test/pages/www.nikkansports.com/www.nikkansports.com/index.html
+http://localhost/page_load_test/pages/www.fedex.com/www.fedex.com/index.html
+http://localhost/page_load_test/pages/www.mobile.de/www.mobile.de/index.html
+http://localhost/page_load_test/pages/www.cams.com/www.cams.com/index.html
+http://localhost/page_load_test/pages/www.kinghost.com/www.kinghost.com/index.html
+http://localhost/page_load_test/pages/www.made-in-china.com/www.made-in-china.com/index.html
+http://localhost/page_load_test/pages/www.24h.com.vn/www14.24h.com.vn/index.php.html
+http://localhost/page_load_test/pages/www.sxc.hu/www.sxc.hu/index.html
+http://localhost/page_load_test/pages/www.tv.com/www.tv.com/index.html
+http://localhost/page_load_test/pages/www.nextag.com/www.nextag.com/index.html
+http://localhost/page_load_test/pages/www.jrj.com.cn/www.jrj.com.cn/index.html
+http://localhost/page_load_test/pages/www.msn.es/es.msn.com/index.html
+http://localhost/page_load_test/pages/www.terra.com.ar/www.terra.com.ar/index.html
+http://localhost/page_load_test/pages/www.wikimedia.org/www.wikimedia.org/index.html
+http://localhost/page_load_test/pages/www.google.lt/www.google.lt/index.html
+http://localhost/page_load_test/pages/www.aftonbladet.se/www.aftonbladet.se/index.html
+http://localhost/page_load_test/pages/72.14.209.104/72.14.209.104/index.html
+http://localhost/page_load_test/pages/www.xrea.com/www.xrea.com/index.html
+http://localhost/page_load_test/pages/www.mysql.com/www.mysql.com/index.html
+http://localhost/page_load_test/pages/www.overstock.com/www.overstock.com/index.html
+http://localhost/page_load_test/pages/www.sitemeter.com/www.sitemeter.com/index.html
+http://localhost/page_load_test/pages/www.yok.com/www.yok.com/index.html
+http://localhost/page_load_test/pages/www.met-art.com/www.met-art.com/index.html
+http://localhost/page_load_test/pages/www.sun.com/www.sun.com/index.html
+http://localhost/page_load_test/pages/www.tripadvisor.com/www.tripadvisor.com/index.html
+http://localhost/page_load_test/pages/www.gc.ca/www.gc.ca/index.html
+http://localhost/page_load_test/pages/www.realtor.com/www.realtor.com/Default.asp@poe=realtor.html
+http://localhost/page_load_test/pages/www.spoluzaci.cz/www.spoluzaci.cz/index.html
+http://localhost/page_load_test/pages/www.netscape.com/www.netscape.com/index.html
+http://localhost/page_load_test/pages/www.asahi.com/www.asahi.com/index.html
+http://localhost/page_load_test/pages/www.fanfiction.net/www.fanfiction.net/index.html
+http://localhost/page_load_test/pages/www.msn.com.hk/www.msn.com.hk/Default.asp.html
+http://localhost/page_load_test/pages/www.travelocity.com/www.travelocity.com/index.html
+http://localhost/page_load_test/pages/www.ninemsn.com.au/ninemsn.com.au/index.html
+http://localhost/page_load_test/pages/www.stumbleupon.com/www.stumbleupon.com/index.html
+http://localhost/page_load_test/pages/www.cafepress.com/www.cafepress.com/index.html
+http://localhost/page_load_test/pages/www.livejasmin.com/www.2.livejasmin.com/index.php.html
+http://localhost/page_load_test/pages/www.ezinearticles.com/www.ezinearticles.com/index.html
+http://localhost/page_load_test/pages/www.pricegrabber.com/www.pricegrabber.com/index.html
+http://localhost/page_load_test/pages/www.sina.com/www.sina.com/index.html
+http://localhost/page_load_test/pages/www.lycos.com/www.lycos.com/index.html
+http://localhost/page_load_test/pages/www.apache.org/www.apache.org/index.html
+http://localhost/page_load_test/pages/www.ringo.com/www.ringo.com/index.html
+http://localhost/page_load_test/pages/www.videosz.com/www.videosz.com/index.html
+http://localhost/page_load_test/pages/www.fotop.net/www.fotop.net/index.html
+http://localhost/page_load_test/pages/www.fatwallet.com/www.fatwallet.com/index.html

testing/performance/talos/run_tests.py

@@ -61,8 +61,7 @@ socket.setdefaulttimeout(480)
 import utils
 import config
 import post_file
-import tp
-import ts
+import ttest
 
 def shortNames(name):
   if name == "tp_loadtime":
@@ -84,137 +83,128 @@ def process_Request(post):
         str += line.split(":")[3] + ":" + shortNames(line.split(":")[1]) + ":" + line.split(":")[2] + '\n'
   return str
 
-def test_file(filename):
-  """Runs the Ts and Tp tests on the given config file and generates a report.
-  
-  Args:
-    filename: the name of the file to run the tests on
-  """
-  
-  test_configs = []
-  test_names = []
-  title = ''
-  filename_prefix = ''
-  testdate = ''
-  
-  # Read in the profile info from the YAML config file
-  config_file = open(filename, 'r')
-  yaml_config = yaml.load(config_file)
-  config_file.close()
-  for item in yaml_config:
-    if item == 'title':
-      title = yaml_config[item]
-    elif item == 'filename':
-      filename_prefix = yaml_config[item]
-    elif item == 'testdate':
-      testdate = yaml_config[item]
-    else:
-      new_config = [yaml_config[item]['preferences'],
-                    yaml_config[item]['extensions'],
-                    yaml_config[item]['firefox'],
-                    yaml_config[item]['branch'],
-                    yaml_config[item]['branchid'],
-                    yaml_config[item]['profile_path'],
-                    yaml_config[item]['env']]
-      test_configs.append(new_config)
-      test_names.append(item)
-  config_file.close()
-
-  print test_configs
-  sys.stdout.flush()
-  if (testdate != ''):
-    date = int(time.mktime(time.strptime(testdate, '%a, %d %b %Y %H:%M:%S GMT')))
-  else:
-    date = int(time.time()) #TODO get this into own file
-  print "using testdate: %d" % date
-  print "actual date: %d" % int(time.time())
- 
-
-  # Run startup time test
-  ts_times = ts.RunStartupTests(test_configs,
-                                config.TS_NUM_RUNS)
-
-  print "finished ts"
-  sys.stdout.flush()
-  for ts_set in ts_times:
-    if len(ts_set) == 0:
-	print "FAIL:no ts results, build failed to run:BAD BUILD"
-	sys.exit(0)
-
-  (res, r_strings, tp_times, tp_counters) = tp.RunPltTests(test_configs,
-                                           config.TP_NUM_CYCLES,
-                                           config.COUNTERS,
-                                           config.TP_RESOLUTION)
-
-  print "finished tp"
-  sys.stdout.flush()
-
-  if not res:
-    print "FAIL:tp did not run to completion"
-    print "FAIL:" + r_strings[0]
-    sys.exit(0)
-
-  #TODO: place this in its own file
-  #send results to the graph server
-  # each line of the string is of the format i;page_name;median;mean;min;max;time vals\n
-  tbox = title
-  url_format = "http://%s/%s"
-  link_format= "<a href = \"%s\">%s</a>"
-  #value, testname, tbox, timeval, date, branch, branchid, type, data
-  result_format = "%.2f,%s,%s,%d,%d,%s,%s,%s,%s,\n"
-  result_format2 = "%.2f,%s,%s,%d,%d,%s,%s,%s,\n"
-  filename = tempfile.mktemp()
-  tmpf = open(filename, "w")
-
-  testname = "ts"
-  print "formating results for: ts"
-  print "# of values: %d" % len(ts_times)
-  for index in range(len(ts_times)):
-    i = 0
-    for tstime in ts_times[index]:
-      tmpf.write(result_format % (float(tstime), testname, tbox, i, date, test_configs[index][3], test_configs[index][4], "discrete", "ms"))
-      i = i+1
-
-  testname = "tp"
-  for index in range(len(r_strings)):
-    r_strings[index].strip('\n')
-    page_results = r_strings[index].splitlines()
-    i = 0
-    print "formating results for: loadtime"
-    print "# of values: %d" % len(page_results)
-    for mypage in page_results[3:]:
-      r = mypage.split(';')
-      tmpf.write(result_format % (float(r[2]), testname + "_loadtime", tbox, i, date, test_configs[index][3], test_configs[index][4], "discrete", r[1]))
-      i = i+1
-
-  for index in range(len(tp_counters)):
-    for count_type in config.COUNTERS:
+def send_to_csv(results):
+  import csv
+  for res in results:
+    browser_dump, counter_dump = results[res]
+    writer = csv.writer(open(config.CSV_FILE + '_' +  res, "wb"))
+    if res == 'ts':
       i = 0
-      print "formating results for: " + count_type
-      print "# of values: %d" % len(tp_counters[index][count_type])
-      for value in tp_counters[index][count_type]:
-        tmpf.write(result_format2 % (float(value), testname + "_" + count_type.replace("%", "Percent"), tbox, i, date, test_configs[index][3], test_configs[index][4], "discrete"))
-        i = i+1
+      writer.writerow(['i', 'val'])
+      for val in browser_dump:
+        writer.writerow([i, val])
+        i += 1
+    if (res.find('tp') > -1) or (res == 'tdhmtl'):
+      writer.writerow(['i', 'page', 'median', 'mean', 'min' , 'max', 'runs'])
+      for bd in browser_dump:
+        bd.rstrip('\n')
+        page_results = bd.splitlines()
+        i = 0
+        for mypage in page_results[2:]:
+          r = mypage.split(';')
+          if r[1].find('/') > -1 :
+             page = r[1].split('/')[1]
+          else:
+             page = r[1]
+          writer.writerow([i, page, r[2], r[3], r[4], r[5], '|'.join(r[6:])])
+          i += 1
+    for cd in counter_dump:
+      for count_type in cd:
+        writer = csv.writer(open(config.CSV_FILE + '_' + res + '_' + count_type, "wb"))
+        writer.writerow(['i', 'value'])
+        i = 0
+        for val in cd[count_type]:
+          writer.writerow([i, val])
+          i += 1
 
-
-  print "finished formating results"
-  tmpf.flush()
-  tmpf.close()
+def post_chunk(id, filename):
   tmpf = open(filename, "r")
   file_data = tmpf.read()
   while True:
     try:
-      ret = post_file.post_multipart(config.RESULTS_SERVER, config.RESULTS_LINK, [("key", "value")], [("filename", filename, file_data)
-   ])
+      ret = post_file.post_multipart(config.RESULTS_SERVER, config.RESULTS_LINK, [("key", "value")], [("filename", filename, file_data)])
     except IOError:
-      print "IOError"
+      print "FAIL: IOError on sending data to the graph server"
     else:
       break
-  print "completed sending results"
   links = process_Request(ret)
-  tmpf.close()
-  os.remove(filename)
+  utils.debug(id + ": sent results")
+  return links
 
+def chunk_list(val_list):
+  """ 
+    divide up a list into manageable chunks
+    currently set at length 500 
+    this is for a failure on mac os x with python 2.4.4 
+  """
+  chunks = []
+  end = 500
+  while (val_list != []):
+    chunks.append(val_list[0:end])
+    val_list = val_list[end:len(val_list)]
+  return chunks
+
+def send_to_graph(title, date, browser_config, results):
+  tbox = title
+  url_format = "http://%s/%s"
+  link_format= "<a href = \"%s\">%s</a>"
+  #value, testname, tbox, timeval, date, branch, buildid, type, data
+  result_format = "%.2f,%s,%s,%d,%d,%s,%s,%s,%s,\n"
+  result_format2 = "%.2f,%s,%s,%d,%d,%s,%s,%s,\n"
+  links = ''
+
+  for res in results:
+    filename = tempfile.mktemp()
+    tmpf = open(filename, "w")
+    browser_dump, counter_dump = results[res]
+    filename = tempfile.mktemp()
+    tmpf = open(filename, "w")
+    if res == 'ts':
+       i = 0
+       for val in browser_dump:
+         tmpf.write(result_format % (float(val), res, tbox, i, date, browser_config['branch'], browser_config['buildid'], "discrete", "ms"))
+         i += 1
+    if (res.find('tp') > -1)  or (res == 'tdhtml'):
+       # each line of the string is of the format i;page_name;median;mean;min;max;time vals\n
+      for bd in browser_dump:
+        bd.rstrip('\n')
+        page_results = bd.splitlines()
+        i = 0
+        for mypage in page_results[2:]:
+          r = mypage.split(';')
+          if r[1].find('/') > -1 :
+             page = r[1].split('/')[1]
+          else:
+             page = r[1]
+          try:
+            val = float(r[2])
+          except ValueError:
+            print 'WARNING: value error for median in tp'
+            val = 0
+          tmpf.write(result_format % (val, res, tbox, i, date, browser_config['branch'], browser_config['buildid'], "discrete", page))
+          i += 1
+    tmpf.flush()
+    tmpf.close()
+    links += post_chunk(res, filename)
+    os.remove(filename)
+    for cd in counter_dump:
+      for count_type in cd:
+        val_list = cd[count_type]
+        chunks = chunk_list(val_list)
+        chunk_link = ''
+        for chunk in chunks:
+          filename = tempfile.mktemp()
+          tmpf = open(filename, "w")
+          i = 0
+          for val in chunk:
+              tmpf.write(result_format2 % (float(val), res + "_" + count_type.replace("%", "Percent"), tbox, i, date, browser_config['branch'], browser_config['buildid'], "discrete"))
+              i += 1
+          tmpf.flush()
+          tmpf.close()
+          chunk_link = post_chunk('%s_%s (%d values)' % (res, count_type, len(chunk)), filename)
+          os.remove(filename)
+        links += chunk_link
+    
   lines = links.split('\n')
   for line in lines:
     if line == "":
@@ -228,6 +218,61 @@ def test_file(filename):
     url = url_format % (config.RESULTS_SERVER, values[0],)
     link = link_format % (url, linkName,)
     print "RETURN: " + link
+
+def test_file(filename):
+  """Runs the Ts and Tp tests on the given config file and generates a report.
+  
+  Args:
+    filename: the name of the file to run the tests on
+  """
+  
+  browser_config = []
+  tests = []
+  title = ''
+  testdate = ''
+  results = {}
+  
+  # Read in the profile info from the YAML config file
+  config_file = open(filename, 'r')
+  yaml_config = yaml.load(config_file)
+  config_file.close()
+  for item in yaml_config:
+    if item == 'title':
+      title = yaml_config[item]
+    elif item == 'testdate':
+      testdate = yaml_config[item]
+  browser_config = {'preferences'  : yaml_config['preferences'],
+                    'extensions'   : yaml_config['extensions'],
+                    'firefox'      : yaml_config['firefox'],
+                    'branch'       : yaml_config['branch'],
+                    'buildid'      : yaml_config['buildid'],
+                    'profile_path' : yaml_config['profile_path'],
+                    'env'          : yaml_config['env'],
+                    'dirs'         : yaml_config['dirs']}
+  tests = yaml_config['tests']
+  config_file.close()
+  if (testdate != ''):
+    date = int(time.mktime(time.strptime(testdate, '%a, %d %b %Y %H:%M:%S GMT')))
+  else:
+    date = int(time.time()) #TODO get this into own file
+  utils.debug("using testdate: %d" % date)
+  utils.debug("actual date: %d" % int(time.time()))
+
+  for test in tests:
+    print "Running test: " + test
+    res, browser_dump, counter_dump = ttest.runTest(browser_config, tests[test])
+    if not res:
+      print 'FAIL: failure to complete test: ' + test
+      sys.exit(0)
+    results[test] = [browser_dump, counter_dump]
+    print "Completed test: " + test
+
+  #process the results
+  if config.TO_GRAPH_SERVER:
+    #send results to the graph server
+    send_to_graph(title, date, browser_config, results)
+  if config.TO_CSV:
+    send_to_csv(results)
   
 if __name__=='__main__':
 

testing/performance/talos/sample.config

@@ -1,48 +1,84 @@
 # Sample Talos configuration file
 
-# Filename will be appended to the timestamp in the report filename.
-# Use letters and underscores only
-filename: testfilename 
-
 # The title of the report
-title: testtitle
+title: firefox_testing
 
-# Name of profile to test
-Test profile 1:
-  # Path to Firefox to test
-  firefox: C:\cygwin\tmp\test\firefox\firefox.exe
+# Path to Firefox to test
+firefox: C:\mozilla\testing\performance\firefox\firefox.exe
 
-  branch: testbranch
+branch: testbranch
 
-  branchid: testbranchid
+buildid: testbuildid
 
-  profile_path: C:\talos\base_profile
+profile_path: C:\mozilla\testing\performance\talos\base_profile
 
-  # Preferences to set in the test (use "preferences : {}" for no prefs)
-  preferences : 
-    browser.shell.checkDefaultBrowser : false
-    dom.allow_scripts_to_close_windows : true
-    dom.disable_open_during_load: false
-    browser.dom.window.dump.enabled: true
-    network.proxy.type : 1
-    network.proxy.http : localhost
-    network.proxy.http_port : 80
-    dom.disable_window_flip : true
-    dom.disable_window_move_resize : true
-    security.enable_java : false
-    extensions.checkCompatibility : false
-    extensions.update.notifyUser: false
-    capability.principal.codebase.p0.granted : UniversalPreferencesWrite UniversalXPConnect UniversalPreferencesRead
-    capability.principal.codebase.p0.id : file://
-    capability.principal.codebase.p1.granted : UniversalXPConnect
+# Preferences to set in the test (use "preferences : {}" for no prefs)
+preferences : 
+  browser.shell.checkDefaultBrowser : false
+  browser.warnOnQuit : false
+  dom.allow_scripts_to_close_windows : true
+  dom.disable_open_during_load: false
+  dom.max_script_run_time : 0
+  browser.dom.window.dump.enabled: true
+  network.proxy.type : 1
+  network.proxy.http : localhost
+  network.proxy.http_port : 80
+  dom.disable_window_flip : true
+  dom.disable_window_move_resize : true
+  security.enable_java : false
+  extensions.checkCompatibility : false
+  extensions.update.notifyUser: false
+  capability.principal.codebase.p0.granted : UniversalPreferencesWrite UniversalXPConnect UniversalPreferencesRead
+  capability.principal.codebase.p0.id : file://
+  capability.principal.codebase.p1.granted : UniversalXPConnect
 
-  # Extensions to install in test (use "extensions: {}" for none)
-  # Need quotes around guid because of curly braces
-  # extensions : 
-  #     "{12345678-1234-1234-1234-abcd12345678}" : c:\path\to\unzipped\xpi
-  #     foo@sample.com : c:\path\to\other\unzipped\xpi
-  extensions : {}
+# Extensions to install in test (use "extensions: {}" for none)
+# Need quotes around guid because of curly braces
+# extensions : 
+#     "{12345678-1234-1234-1234-abcd12345678}" : c:\path\to\unzipped\xpi
+#     foo@sample.com : c:\path\to\other\unzipped\xpi
+extensions : {}
+
+#any directories whose contents need to be installed in the browser before running the tests
+# this assumes that the directories themselves already exist in the firefox path
+dirs:
+  chrome : page_load_test\chrome
+  components : page_load_test\components
+
+# Environment variables to set during test (use env: {} for none)
+env : 
+  NO_EM_RESTART : 1
+# Tests to run
+#  url       : (REQUIRED) url to load into the given firefox browser
+#  url_mod   : (OPTIONAL) a bit of code to be evaled and added to the given url during each cycle of the test
+#  resolution: (REQUIRED) how long (in seconds) to pause between counter sampling
+#  cycles    : (REQUIRED) how many times to run the test
+#  counters  : (REQUIRED) types of system activity to monitor during test run, can be empty 
+#            For possible values of counters argument on Windows, see
+#            http://technet2.microsoft.com/WindowsServer/en/Library/86b5d116-6fb3-427b-af8c-9077162125fe1033.mspx?mfr=true
+#            Possible values on Linux and Mac:
+#            counters : ['Private Bytes', 'RSS']
+#            Standard windows values:
+#            counters : ['Working Set', 'Private Bytes', '% Processor Time']
+
+# to set up a new test it must have the correct configuration options and drop information in a standard format
+# the format is seen in the regular expressions in ttest.py
+# to see how the data passed from the browser is processed see send_to_graph and send_to_csv in run_tests.py
+tests :
+  ts :
+    url : file:///c:/mozilla/testing/performance/talos/startup_test/startup_test.html?begin=
+    url_mod : str(int(time.time()*1000))
+    resolution : 1 
+    cycles : 20
+    counters : []
+  tp: 
+    url :  '-tp c:\\\mozilla\\\testing\\\performance\\\talos\\\page_load_test\\\manifest.txt -tpchrome -tpformat tinderbox -tpcycles 5'
+    resolution : 1
+    cycles : 1
+    counters : ['Working Set', 'Private Bytes', '% Processor Time']
+  tp_js: 
+    url : '"http://localhost/page_load_test/framecycler.html?quit=1&cycles=5"'
+    resolution : 1
+    cycles : 1
+    counters : ['Working Set', 'Private Bytes', '% Processor Time']
   
-  # Environment variables to set during test (use env: {} for none)
-  env : 
-    NO_EM_RESTART : 1

testing/performance/talos/startup_test/startup_test.html

@@ -46,7 +46,7 @@
     var startupTime = now - begin;
 	document.write('\n\nStartup time = ' + startupTime + ' ms<br>');
     if (window.dump) {
-      dump('__startuptime,' + startupTime + '\n\n');
+      dump('__start_report' + startupTime + '__end_report\n\n');
     }
     window.close();
 ">

testing/performance/talos/ttest.py

@@ -0,0 +1,195 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is standalone Firefox Windows performance test.
+#
+# The Initial Developer of the Original Code is Google Inc.
+# Portions created by the Initial Developer are Copyright (C) 2006
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Annie Sullivan <annie.sullivan@gmail.com> (original author)
+#   Ben Hearsum    <bhearsum@wittydomain.com> (OS independence)
+#   Alice Nodelman <anodelman@mozilla.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+"""A generic means of running an URL based browser test
+   follows the following steps
+     - creates a profile
+     - tests the profile
+     - gets metrics for the current test environment
+     - loads the url
+     - collects info on any counters while test runs
+     - waits for a 'dump' from the browser
+"""
+
+__author__ = 'annie.sullivan@gmail.com (Annie Sullivan)'
+
+
+import platform
+import os
+import re
+import shutil
+import time
+import sys
+import subprocess
+import utils
+
+import ffprocess
+import ffsetup
+
+if platform.system() == "Linux":
+    from cmanager_linux import *
+elif platform.system() == "Windows":
+    from cmanager_win32 import *
+elif platform.system() == "Darwin":
+    from cmanager_mac import *
+
+
+# Regula expression for getting results from most tests
+RESULTS_REGEX = re.compile('__start_report(.*)__end_report',
+                      re.DOTALL | re.MULTILINE)
+# Regular expression to get stats for page load test (Tp) - should go away once data passing is standardized
+RESULTS_TP_REGEX = re.compile('__start_tp_report(.*)__end_tp_report',
+                      re.DOTALL | re.MULTILINE)
+RESULTS_REGEX_FAIL = re.compile('__FAIL(.*)__FAIL', re.DOTALL|re.MULTILINE)
+
+
+def runTest(browser_config, test_config):
+  """
+  Runs an url based test on the browser as specified in the browser_config dictionary
+  
+  Args:
+    browser_config:  Dictionary of configuration options for the browser (paths, prefs, etc)
+    test_config   :  Dictionary of configuration for the given test (url, cycles, counters, etc)
+  
+  """
+ 
+  res = 0
+  counters = test_config['counters']
+  resolution = test_config['resolution']
+  all_browser_results = []
+  all_counter_results = []
+  utils.setEnvironmentVars(browser_config['env'])
+
+  # add any provided directories to the installed firefox
+  for dir in browser_config['dirs']:
+    ffsetup.InstallInBrowser(browser_config['firefox'], browser_config['dirs'][dir])
+  # Create the new profile
+  profile_dir = ffsetup.CreateTempProfileDir(browser_config['profile_path'],
+                                               browser_config['preferences'],
+                                               browser_config['extensions'])
+  utils.debug("created profile") 
+  # Run Firefox once with new profile so initializing it doesn't cause
+  # a performance hit, and the second Firefox that gets created is properly
+  # terminated.
+  res = ffsetup.InitializeNewProfile(browser_config['firefox'], profile_dir)
+  if not res:
+    print "FAIL: couldn't initialize firefox"
+    return (res, all_browser_results, all_counter_results)
+  res = 0
+
+  utils.debug("initialized firefox")
+  sys.stdout.flush()
+  ffprocess.Sleep()
+
+  for i in range(test_config['cycles']):
+    # Run the test 
+    browser_results = ""
+    timeout = 18000 # 5 hours
+    total_time = 0
+    output = ''
+    url = test_config['url']
+    if 'url_mod' in test_config:
+      url += eval(test_config['url_mod']) 
+    command_line = ffprocess.GenerateFirefoxCommandLine(browser_config['firefox'], profile_dir, url)
+    process = subprocess.Popen(command_line, stdout=subprocess.PIPE, universal_newlines=True, shell=True, bufsize=0, env=os.environ)
+    handle = process.stdout
+
+    #give firefox a chance to open
+    # this could mean that we are losing the first couple of data points as the tests starts, but if we don't provide
+    # some time for the browser to start we have trouble connecting the CounterManager to it
+    ffprocess.Sleep()
+    #set up the counters for this test
+    cm = CounterManager("firefox", counters)
+    cm.startMonitor()
+    counter_results = {}
+    for counter in counters:
+      counter_results[counter] = []
+    
+    while total_time < timeout:
+      # Sleep for [resolution] seconds
+      time.sleep(resolution)
+      total_time += resolution
+      
+      # Get the output from all the possible counters
+      for count_type in counters:
+        val = cm.getCounterValue(count_type)
+
+        if (val):
+          counter_results[count_type].append(val)
+
+      # Check to see if page load times were outputted
+      (bytes, current_output) = ffprocess.NonBlockingReadProcessOutput(handle)
+      output += current_output
+      match = RESULTS_REGEX.search(output)
+      if match:
+        browser_results += match.group(1)
+        res = 1
+        break
+      #TODO: this a stop gap until all of the tests start outputting the same format
+      match = RESULTS_TP_REGEX.search(output)
+      if match:
+        browser_results += match.group(1)
+        res = 1
+        break
+      match = RESULTS_REGEX_FAIL.search(output)
+      if match:
+        browser_results += match.group(1)
+        print "FAIL: " + match.group(1)
+        break
+
+    if total_time > timeout:
+      print "FAIL: timeout from test"
+
+    #stop the counter manager since this test is complete
+    cm.stopMonitor()
+
+    #kill any remaining firefox processes
+    ffprocess.TerminateAllProcesses("firefox")
+    all_browser_results.append(browser_results)
+    all_counter_results.append(counter_results)
+    
+  # Delete the temp profile directory  Make it writeable first,
+  # because every once in a while Firefox seems to drop a read-only
+  # file into it.
+  ffprocess.Sleep()
+  ffsetup.MakeDirectoryContentsWritable(profile_dir)
+  shutil.rmtree(profile_dir)
+    
+  utils.restoreEnvironmentVars()
+    
+  return (res, all_browser_results, all_counter_results)