Bug 396232 - one sample.config for all platforms. a=anodelman, r=bhearsum

testing/performance/talos/config.py

@@ -1,55 +0,0 @@
-# ***** BEGIN LICENSE BLOCK *****
-# Version: MPL 1.1/GPL 2.0/LGPL 2.1
-#
-# The contents of this file are subject to the Mozilla Public License Version
-# 1.1 (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-# http://www.mozilla.org/MPL/
-#
-# Software distributed under the License is distributed on an "AS IS" basis,
-# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
-# for the specific language governing rights and limitations under the
-# License.
-#
-# The Original Code is standalone Firefox Windows performance test.
-#
-# The Initial Developer of the Original Code is Google Inc.
-# Portions created by the Initial Developer are Copyright (C) 2006
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-#   Annie Sullivan <annie.sullivan@gmail.com> (original author)
-#   Alice Nodelman <anodelman@mozilla.com>
-#
-# Alternatively, the contents of this file may be used under the terms of
-# either the GNU General Public License Version 2 or later (the "GPL"), or
-# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
-# in which case the provisions of the GPL or the LGPL are applicable instead
-# of those above. If you wish to allow use of your version of this file only
-# under the terms of either the GPL or the LGPL, and not to allow others to
-# use your version of this file under the terms of the MPL, indicate your
-# decision by deleting the provisions above and replace them with the notice
-# and other provisions required by the GPL or the LGPL. If you do not delete
-# the provisions above, a recipient may use your version of this file under
-# the terms of any one of the MPL, the GPL or the LGPL.
-#
-# ***** END LICENSE BLOCK *****
-
-"""A list of constants containing the paths to programs and files
-   needed by the performance testing scripts.
-"""
-__author__ = 'annie.sullivan@gmail.com (Annie Sullivan)'
-
-"""The path to the file url to load when initializing/collecting information from the browser"""
-INIT_URL = 'file:///c:/mozilla/testing/performance/talos/getInfo.html'
-
-"""Dump results locally to csv"""
-TO_CSV = 0
-CSV_FILE = r'c:\mozilla\testing\performance\talos\output\out'
-"""URL for the results server"""
-TO_GRAPH_SERVER = 1
-RESULTS_SERVER = 'graphserver.url.here'
-RESULTS_LINK = '/bulk.cgi'
-
-"""Enable/disable debugging output"""
-DEBUG = 0

testing/performance/talos/ffprocess.py

@@ -47,8 +47,6 @@ import re
 import time
 import subprocess
 
-import config
-
 if platform.system() == "Linux":
     from ffprocess_linux import *
 elif platform.system() == "Windows":

testing/performance/talos/ffprocess_linux.py

@@ -40,7 +40,6 @@ import signal
 import os
 from select import select
 
-import config
 
 
 def GenerateFirefoxCommandLine(firefox_path, profile_dir, url):

testing/performance/talos/ffprocess_mac.py

@@ -40,7 +40,6 @@ import subprocess
 import signal
 import os
 import time
-import config
 from select import select
 
 

testing/performance/talos/ffprocess_win32.py

@@ -41,8 +41,6 @@ import win32pdhutil
 import win32pipe
 import msvcrt
 
-import config
-
 
 def GenerateFirefoxCommandLine(firefox_path, profile_dir, url):
   """Generates the command line for a process to run Firefox

testing/performance/talos/ffsetup.py

@@ -53,7 +53,6 @@ import glob
 
 import utils
 import ffprocess
-import config
 
 if platform.system() == "Linux":
     from ffprofile_unix import *
@@ -141,7 +140,7 @@ def InstallInBrowser(firefox_path, dir_path):
       else:
           utils.debug("WARNING: file already installed (" + fromfile + ")")
 
-def InitializeNewProfile(firefox_path, profile_dir):
+def InitializeNewProfile(firefox_path, profile_dir, init_url):
   """Runs Firefox with the new profile directory, to negate any performance
      hit that could occur as a result of starting up with a new profile.  
      Also kills the "extra" Firefox that gets spawned the first time Firefox
@@ -153,7 +152,7 @@ def InitializeNewProfile(firefox_path, profile_dir):
   """
   PROFILE_REGEX = re.compile('__metrics(.*)__metrics', re.DOTALL|re.MULTILINE)
   res = 1
-  cmd = ffprocess.GenerateFirefoxCommandLine(firefox_path, profile_dir, config.INIT_URL)
+  cmd = ffprocess.GenerateFirefoxCommandLine(firefox_path, profile_dir, init_url)
   (match, timed_out) = ffprocess.RunProcessAndWaitForOutput(cmd,
                                                               'firefox',
                                                               PROFILE_REGEX,

testing/performance/talos/run_tests.py

@@ -57,9 +57,9 @@ import os
 import string
 import socket
 socket.setdefaulttimeout(480)
+import getopt
 
 import utils
-import config
 import post_file
 import ttest
 
@@ -83,11 +83,11 @@ def process_Request(post):
         str += line.split(":")[3] + ":" + shortNames(line.split(":")[1]) + ":" + line.split(":")[2] + '\n'
   return str
 
-def send_to_csv(results):
+def send_to_csv(csv_file, results):
   import csv
   for res in results:
     browser_dump, counter_dump = results[res]
-    writer = csv.writer(open(config.CSV_FILE + '_' +  res, "wb"))
+    writer = csv.writer(open(csv_file + '_' +  res, "wb"))
     if res == 'ts':
       i = 0
       writer.writerow(['i', 'val'])
@@ -110,19 +110,19 @@ def send_to_csv(results):
           i += 1
     for cd in counter_dump:
       for count_type in cd:
-        writer = csv.writer(open(config.CSV_FILE + '_' + res + '_' + count_type, "wb"))
+        writer = csv.writer(open(csv_file + '_' + res + '_' + count_type, "wb"))
         writer.writerow(['i', 'value'])
         i = 0
         for val in cd[count_type]:
           writer.writerow([i, val])
           i += 1
 
-def post_chunk(id, filename):
+def post_chunk(results_server, results_link, id, filename):
   tmpf = open(filename, "r")
   file_data = tmpf.read()
   while True:
     try:
-      ret = post_file.post_multipart(config.RESULTS_SERVER, config.RESULTS_LINK, [("key", "value")], [("filename", filename, file_data)])
+      ret = post_file.post_multipart(results_server, results_link, [("key", "value")], [("filename", filename, file_data)])
     except IOError:
       print "FAIL: IOError on sending data to the graph server"
     else:
@@ -144,7 +144,7 @@ def chunk_list(val_list):
     val_list = val_list[end:len(val_list)]
   return chunks
 
-def send_to_graph(title, date, browser_config, results):
+def send_to_graph(results_server, results_link, title, date, browser_config, results):
   tbox = title
   url_format = "http://%s/%s"
   link_format= "<a href = \"%s\">%s</a>"
@@ -185,7 +185,7 @@ def send_to_graph(title, date, browser_config, results):
           i += 1
     tmpf.flush()
     tmpf.close()
-    links += post_chunk(res, filename)
+    links += post_chunk(results_server, results_link, res, filename)
     os.remove(filename)
     for cd in counter_dump:
       for count_type in cd:
@@ -201,7 +201,7 @@ def send_to_graph(title, date, browser_config, results):
               i += 1
           tmpf.flush()
           tmpf.close()
-          chunk_link = post_chunk('%s_%s (%d values)' % (res, count_type, len(chunk)), filename)
+          chunk_link = post_chunk(results_server, results_link, '%s_%s (%d values)' % (res, count_type, len(chunk)), filename)
           os.remove(filename)
         links += chunk_link
     
@@ -215,7 +215,7 @@ def send_to_graph(title, date, browser_config, results):
       linkName += "_T: " + values[2]
     else:
       linkName += "_1"
-    url = url_format % (config.RESULTS_SERVER, values[0],)
+    url = url_format % (results_server, values[0],)
     link = link_format % (url, linkName,)
     print "RETURN: " + link
 
@@ -230,6 +230,9 @@ def test_file(filename):
   tests = []
   title = ''
   testdate = ''
+  csv_file = ''
+  results_server = ''
+  results_link = ''
   results = {}
   
   # Read in the profile info from the YAML config file
@@ -241,6 +244,12 @@ def test_file(filename):
       title = yaml_config[item]
     elif item == 'testdate':
       testdate = yaml_config[item]
+    elif item == 'csv_file':
+       csv_file = os.path.normpath(yaml_config[item])
+    elif item == 'results_server':
+       results_server = yaml_config[item]
+    elif item == 'results_link' :
+       results_link = yaml_config[item]
   browser_config = {'preferences'  : yaml_config['preferences'],
                     'extensions'   : yaml_config['extensions'],
                     'firefox'      : yaml_config['firefox'],
@@ -248,7 +257,13 @@ def test_file(filename):
                     'buildid'      : yaml_config['buildid'],
                     'profile_path' : yaml_config['profile_path'],
                     'env'          : yaml_config['env'],
-                    'dirs'         : yaml_config['dirs']}
+                    'dirs'         : yaml_config['dirs'],
+                    'init_url'     : yaml_config['init_url']}
+  #normalize paths to work accross platforms
+  browser_config['firefox'] = os.path.normpath(browser_config['firefox'])
+  browser_config['profile_path'] = os.path.normpath(browser_config['profile_path'])
+  for dir in browser_config['dirs']:
+    browser_config['dirs'][dir] = os.path.normpath(browser_config['dirs'][dir])
   tests = yaml_config['tests']
   config_file.close()
   if (testdate != ''):
@@ -268,16 +283,20 @@ def test_file(filename):
     print "Completed test: " + test
 
   #process the results
-  if config.TO_GRAPH_SERVER:
+  if (results_server != '') and (results_link != ''):
     #send results to the graph server
-    send_to_graph(title, date, browser_config, results)
-  if config.TO_CSV:
-    send_to_csv(results)
+    send_to_graph(results_server, results_link, title, date, browser_config, results)
+  if csv_file != '':
+    send_to_csv(csv_file, results)
   
 if __name__=='__main__':
-
+  optlist, args = getopt.getopt(sys.argv[1:], 'd', ['debug'])
+  for o, a in optlist:
+    if o in ('-d', "--debug"):
+      print 'setting debug'
+      utils.setdebug(1)
   # Read in each config file and run the tests on it.
-  for i in range(1, len(sys.argv)):
-    utils.debug("running test file " + sys.argv[i])
-    test_file(sys.argv[i])
+  for arg in args:
+    utils.debug("running test file " + arg)
+    test_file(arg)
 

testing/performance/talos/sample.config

@@ -3,14 +3,23 @@
 # The title of the report
 title: firefox_testing
 
+#*** output options ****
+#uncomment to turn on dump to csv option
+#csv_file: 'output/out'
+#comment out next two lines to disable send to graph server
+results_server: 'url.of.graphserver'
+results_link: '/bulk.cgi'
+
 # Path to Firefox to test
-firefox: C:\mozilla\testing\performance\firefox\firefox.exe
+firefox: firefox/firefox.exe
 
 branch: testbranch
 
 buildid: testbuildid
 
-profile_path: C:\mozilla\testing\performance\talos\base_profile
+profile_path: base_profile
+
+init_url: getInfo.html
 
 # Preferences to set in the test (use "preferences : {}" for no prefs)
 preferences : 
@@ -39,8 +48,8 @@ extensions : {}
 #any directories whose contents need to be installed in the browser before running the tests
 # this assumes that the directories themselves already exist in the firefox path
 dirs:
-  chrome : page_load_test\chrome
-  components : page_load_test\components
+  chrome : page_load_test/chrome
+  components : page_load_test/components
 
 # Environment variables to set during test (use env: {} for none)
 env : 
@@ -63,19 +72,22 @@ env :
 # to see how the data passed from the browser is processed see send_to_graph and send_to_csv in run_tests.py
 tests :
   ts :
-    url : file:///c:/mozilla/testing/performance/talos/startup_test/startup_test.html?begin=
+    url : startup_test/startup_test.html?begin=
     url_mod : str(int(time.time()*1000))
     resolution : 1 
     cycles : 20
-    counters : []
+    win_counters : []
+    unix_counters : []
   tp: 
-    url :  '-tp c:\\\mozilla\\\testing\\\performance\\\talos\\\page_load_test\\\manifest.txt -tpchrome -tpformat tinderbox -tpcycles 5'
+    url :  '-tp page_load_test/manifest.txt -tpchrome -tpformat tinderbox -tpcycles 5'
     resolution : 1
     cycles : 1
-    counters : ['Working Set', 'Private Bytes', '% Processor Time']
+    win_counters : ['Working Set', 'Private Bytes', '% Processor Time']
+    unix_counters : [ 'Private Bytes', 'RSS']
   tp_js: 
     url : '"http://localhost/page_load_test/framecycler.html?quit=1&cycles=5"'
     resolution : 1
     cycles : 1
-    counters : ['Working Set', 'Private Bytes', '% Processor Time']
+    win_counters : ['Working Set', 'Private Bytes', '% Processor Time']
+    unix_counters : [ 'Private Bytes', 'RSS']
   

testing/performance/talos/ttest.py

@@ -63,10 +63,13 @@ import ffsetup
 
 if platform.system() == "Linux":
     from cmanager_linux import *
+    platform_type = 'unix_'
 elif platform.system() == "Windows":
     from cmanager_win32 import *
+    platform_type = 'win_'
 elif platform.system() == "Darwin":
     from cmanager_mac import *
+    platform_type = 'unix_'
 
 
 # Regula expression for getting results from most tests
@@ -89,7 +92,8 @@ def runTest(browser_config, test_config):
   """
  
   res = 0
-  counters = test_config['counters']
+  utils.debug("operating with platform_type : " + platform_type)
+  counters = test_config[platform_type + 'counters']
   resolution = test_config['resolution']
   all_browser_results = []
   all_counter_results = []
@@ -106,7 +110,7 @@ def runTest(browser_config, test_config):
   # Run Firefox once with new profile so initializing it doesn't cause
   # a performance hit, and the second Firefox that gets created is properly
   # terminated.
-  res = ffsetup.InitializeNewProfile(browser_config['firefox'], profile_dir)
+  res = ffsetup.InitializeNewProfile(browser_config['firefox'], profile_dir, browser_config['init_url'])
   if not res:
     print "FAIL: couldn't initialize firefox"
     return (res, all_browser_results, all_counter_results)

testing/performance/talos/utils.py

@@ -36,23 +36,27 @@
 
 """Utility functions"""
 
-import config
 import os
-
+DEBUG = 0
 saved_environment = {}
 
+def setdebug(val):
+  global DEBUG
+  DEBUG = val
+
 def debug(message):
   """Prints a debug message to the console if the DEBUG switch is turned on 
-     in config.py
+     debug switch is controlled through command line switch (-d or --debug)
      Args:
        message: string containing a debugging statement
      """
-  if config.DEBUG == 1:
+  if DEBUG == 1:
     print message
 
 def setEnvironmentVars(newVars): 
    """Sets environment variables as specified by env, an array of variables 
-   from config.py"""
+   from sample.config"""
+   global saved_environment
    env = os.environ
    for var in newVars:
      # save the old values so they can be restored later:
@@ -65,5 +69,6 @@ def setEnvironmentVars(newVars):
 def restoreEnvironmentVars():
   """Restores environment variables to the state they were in before 
   setEnvironmentVars() was last called"""
+  global saved_environment
   for var in saved_environment:
     os.environ[var] = saved_environment[var]