From afae3224eb815803c10f11a77d0b4750587619c8 Mon Sep 17 00:00:00 2001 From: "anodelman@mozilla.com" Date: Wed, 19 Sep 2007 11:02:07 -0700 Subject: [PATCH] Bug 396232 - one sample.config for all platforms. a=anodelman, r=bhearsum --- testing/performance/talos/config.py | 55 ------------------- testing/performance/talos/ffprocess.py | 2 - testing/performance/talos/ffprocess_linux.py | 1 - testing/performance/talos/ffprocess_mac.py | 1 - testing/performance/talos/ffprocess_win32.py | 2 - testing/performance/talos/ffsetup.py | 5 +- testing/performance/talos/run_tests.py | 57 +++++++++++++------- testing/performance/talos/sample.config | 30 +++++++---- testing/performance/talos/ttest.py | 8 ++- testing/performance/talos/utils.py | 15 ++++-- 10 files changed, 77 insertions(+), 99 deletions(-) delete mode 100644 testing/performance/talos/config.py diff --git a/testing/performance/talos/config.py b/testing/performance/talos/config.py deleted file mode 100644 index 5c4602db66f..00000000000 --- a/testing/performance/talos/config.py +++ /dev/null @@ -1,55 +0,0 @@ -# ***** BEGIN LICENSE BLOCK ***** -# Version: MPL 1.1/GPL 2.0/LGPL 2.1 -# -# The contents of this file are subject to the Mozilla Public License Version -# 1.1 (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# http://www.mozilla.org/MPL/ -# -# Software distributed under the License is distributed on an "AS IS" basis, -# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License -# for the specific language governing rights and limitations under the -# License. -# -# The Original Code is standalone Firefox Windows performance test. -# -# The Initial Developer of the Original Code is Google Inc. -# Portions created by the Initial Developer are Copyright (C) 2006 -# the Initial Developer. All Rights Reserved. -# -# Contributor(s): -# Annie Sullivan (original author) -# Alice Nodelman -# -# Alternatively, the contents of this file may be used under the terms of -# either the GNU General Public License Version 2 or later (the "GPL"), or -# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), -# in which case the provisions of the GPL or the LGPL are applicable instead -# of those above. If you wish to allow use of your version of this file only -# under the terms of either the GPL or the LGPL, and not to allow others to -# use your version of this file under the terms of the MPL, indicate your -# decision by deleting the provisions above and replace them with the notice -# and other provisions required by the GPL or the LGPL. If you do not delete -# the provisions above, a recipient may use your version of this file under -# the terms of any one of the MPL, the GPL or the LGPL. -# -# ***** END LICENSE BLOCK ***** - -"""A list of constants containing the paths to programs and files - needed by the performance testing scripts. -""" -__author__ = 'annie.sullivan@gmail.com (Annie Sullivan)' - -"""The path to the file url to load when initializing/collecting information from the browser""" -INIT_URL = 'file:///c:/mozilla/testing/performance/talos/getInfo.html' - -"""Dump results locally to csv""" -TO_CSV = 0 -CSV_FILE = r'c:\mozilla\testing\performance\talos\output\out' -"""URL for the results server""" -TO_GRAPH_SERVER = 1 -RESULTS_SERVER = 'graphserver.url.here' -RESULTS_LINK = '/bulk.cgi' - -"""Enable/disable debugging output""" -DEBUG = 0 diff --git a/testing/performance/talos/ffprocess.py b/testing/performance/talos/ffprocess.py index 854d314110d..2435afa1e78 100755 --- a/testing/performance/talos/ffprocess.py +++ b/testing/performance/talos/ffprocess.py @@ -47,8 +47,6 @@ import re import time import subprocess -import config - if platform.system() == "Linux": from ffprocess_linux import * elif platform.system() == "Windows": diff --git a/testing/performance/talos/ffprocess_linux.py b/testing/performance/talos/ffprocess_linux.py index 7a9154d88d3..bf7e8f2bc56 100644 --- a/testing/performance/talos/ffprocess_linux.py +++ b/testing/performance/talos/ffprocess_linux.py @@ -40,7 +40,6 @@ import signal import os from select import select -import config def GenerateFirefoxCommandLine(firefox_path, profile_dir, url): diff --git a/testing/performance/talos/ffprocess_mac.py b/testing/performance/talos/ffprocess_mac.py index 66b5370d4cb..e2ff3b99ff1 100644 --- a/testing/performance/talos/ffprocess_mac.py +++ b/testing/performance/talos/ffprocess_mac.py @@ -40,7 +40,6 @@ import subprocess import signal import os import time -import config from select import select diff --git a/testing/performance/talos/ffprocess_win32.py b/testing/performance/talos/ffprocess_win32.py index 2cff6b31f97..83818e5db34 100644 --- a/testing/performance/talos/ffprocess_win32.py +++ b/testing/performance/talos/ffprocess_win32.py @@ -41,8 +41,6 @@ import win32pdhutil import win32pipe import msvcrt -import config - def GenerateFirefoxCommandLine(firefox_path, profile_dir, url): """Generates the command line for a process to run Firefox diff --git a/testing/performance/talos/ffsetup.py b/testing/performance/talos/ffsetup.py index aa2dec854df..bf62bc19573 100644 --- a/testing/performance/talos/ffsetup.py +++ b/testing/performance/talos/ffsetup.py @@ -53,7 +53,6 @@ import glob import utils import ffprocess -import config if platform.system() == "Linux": from ffprofile_unix import * @@ -141,7 +140,7 @@ def InstallInBrowser(firefox_path, dir_path): else: utils.debug("WARNING: file already installed (" + fromfile + ")") -def InitializeNewProfile(firefox_path, profile_dir): +def InitializeNewProfile(firefox_path, profile_dir, init_url): """Runs Firefox with the new profile directory, to negate any performance hit that could occur as a result of starting up with a new profile. Also kills the "extra" Firefox that gets spawned the first time Firefox @@ -153,7 +152,7 @@ def InitializeNewProfile(firefox_path, profile_dir): """ PROFILE_REGEX = re.compile('__metrics(.*)__metrics', re.DOTALL|re.MULTILINE) res = 1 - cmd = ffprocess.GenerateFirefoxCommandLine(firefox_path, profile_dir, config.INIT_URL) + cmd = ffprocess.GenerateFirefoxCommandLine(firefox_path, profile_dir, init_url) (match, timed_out) = ffprocess.RunProcessAndWaitForOutput(cmd, 'firefox', PROFILE_REGEX, diff --git a/testing/performance/talos/run_tests.py b/testing/performance/talos/run_tests.py index 0b60977cbf3..685bdd6e7d9 100755 --- a/testing/performance/talos/run_tests.py +++ b/testing/performance/talos/run_tests.py @@ -57,9 +57,9 @@ import os import string import socket socket.setdefaulttimeout(480) +import getopt import utils -import config import post_file import ttest @@ -83,11 +83,11 @@ def process_Request(post): str += line.split(":")[3] + ":" + shortNames(line.split(":")[1]) + ":" + line.split(":")[2] + '\n' return str -def send_to_csv(results): +def send_to_csv(csv_file, results): import csv for res in results: browser_dump, counter_dump = results[res] - writer = csv.writer(open(config.CSV_FILE + '_' + res, "wb")) + writer = csv.writer(open(csv_file + '_' + res, "wb")) if res == 'ts': i = 0 writer.writerow(['i', 'val']) @@ -110,19 +110,19 @@ def send_to_csv(results): i += 1 for cd in counter_dump: for count_type in cd: - writer = csv.writer(open(config.CSV_FILE + '_' + res + '_' + count_type, "wb")) + writer = csv.writer(open(csv_file + '_' + res + '_' + count_type, "wb")) writer.writerow(['i', 'value']) i = 0 for val in cd[count_type]: writer.writerow([i, val]) i += 1 -def post_chunk(id, filename): +def post_chunk(results_server, results_link, id, filename): tmpf = open(filename, "r") file_data = tmpf.read() while True: try: - ret = post_file.post_multipart(config.RESULTS_SERVER, config.RESULTS_LINK, [("key", "value")], [("filename", filename, file_data)]) + ret = post_file.post_multipart(results_server, results_link, [("key", "value")], [("filename", filename, file_data)]) except IOError: print "FAIL: IOError on sending data to the graph server" else: @@ -144,7 +144,7 @@ def chunk_list(val_list): val_list = val_list[end:len(val_list)] return chunks -def send_to_graph(title, date, browser_config, results): +def send_to_graph(results_server, results_link, title, date, browser_config, results): tbox = title url_format = "http://%s/%s" link_format= "%s" @@ -185,7 +185,7 @@ def send_to_graph(title, date, browser_config, results): i += 1 tmpf.flush() tmpf.close() - links += post_chunk(res, filename) + links += post_chunk(results_server, results_link, res, filename) os.remove(filename) for cd in counter_dump: for count_type in cd: @@ -201,7 +201,7 @@ def send_to_graph(title, date, browser_config, results): i += 1 tmpf.flush() tmpf.close() - chunk_link = post_chunk('%s_%s (%d values)' % (res, count_type, len(chunk)), filename) + chunk_link = post_chunk(results_server, results_link, '%s_%s (%d values)' % (res, count_type, len(chunk)), filename) os.remove(filename) links += chunk_link @@ -215,7 +215,7 @@ def send_to_graph(title, date, browser_config, results): linkName += "_T: " + values[2] else: linkName += "_1" - url = url_format % (config.RESULTS_SERVER, values[0],) + url = url_format % (results_server, values[0],) link = link_format % (url, linkName,) print "RETURN: " + link @@ -230,6 +230,9 @@ def test_file(filename): tests = [] title = '' testdate = '' + csv_file = '' + results_server = '' + results_link = '' results = {} # Read in the profile info from the YAML config file @@ -241,6 +244,12 @@ def test_file(filename): title = yaml_config[item] elif item == 'testdate': testdate = yaml_config[item] + elif item == 'csv_file': + csv_file = os.path.normpath(yaml_config[item]) + elif item == 'results_server': + results_server = yaml_config[item] + elif item == 'results_link' : + results_link = yaml_config[item] browser_config = {'preferences' : yaml_config['preferences'], 'extensions' : yaml_config['extensions'], 'firefox' : yaml_config['firefox'], @@ -248,7 +257,13 @@ def test_file(filename): 'buildid' : yaml_config['buildid'], 'profile_path' : yaml_config['profile_path'], 'env' : yaml_config['env'], - 'dirs' : yaml_config['dirs']} + 'dirs' : yaml_config['dirs'], + 'init_url' : yaml_config['init_url']} + #normalize paths to work accross platforms + browser_config['firefox'] = os.path.normpath(browser_config['firefox']) + browser_config['profile_path'] = os.path.normpath(browser_config['profile_path']) + for dir in browser_config['dirs']: + browser_config['dirs'][dir] = os.path.normpath(browser_config['dirs'][dir]) tests = yaml_config['tests'] config_file.close() if (testdate != ''): @@ -268,16 +283,20 @@ def test_file(filename): print "Completed test: " + test #process the results - if config.TO_GRAPH_SERVER: + if (results_server != '') and (results_link != ''): #send results to the graph server - send_to_graph(title, date, browser_config, results) - if config.TO_CSV: - send_to_csv(results) + send_to_graph(results_server, results_link, title, date, browser_config, results) + if csv_file != '': + send_to_csv(csv_file, results) if __name__=='__main__': - + optlist, args = getopt.getopt(sys.argv[1:], 'd', ['debug']) + for o, a in optlist: + if o in ('-d', "--debug"): + print 'setting debug' + utils.setdebug(1) # Read in each config file and run the tests on it. - for i in range(1, len(sys.argv)): - utils.debug("running test file " + sys.argv[i]) - test_file(sys.argv[i]) + for arg in args: + utils.debug("running test file " + arg) + test_file(arg) diff --git a/testing/performance/talos/sample.config b/testing/performance/talos/sample.config index aa4a9df7891..0ba8b80d864 100755 --- a/testing/performance/talos/sample.config +++ b/testing/performance/talos/sample.config @@ -3,14 +3,23 @@ # The title of the report title: firefox_testing +#*** output options **** +#uncomment to turn on dump to csv option +#csv_file: 'output/out' +#comment out next two lines to disable send to graph server +results_server: 'url.of.graphserver' +results_link: '/bulk.cgi' + # Path to Firefox to test -firefox: C:\mozilla\testing\performance\firefox\firefox.exe +firefox: firefox/firefox.exe branch: testbranch buildid: testbuildid -profile_path: C:\mozilla\testing\performance\talos\base_profile +profile_path: base_profile + +init_url: getInfo.html # Preferences to set in the test (use "preferences : {}" for no prefs) preferences : @@ -39,8 +48,8 @@ extensions : {} #any directories whose contents need to be installed in the browser before running the tests # this assumes that the directories themselves already exist in the firefox path dirs: - chrome : page_load_test\chrome - components : page_load_test\components + chrome : page_load_test/chrome + components : page_load_test/components # Environment variables to set during test (use env: {} for none) env : @@ -63,19 +72,22 @@ env : # to see how the data passed from the browser is processed see send_to_graph and send_to_csv in run_tests.py tests : ts : - url : file:///c:/mozilla/testing/performance/talos/startup_test/startup_test.html?begin= + url : startup_test/startup_test.html?begin= url_mod : str(int(time.time()*1000)) resolution : 1 cycles : 20 - counters : [] + win_counters : [] + unix_counters : [] tp: - url : '-tp c:\\\mozilla\\\testing\\\performance\\\talos\\\page_load_test\\\manifest.txt -tpchrome -tpformat tinderbox -tpcycles 5' + url : '-tp page_load_test/manifest.txt -tpchrome -tpformat tinderbox -tpcycles 5' resolution : 1 cycles : 1 - counters : ['Working Set', 'Private Bytes', '% Processor Time'] + win_counters : ['Working Set', 'Private Bytes', '% Processor Time'] + unix_counters : [ 'Private Bytes', 'RSS'] tp_js: url : '"http://localhost/page_load_test/framecycler.html?quit=1&cycles=5"' resolution : 1 cycles : 1 - counters : ['Working Set', 'Private Bytes', '% Processor Time'] + win_counters : ['Working Set', 'Private Bytes', '% Processor Time'] + unix_counters : [ 'Private Bytes', 'RSS'] diff --git a/testing/performance/talos/ttest.py b/testing/performance/talos/ttest.py index 571070c4b72..cf32b5b144a 100644 --- a/testing/performance/talos/ttest.py +++ b/testing/performance/talos/ttest.py @@ -63,10 +63,13 @@ import ffsetup if platform.system() == "Linux": from cmanager_linux import * + platform_type = 'unix_' elif platform.system() == "Windows": from cmanager_win32 import * + platform_type = 'win_' elif platform.system() == "Darwin": from cmanager_mac import * + platform_type = 'unix_' # Regula expression for getting results from most tests @@ -89,7 +92,8 @@ def runTest(browser_config, test_config): """ res = 0 - counters = test_config['counters'] + utils.debug("operating with platform_type : " + platform_type) + counters = test_config[platform_type + 'counters'] resolution = test_config['resolution'] all_browser_results = [] all_counter_results = [] @@ -106,7 +110,7 @@ def runTest(browser_config, test_config): # Run Firefox once with new profile so initializing it doesn't cause # a performance hit, and the second Firefox that gets created is properly # terminated. - res = ffsetup.InitializeNewProfile(browser_config['firefox'], profile_dir) + res = ffsetup.InitializeNewProfile(browser_config['firefox'], profile_dir, browser_config['init_url']) if not res: print "FAIL: couldn't initialize firefox" return (res, all_browser_results, all_counter_results) diff --git a/testing/performance/talos/utils.py b/testing/performance/talos/utils.py index 904090fed74..071971c8cee 100755 --- a/testing/performance/talos/utils.py +++ b/testing/performance/talos/utils.py @@ -36,23 +36,27 @@ """Utility functions""" -import config import os - +DEBUG = 0 saved_environment = {} +def setdebug(val): + global DEBUG + DEBUG = val + def debug(message): """Prints a debug message to the console if the DEBUG switch is turned on - in config.py + debug switch is controlled through command line switch (-d or --debug) Args: message: string containing a debugging statement """ - if config.DEBUG == 1: + if DEBUG == 1: print message def setEnvironmentVars(newVars): """Sets environment variables as specified by env, an array of variables - from config.py""" + from sample.config""" + global saved_environment env = os.environ for var in newVars: # save the old values so they can be restored later: @@ -65,5 +69,6 @@ def setEnvironmentVars(newVars): def restoreEnvironmentVars(): """Restores environment variables to the state they were in before setEnvironmentVars() was last called""" + global saved_environment for var in saved_environment: os.environ[var] = saved_environment[var]