From 04a25ecb141b0c1070e41d43e98bd8350bc7460c Mon Sep 17 00:00:00 2001 From: Narcis Beleuzu Date: Fri, 8 Dec 2017 12:03:06 +0200 Subject: [PATCH] Backed out changeset 6e523917ee97 (bug 1423353) as requested by igoldan. a=backout --- testing/talos/talos/unittests/python.ini | 1 - testing/talos/talos/unittests/test_config.py | 5 +- .../unittests/test_xtalos/test_etlparser.py | 64 -------- testing/talos/talos/xtalos/__init__.py | 6 +- testing/talos/talos/xtalos/etlparser.py | 28 +--- testing/talos/talos/xtalos/start_xperf.py | 2 +- .../talos/talos/xtalos/xperf_whitelist.json | 155 +++--------------- 7 files changed, 39 insertions(+), 222 deletions(-) delete mode 100644 testing/talos/talos/unittests/test_xtalos/test_etlparser.py diff --git a/testing/talos/talos/unittests/python.ini b/testing/talos/talos/unittests/python.ini index 1a88b76feda7..5bfc864d7810 100644 --- a/testing/talos/talos/unittests/python.ini +++ b/testing/talos/talos/unittests/python.ini @@ -4,4 +4,3 @@ subsuite = talos [test_config.py] [test_ffsetup.py] [test_test.py] -[test_xtalos/test_etlparser.py] diff --git a/testing/talos/talos/unittests/test_config.py b/testing/talos/talos/unittests/test_config.py index 4f88f3d8ecec..0e9321ca641c 100644 --- a/testing/talos/talos/unittests/test_config.py +++ b/testing/talos/talos/unittests/test_config.py @@ -122,9 +122,6 @@ class Test_get_browser_config(object): optional = ['bcontroller_config', 'branch_name', 'child_process', - 'debug', - 'debugger', - 'debugger_args', 'develop', 'e10s', 'process', @@ -179,7 +176,7 @@ class Test_get_browser_config(object): browser_config = get_browser_config(config_extensive) assert browser_config != config_extensive - assert set(browser_config).issubset(set(config_extensive)) + assert set(browser_config.keys()).issubset(set(config_extensive.keys())) class Test_get_config(object): diff --git a/testing/talos/talos/unittests/test_xtalos/test_etlparser.py b/testing/talos/talos/unittests/test_xtalos/test_etlparser.py deleted file mode 100644 index 19abe1e2207d..000000000000 --- a/testing/talos/talos/unittests/test_xtalos/test_etlparser.py +++ /dev/null @@ -1,64 +0,0 @@ -from __future__ import absolute_import - -import re - -import mozunit - -from talos.xtalos.etlparser import NAME_SUBSTITUTIONS - - -def test_NAME_SUBSTITUTIONS(): - filepaths_map = { - # tp5n files - r'{talos}\talos\tests\tp5n\alibaba.com\i03.i.aliimg.com\images\eng\style\css_images': - r'{talos}\talos\tests\{tp5n_files}', - r'{talos}\talos\tests\tp5n\cnet.com\i.i.com.com\cnwk.1d\i\tron\fd': - r'{talos}\talos\tests\{tp5n_files}', - r'{talos}\talos\tests\tp5n\tp5n.manifest': - r'{talos}\talos\tests\{tp5n_files}', - r'{talos}\talos\tests\tp5n\tp5n.manifest.develop': - r'{talos}\talos\tests\{tp5n_files}', - r'{talos}\talos\tests\tp5n\yelp.com\media1.ct.yelpcdn.com\photo': - r'{talos}\talos\tests\{tp5n_files}', - - # cltbld for Windows 7 32bit - r'c:\users\cltbld.t-w732-ix-015.000\appdata\locallow\mozilla': - r'c:\users\{cltbld}\appdata\locallow\mozilla', - r'c:\users\cltbld.t-w732-ix-035.000\appdata\locallow\mozilla': - r'c:\users\{cltbld}\appdata\locallow\mozilla', - r'c:\users\cltbld.t-w732-ix-058.000\appdata\locallow\mozilla': - r'c:\users\{cltbld}\appdata\locallow\mozilla', - r'c:\users\cltbld.t-w732-ix-112.001\appdata\local\temp': - r'c:\users\{cltbld}\appdata\local\temp', - - # nvidia's 3D Vision - r'c:\program files\nvidia corporation\3d vision\npnv3dv.dll': - r'c:\program files\{nvidia_3d_vision}', - r'c:\program files\nvidia corporation\3d vision\npnv3dvstreaming.dll': - r'c:\program files\{nvidia_3d_vision}', - r'c:\program files\nvidia corporation\3d vision\nvstereoapii.dll': - r'c:\program files\{nvidia_3d_vision}', - - r'{firefox}\browser\extensions\{45b6d270-f6ec-4930-a6ad-14bac5ea2204}.xpi': - r'{firefox}\browser\extensions\{uuid}.xpi', - - r'c:\slave\test\build\venv\lib\site-packages\pip\_vendor\html5lib\treebuilders': - r'c:\slave\test\build\venv\lib\site-packages\{pip_vendor}', - r'c:\slave\test\build\venv\lib\site-packages\pip\_vendor\colorama': - r'c:\slave\test\build\venv\lib\site-packages\{pip_vendor}', - r'c:\slave\test\build\venv\lib\site-packages\pip\_vendor\cachecontrol\caches': - r'c:\slave\test\build\venv\lib\site-packages\{pip_vendor}', - r'c:\slave\test\build\venv\lib\site-packages\pip\_vendor\requests\packages\urllib3' - r'\packages\ssl_match_hostname': - r'c:\slave\test\build\venv\lib\site-packages\{pip_vendor}', - } - - for given_raw_path, exp_normal_path in filepaths_map.items(): - normal_path = given_raw_path - for pattern, substitution in NAME_SUBSTITUTIONS: - normal_path = re.sub(pattern, substitution, normal_path) - assert exp_normal_path == normal_path - - -if __name__ == '__main__': - mozunit.main() diff --git a/testing/talos/talos/xtalos/__init__.py b/testing/talos/talos/xtalos/__init__.py index b8e495152a13..2e574590e9ce 100644 --- a/testing/talos/talos/xtalos/__init__.py +++ b/testing/talos/talos/xtalos/__init__.py @@ -1,6 +1,6 @@ # xtalos: talos + xperf from __future__ import absolute_import -from . import etlparser # noqa -from .start_xperf import start # noqa -from .start_xperf import start_from_config # noqa +from start_xperf import start # noqa +from start_xperf import start_from_config # noqa +import etlparser # noqa diff --git a/testing/talos/talos/xtalos/etlparser.py b/testing/talos/talos/xtalos/etlparser.py index 482769ec6752..d454ff629ffb 100644 --- a/testing/talos/talos/xtalos/etlparser.py +++ b/testing/talos/talos/xtalos/etlparser.py @@ -14,7 +14,7 @@ import subprocess import sys import mozfile -from . import xtalos +import xtalos EVENTNAME_INDEX = 0 PROCESS_INDEX = 2 @@ -29,16 +29,7 @@ NUMBYTES_COL = "NumBytes" CEVT_WINDOWS_RESTORED = "{917b96b1-ecad-4dab-a760-8d49027748ae}" CEVT_XPCOM_SHUTDOWN = "{26d1e091-0ae7-4f49-a554-4214445c505c}" NAME_SUBSTITUTIONS = [ - # Careful with your regex! - # Substitution happens after combinations like \t \s \n ... are replaced - # with their real representations. So, prepend them with extra backslash. - # Read more: https://docs.python.org/2.7/library/re.html#re.sub - (re.compile(r'{\w{8}-\w{4}-\w{4}-\w{4}-\w{12}}'), '{uuid}'), - (re.compile(r'talos\\tests\\tp5n\\.*'), r'talos\\tests\{tp5n_files}'), - (re.compile(r'nvidia corporation\\3d vision\\.*'), '{nvidia_3d_vision}'), - (re.compile(r'cltbld\.t-w732-ix-\d+\.\d+'), '{cltbld}'), - (re.compile(r'venv\\lib\\site-packages\\pip\\_vendor\\.*'), - r'venv\lib\\site-packages\\{pip_vendor}'), + (re.compile(r'{\w{8}-\w{4}-\w{4}-\w{4}-\w{12}}'), '{uuid}') ] stages = ["startup", "normal", "shutdown"] net_events = { @@ -377,19 +368,19 @@ def etlparser(xperf_path, etl_filename, processID, approot=None, # We still like to have the outputfile to record the raw data, now # filter out acceptable files/ranges - whitelist_path = None + filename = None wl_temp = {} dirname = os.path.dirname(__file__) if os.path.exists(os.path.join(dirname, 'xperf_whitelist.json')): - whitelist_path = os.path.join(dirname, 'xperf_whitelist.json') + filename = os.path.join(dirname, 'xperf_whitelist.json') elif os.path.exists(os.path.join(dirname, 'xtalos')) and \ os.path.exists(os.path.join(dirname, 'xtalos', 'xperf_whitelist.json')): - whitelist_path = os.path.join(dirname, 'xtalos', 'xperf_whitelist.json') + filename = os.path.join(dirname, 'xtalos', 'xperf_whitelist.json') wl_temp = {} - if whitelist_path: - with open(whitelist_path, 'r') as fHandle: + if filename: + with open(filename, 'r') as fHandle: wl_temp = json.load(fHandle) # Approot is the full path where the application is located at @@ -411,7 +402,7 @@ def etlparser(xperf_path, etl_filename, processID, approot=None, errors = [] for row in filekeys: - filename = original_filename = row[0] + filename = row[0] filename = filename.lower() # take care of 'program files (x86)' matching 'program files' filename = filename.replace(" (x86)", '') @@ -474,11 +465,10 @@ def etlparser(xperf_path, etl_filename, processID, approot=None, # files[row]['DiskWriteCount']), # wl[filename]['maxcount'])) else: - errors.append("File '%s' (normalized from '%s') was accessed and we were not expecting" + errors.append("File '%s' was accessed and we were not expecting" " it. DiskReadCount: %s, DiskWriteCount: %s," " DiskReadBytes: %s, DiskWriteBytes: %s" % (filename, - original_filename, files[row]['DiskReadCount'], files[row]['DiskWriteCount'], files[row]['DiskReadBytes'], diff --git a/testing/talos/talos/xtalos/start_xperf.py b/testing/talos/talos/xtalos/start_xperf.py index 83464f848b3a..26ed421274c2 100644 --- a/testing/talos/talos/xtalos/start_xperf.py +++ b/testing/talos/talos/xtalos/start_xperf.py @@ -9,7 +9,7 @@ import os import subprocess import sys -from . import xtalos +import xtalos def start(xperf_path, xperf_providers, xperf_stackwalk, xperf_user_providers, diff --git a/testing/talos/talos/xtalos/xperf_whitelist.json b/testing/talos/talos/xtalos/xperf_whitelist.json index 83259cfff1ea..3a3f63f3caab 100644 --- a/testing/talos/talos/xtalos/xperf_whitelist.json +++ b/testing/talos/talos/xtalos/xperf_whitelist.json @@ -1,20 +1,20 @@ { - "\\fi_unknown": { - "mincount": 0, - "maxcount": 2, - "minbytes": 0, - "maxbytes": 8192 - }, - "c:\\$extend\\$usnjrnl:$j": { + "C:\\$Extend\\$UsnJrnl:$J": { "ignore": true }, - "c:\\$logfile": { + "C:\\$Mft": { "ignore": true }, - "c:\\$mft": { + "C:\\$Secure": { "ignore": true }, - "c:\\$secure": { + "C:\\$logfile": { + "ignore": true + }, + "Z:\\$logfile": { + "ignore": true + }, + "C:\\Windows\\Prefetch\\{prefetch}.pf": { "ignore": true }, "c:\\program files\\desktop.ini": { @@ -23,11 +23,11 @@ "minbytes": 352, "maxbytes": 352 }, - "c:\\program files\\{nvidia_3d_vision}": { + "c:\\program files\\nvidia corporation\\3d vision\\nvstereoapii.dll": { "mincount": 0, - "maxcount": 24, - "minbytes": 0, - "maxbytes": 578560 + "maxcount": 4, + "minbytes": 2, + "maxbytes": 33792 }, "c:\\programdata\\nvidia corporation\\drs\\nvapptimestamps": { "mincount": 22, @@ -41,36 +41,18 @@ "minbytes": 2, "maxbytes": 2 }, - "c:\\slave\\test\\build\\venv\\lib\\site-packages\\{pip_vendor}": { - "ignore": true - }, - "c:\\users\\{cltbld}\\appdata\\locallow\\mozilla": { - "mincount": 0, - "maxcount": 8, - "minbytes": 0, - "maxbytes": 32768 - }, "c:\\users\\desktop.ini": { "mincount": 2, "maxcount": 2, "minbytes": 352, "maxbytes": 352 }, - "c:\\windows\\fonts\\segoeuii.ttf": { - "mincount": 0, - "maxcount": 4, - "minbytes": 0, - "maxbytes": 90112 - }, "c:\\windows\\fonts\\staticcache.dat": { "mincount": 2, "maxcount": 2, "minbytes": 120, "maxbytes": 120 }, - "c:\\windows\\prefetch\\{prefetch}.pf": { - "ignore": true - }, "c:\\windows\\system32\\audioses.dll": { "mincount": 4, "maxcount": 4, @@ -83,45 +65,18 @@ "minbytes": 16384, "maxbytes": 90112 }, - "c:\\windows\\system32\\gdi32.dll": { - "mincount": 0, - "maxcount": 4, - "minbytes": 0, - "maxbytes": 114688 - }, - "c:\\windows\\system32\\kernel32.dll": { - "mincount": 0, - "maxcount": 2, - "minbytes": 0, - "maxbytes": 65536 - }, - "c:\\windows\\system32\\mscms.dll": { - "mincount": 0, - "maxcount": 73728, - "minbytes": 0, - "maxbytes": 73728 - }, "c:\\windows\\system32\\msctf.dll": { "mincount": 0, "maxcount": 2, "minbytes": 0, "maxbytes": 65536 }, - "c:\\windows\\system32\\spool\\drivers\\color": { - "mincount": 0, - "maxcount": 2, - "minbytes": 0, - "maxbytes": 8192 - }, "c:\\windows\\system32\\spool\\drivers\\color\\srgb color space profile.icm": { "mincount": 2, "maxcount": 2, "minbytes": 8192, "maxbytes": 8192 }, - "z:\\$logfile": { - "ignore": true - }, "{appdata}\\local\\temp": { "mincount": 0, "maxcount": 4, @@ -134,29 +89,8 @@ "minbytes": 1692, "maxbytes": 1692 }, - "{firefox}\\browser\\blocklist.xml": { - "mincount": 0, - "maxcount": 66, - "minbytes": 0, - "maxbytes": 540672 - }, - "{firefox}\\browser\\extensions\\{uuid}.xpi": { - "mincount": 0, - "maxcount": 2, - "minbytes": 0, - "maxbytes": 8192 - }, - "{firefox}\\browser\\features\\aushelper@mozilla.org.xpi": { - "mincount": 0, - "maxcount": 2, - "minbytes": 0, - "maxbytes": 24576 - }, - "{firefox}\\browser\\features\\flyweb@mozilla.org.xpi": { - "mincount": 0, - "maxcount": 2, - "minbytes": 0, - "maxbytes": 32768 + "{firefox}\\Crash Reports\\{time}": { + "ignore": true }, "{firefox}\\browser\\omni.ja": { "mincount": 0, @@ -164,15 +98,6 @@ "minbytes": 0, "maxbytes": 1835008 }, - "{firefox}\\crash reports\\lastcrash": { - "mincount": 0, - "maxcount": 2, - "minbytes": 0, - "maxbytes": 20 - }, - "{firefox}\\crash reports\\{time}": { - "ignore": true - }, "{firefox}\\defaults\\pref\\channel-prefs.js": { "mincount": 4, "maxcount": 4, @@ -245,12 +170,6 @@ "minbytes": 65768, "maxbytes": 65768 }, - "{profile}\\content-prefs.sqlite-journal": { - "mincount": 14, - "maxcount": 116, - "minbytes": 112, - "maxbytes": 1244640 - }, "{profile}\\cookies.sqlite": { "mincount": 1, "maxcount": 6, @@ -323,12 +242,6 @@ "minbytes": 0, "maxbytes": 127000 }, - "{profile}\\extensions\\pageloader@mozilla.org\\install.rdf": { - "mincount": 0, - "maxcount": 2, - "minbytes": 0, - "maxbytes": 8192 - }, "{profile}\\extensions\\talos-powers@mozilla.org\\chrome.manifest": { "mincount": 2, "maxcount": 2, @@ -347,12 +260,6 @@ "minbytes": 196808, "maxbytes": 393216 }, - "{profile}\\favicons.sqlite-journal": { - "mincount": 2, - "maxcount": 8, - "minbytes": 16, - "maxbytes": 9264 - }, "{profile}\\favicons.sqlite-shm": { "mincount": 0, "maxcount": 4, @@ -371,12 +278,6 @@ "minbytes": 0, "maxbytes": 1388 }, - "{profile}\\invalidprefs.js": { - "mincount": 0, - "maxcount": 2, - "minbytes": 0, - "maxbytes": 25412 - }, "{profile}\\key3.db": { "mincount": 0, "maxcount": 12, @@ -419,12 +320,6 @@ "minbytes": 196808, "maxbytes": 196808 }, - "{profile}\\places.sqlite-journal": { - "mincount": 2, - "maxcount": 8, - "minbytes": 16, - "maxbytes": 9264 - }, "{profile}\\places.sqlite-shm": { "mincount": 2, "maxcount": 2, @@ -443,12 +338,6 @@ "minbytes": 1892, "maxbytes": 1892 }, - "{profile}\\pluginreg.dat.tmp": { - "mincount": 0, - "maxcount": 30, - "minbytes": 0, - "maxbytes": 958 - }, "{profile}\\prefs-1.js": { "mincount": 0, "maxcount": 10, @@ -515,10 +404,16 @@ "minbytes": 0, "maxbytes": 702 }, - "{talos}\\talos\\tests\\{tp5n_files}": { + "{talos}\\talos\\tests\\tp5n\\tp5n.manifest": { "mincount": 0, - "maxcount": 2, + "maxcount": 8, "minbytes": 0, - "maxbytes": 16384 + "maxbytes": 32786 + }, + "{talos}\\talos\\tests\\tp5n\\tp5n.manifest.develop": { + "mincount": 0, + "maxcount": 8, + "minbytes": 0, + "maxbytes": 32786 } }