From 9543a455592b07a32c1f9607e516abe557842f84 Mon Sep 17 00:00:00 2001 From: Jeff Gilbert Date: Fri, 22 Aug 2014 17:51:01 -0700 Subject: [PATCH] Bug 1052240 - Mark tests to fail/skip. - r=kamidphish --- .../test/webgl-conformance/00_test_list.txt | 2 + .../test/webgl-conformance/_mochitest.ini | 18 ++- .../_wrappers/test_always-fail.html | 16 +++ .../test/webgl-conformance/always-fail.html | 25 ++++ .../generate-wrappers-and-manifest.py | 76 ++++++++++- .../test/webgl-conformance/mochi-single.html | 121 +++++++++++++++++- .../webgl-conformance/mochitest-errata.ini | 72 +++++++++++ .../webgl-conformance/mochitest.ini.template | 3 +- 8 files changed, 323 insertions(+), 10 deletions(-) create mode 100644 dom/canvas/test/webgl-conformance/_wrappers/test_always-fail.html create mode 100644 dom/canvas/test/webgl-conformance/always-fail.html create mode 100644 dom/canvas/test/webgl-conformance/mochitest-errata.ini diff --git a/dom/canvas/test/webgl-conformance/00_test_list.txt b/dom/canvas/test/webgl-conformance/00_test_list.txt index 03c181f2393a..0dfec1244325 100644 --- a/dom/canvas/test/webgl-conformance/00_test_list.txt +++ b/dom/canvas/test/webgl-conformance/00_test_list.txt @@ -1,6 +1,8 @@ // files that end in .txt list other tests // other lines are assumed to be .html files +always-fail.html + conformance/00_test_list.txt conformance/more/00_test_list.txt diff --git a/dom/canvas/test/webgl-conformance/_mochitest.ini b/dom/canvas/test/webgl-conformance/_mochitest.ini index 5029548bb667..c2982c0d43c7 100644 --- a/dom/canvas/test/webgl-conformance/_mochitest.ini +++ b/dom/canvas/test/webgl-conformance/_mochitest.ini @@ -1,7 +1,9 @@ [DEFAULT] -skip-if = e10s +skip-if = e10s || ((os == 'linux') && (buildapp == 'b2g')) + support-files = mochi-single.html mochi-wrapper.css + always-fail.html conformance/00_readme.txt conformance/00_test_list.txt conformance/LICENSE_CHROMIUM @@ -477,6 +479,7 @@ support-files = mochi-single.html resources/webgl-logo.png resources/webgl-test-harness.js +[_wrappers/test_always-fail.html] [_wrappers/test_conformance__attribs__gl-enable-vertex-attrib.html] [_wrappers/test_conformance__attribs__gl-vertex-attrib-zero-issues.html] [_wrappers/test_conformance__attribs__gl-vertex-attrib.html] @@ -498,6 +501,7 @@ support-files = mochi-single.html [_wrappers/test_conformance__canvas__viewport-unchanged-upon-resize.html] [_wrappers/test_conformance__context__constants.html] [_wrappers/test_conformance__context__context-attributes-alpha-depth-stencil-antialias.html] +skip-if = (os == 'b2g') [_wrappers/test_conformance__context__context-lost-restored.html] [_wrappers/test_conformance__context__context-lost.html] [_wrappers/test_conformance__context__context-type-test.html] @@ -687,13 +691,16 @@ support-files = mochi-single.html [_wrappers/test_conformance__misc__error-reporting.html] [_wrappers/test_conformance__misc__instanceof-test.html] [_wrappers/test_conformance__misc__invalid-passed-params.html] +skip-if = (os == 'android') || (os == 'b2g') || (os == 'linux') [_wrappers/test_conformance__misc__is-object.html] [_wrappers/test_conformance__misc__null-object-behaviour.html] [_wrappers/test_conformance__misc__functions-returning-strings.html] [_wrappers/test_conformance__misc__object-deletion-behaviour.html] [_wrappers/test_conformance__misc__shader-precision-format.html] [_wrappers/test_conformance__misc__type-conversion-test.html] +skip-if = (os == 'android') || (os == 'b2g') || (os == 'linux') [_wrappers/test_conformance__misc__uninitialized-test.html] +skip-if = os == 'android' [_wrappers/test_conformance__misc__webgl-specific.html] [_wrappers/test_conformance__programs__get-active-test.html] [_wrappers/test_conformance__programs__gl-bind-attrib-location-test.html] @@ -705,7 +712,9 @@ support-files = mochi-single.html [_wrappers/test_conformance__programs__program-test.html] [_wrappers/test_conformance__reading__read-pixels-pack-alignment.html] [_wrappers/test_conformance__reading__read-pixels-test.html] +skip-if = (os == 'android') || (os == 'b2g') || (os == 'linux') [_wrappers/test_conformance__renderbuffers__framebuffer-object-attachment.html] +skip-if = os == 'android' [_wrappers/test_conformance__renderbuffers__framebuffer-test.html] [_wrappers/test_conformance__renderbuffers__renderbuffer-initialization.html] [_wrappers/test_conformance__rendering__draw-arrays-out-of-bounds.html] @@ -727,16 +736,21 @@ support-files = mochi-single.html [_wrappers/test_conformance__textures__copy-tex-image-and-sub-image-2d.html] [_wrappers/test_conformance__textures__gl-pixelstorei.html] [_wrappers/test_conformance__textures__gl-teximage.html] +skip-if = (os == 'android') || (os == 'b2g') || (os == 'linux') [_wrappers/test_conformance__textures__origin-clean-conformance.html] [_wrappers/test_conformance__textures__tex-image-and-sub-image-2d-with-array-buffer-view.html] [_wrappers/test_conformance__textures__tex-image-and-sub-image-2d-with-canvas.html] [_wrappers/test_conformance__textures__tex-image-and-sub-image-2d-with-image-data.html] [_wrappers/test_conformance__textures__tex-image-and-sub-image-2d-with-image.html] [_wrappers/test_conformance__textures__tex-image-and-sub-image-2d-with-video.html] +skip-if = (os == 'android') || (os == 'b2g') || (os == 'linux') [_wrappers/test_conformance__textures__tex-image-and-uniform-binding-bugs.html] +skip-if = (os == 'b2g') [_wrappers/test_conformance__textures__tex-image-with-format-and-type.html] +skip-if = (os == 'android') || (os == 'b2g') || (os == 'linux') [_wrappers/test_conformance__textures__tex-image-with-invalid-data.html] [_wrappers/test_conformance__textures__tex-input-validation.html] +skip-if = (os == 'android') || (os == 'b2g') || (os == 'linux') [_wrappers/test_conformance__textures__tex-sub-image-2d-bad-args.html] [_wrappers/test_conformance__textures__tex-sub-image-2d.html] [_wrappers/test_conformance__textures__texparameter-test.html] @@ -748,7 +762,9 @@ support-files = mochi-single.html [_wrappers/test_conformance__textures__texture-npot-video.html] [_wrappers/test_conformance__textures__texture-npot.html] [_wrappers/test_conformance__textures__texture-size.html] +skip-if = os == 'android' [_wrappers/test_conformance__textures__texture-size-cube-maps.html] +skip-if = os == 'android' [_wrappers/test_conformance__textures__texture-transparent-pixels-initialized.html] [_wrappers/test_conformance__typedarrays__array-buffer-crash.html] [_wrappers/test_conformance__typedarrays__array-buffer-view-crash.html] diff --git a/dom/canvas/test/webgl-conformance/_wrappers/test_always-fail.html b/dom/canvas/test/webgl-conformance/_wrappers/test_always-fail.html new file mode 100644 index 000000000000..fa21cf050e9a --- /dev/null +++ b/dom/canvas/test/webgl-conformance/_wrappers/test_always-fail.html @@ -0,0 +1,16 @@ + + + + + +Mochitest wrapper for WebGL Conformance Test Suite tests + + + + + + + + + diff --git a/dom/canvas/test/webgl-conformance/always-fail.html b/dom/canvas/test/webgl-conformance/always-fail.html new file mode 100644 index 000000000000..397c950e54e4 --- /dev/null +++ b/dom/canvas/test/webgl-conformance/always-fail.html @@ -0,0 +1,25 @@ + + + + + Always fail test + + + + + + +
+
+ + + + + + diff --git a/dom/canvas/test/webgl-conformance/generate-wrappers-and-manifest.py b/dom/canvas/test/webgl-conformance/generate-wrappers-and-manifest.py index ec190773af23..0876d343eb84 100644 --- a/dom/canvas/test/webgl-conformance/generate-wrappers-and-manifest.py +++ b/dom/canvas/test/webgl-conformance/generate-wrappers-and-manifest.py @@ -220,16 +220,41 @@ def WriteWrappers(testFilePathList): def WriteManifest(wrapperFilePathList, supportFilePathList): - manifestTestList = [] - for cur in wrapperFilePathList: - manifestTestList.append('[' + cur + ']') + errataMap = LoadErrata() + # DEFAULT_ERRATA + defaultHeader = '[DEFAULT]' + defaultErrataStr = '' + if defaultHeader in errataMap: + defaultErrataStr = '\n'.join(errataMap[defaultHeader]) + del errataMap[defaultHeader] + + # SUPPORT_FILES supportFilePathList = sorted(supportFilePathList) - supportFilesStr = '\n'.join(supportFilePathList) - manifestTestsStr = '\n'.join(manifestTestList) + # MANIFEST_TESTS + headerList = ['[' + x + ']' for x in wrapperFilePathList] + + manifestTestLineList = [] + for header in headerList: + manifestTestLineList.append(header) + + if not header in errataMap: + continue + + errataLineList = errataMap[header] + del errataMap[header] + manifestTestLineList += errataLineList + continue + + assert not errataMap, 'Errata left in map: {}'.format(str(errataMap)) + + manifestTestsStr = '\n'.join(manifestTestLineList) + + # Fill the template. templateDict = { + 'DEFAULT_ERRATA': defaultErrataStr, 'SUPPORT_FILES': supportFilesStr, 'MANIFEST_TESTS': manifestTestsStr, } @@ -245,6 +270,39 @@ WRAPPER_TEMPLATE_FILEPATH = 'mochi-wrapper.html.template' WRAPPERS_DIR = '_wrappers' MANIFEST_TEMPLATE_FILEPATH = 'mochitest.ini.template' MANIFEST_OUTPUT_FILEPATH = '_mochitest.ini' +ERRATA_FILEPATH = 'mochitest-errata.ini' +kManifestHeaderRegex = re.compile(r'\[[^\]]*?\]') + + +def LoadErrata(): + nodeMap = {} + + nodeHeader = None + nodeLineList = [] + with open(ERRATA_FILEPATH, 'r') as f: + for line in f: + line = line.rstrip() + cur = line.lstrip() + if cur.startswith('#'): + continue + + if not cur: + continue + + if not cur.startswith('['): + nodeLineList.append(line) + continue + + match = kManifestHeaderRegex.search(cur) + assert match, line + + nodeHeader = match.group() + assert not nodeHeader in nodeMap, 'Duplicate header: ' + nodeHeader + nodeLineList = [] + nodeMap[nodeHeader] = nodeLineList + continue + + return nodeMap ######################################################################## @@ -253,13 +311,21 @@ SUPPORT_DIRS = [ 'resources', ] +EXTRA_SUPPORT_FILES = [ + 'always-fail.html', +] + + def GetSupportFileList(): ret = [] for supportDir in SUPPORT_DIRS: ret += GetFilePathListForDir(supportDir) + ret += EXTRA_SUPPORT_FILES + return ret + def GetFilePathListForDir(baseDir): ret = [] for root, folders, files in os.walk(baseDir): diff --git a/dom/canvas/test/webgl-conformance/mochi-single.html b/dom/canvas/test/webgl-conformance/mochi-single.html index 9b1b88352d07..8919985ec209 100644 --- a/dom/canvas/test/webgl-conformance/mochi-single.html +++ b/dom/canvas/test/webgl-conformance/mochi-single.html @@ -57,12 +57,120 @@ if (!window.SimpleTest) { }; } +//////////////////////////////////////////////////////////////////////// +// Implement our own version of `fail-if` expected failure handling. +// `fail-if` in mochitest.ini doesn't work yet. (bug 987849) + +var OS_VERSION_WIN7 = 6.1; +var OS_VERSION_OSX_10_6 = 10.6; +var OS_VERSION_OSX_10_8 = 10.8; + +// ICS 4.0-4.0.2 was 14, 4.0.3+ was 15. +var OS_VERSION_ANDROID_ICS = 14; + +var ALWAYS_FAIL_TEST_FILEPATH = 'always-fail.html'; + +function GetExpectedTestFailSet() { + var failSet = {}; + + failSet[ALWAYS_FAIL_TEST_FILEPATH] = true; + + switch (DriverInfo.getOS()) { + case DriverInfo.OS.WINDOWS: + if (DriverInfo.getOSVersion() >= OS_VERSION_WIN7) { + // Win7 and Win8 slaves. + failSet['conformance/textures/tex-image-and-sub-image-2d-with-video.html'] = true; + failSet['conformance/textures/texture-npot-video.html'] = true; + } + break; + + case DriverInfo.OS.MAC: + if (DriverInfo.getOSVersion() == OS_VERSION_OSX_10_8) { + failSet['conformance/glsl/functions/glsl-function-smoothstep-gentype.html'] = true; + failSet['conformance/glsl/variables/gl-pointcoord.html'] = true; + failSet['conformance/limits/gl-max-texture-dimensions.html'] = true; + failSet['conformance/textures/texture-size.html'] = true; + } else if (DriverInfo.getOSVersion() == OS_VERSION_OSX_10_6) { + failSet['conformance/glsl/misc/glsl-function-nodes.html'] = true; + } + break; + + case DriverInfo.OS.LINUX: + failSet['conformance/extensions/oes-texture-float.html'] = true; + failSet['conformance/glsl/functions/glsl-function-sin.html'] = true; + failSet['conformance/misc/type-conversion-test.html'] = true; + failSet['conformance/textures/texture-mips.html'] = true; + failSet['conformance/textures/texture-size-cube-maps.html'] = true; + break; + + case DriverInfo.OS.ANDROID: + failSet['conformance/extensions/oes-texture-float.html'] = true; + + if (DriverInfo.getOSVersion() >= OS_VERSION_ANDROID_ICS) { + // Android 4.0 slaves. + failSet['conformance/extensions/oes-vertex-array-object.html'] = true; + failSet['conformance/glsl/functions/glsl-function-abs.html'] = true; + failSet['conformance/glsl/functions/glsl-function-faceforward.html'] = true; + failSet['conformance/glsl/functions/glsl-function-sign.html'] = true; + failSet['conformance/glsl/functions/glsl-function-smoothstep-float.html'] = true; + failSet['conformance/glsl/functions/glsl-function-step-float.html'] = true; + failSet['conformance/glsl/functions/glsl-function-step-gentype.html'] = true; + failSet['conformance/limits/gl-max-texture-dimensions.html'] = true; + failSet['conformance/limits/gl-min-textures.html'] = true; + failSet['conformance/rendering/draw-elements-out-of-bounds.html'] = true; + failSet['conformance/state/gl-get-calls.html'] = true; + failSet['conformance/textures/tex-image-with-format-and-type.html'] = true; + failSet['conformance/textures/tex-sub-image-2d.html'] = true; + failSet['conformance/textures/texture-mips.html'] = true; + failSet['conformance/textures/texture-npot.html'] = true; + failSet['conformance/textures/texture-size-cube-maps.html'] = true; + } else { + // Android 2.3 slaves. + failSet['conformance/glsl/functions/glsl-function-sin.html'] = true; + failSet['conformance/misc/error-reporting.html'] = true; + failSet['conformance/misc/object-deletion-behaviour.html'] = true; + failSet['conformance/programs/get-active-test.html'] = true; + failSet['conformance/textures/tex-image-and-sub-image-2d-with-video.html'] = true; + failSet['conformance/textures/texture-mips.html'] = true; + failSet['conformance/textures/texture-npot.html'] = true; + } + break; + + case DriverInfo.OS.B2G: + failSet['conformance/context/context-attributes-alpha-depth-stencil-antialias.html'] = true; + failSet['conformance/extensions/oes-texture-float.html'] = true; + failSet['conformance/glsl/functions/glsl-function-sin.html'] = true; + failSet['conformance/glsl/reserved/_webgl_function.vert.html'] = true; + failSet['conformance/glsl/reserved/webgl_function.vert.html'] = true; + failSet['conformance/misc/error-reporting.html'] = true; + failSet['conformance/misc/object-deletion-behaviour.html'] = true; + failSet['conformance/programs/get-active-test.html'] = true; + failSet['conformance/textures/tex-input-validation.html'] = true; + failSet['conformance/textures/texture-mips.html'] = true; + failSet['conformance/textures/texture-npot.html'] = true; + failSet['conformance/textures/texture-size-cube-maps.html'] = true; + failSet['conformance/textures/texture-size.html'] = true; + break; + } + + return failSet; +} + //////////////////////////////////////////////////////////////////////// // Test running and harness. var gTestPath = null; function RunTest(testPath) { + console.log('testPath: ' + testPath); + + if (testPath == ALWAYS_FAIL_TEST_FILEPATH) { + // Make it easier to respond to new test failures. + console.log('OS: ' + DriverInfo.getOS()); + console.log('OS version: ' + DriverInfo.getOSVersion()); + console.log('Driver: ' + DriverInfo.getDriver()); + } + pathElem.innerHTML = testPath; gTestPath = testPath; @@ -115,7 +223,15 @@ function OnTestComplete() { statusElem.innerHTML = 'Complete'; var passed = failureCount == 0; - ok(passed, 'Should pass: ' + gTestPath); + + var passExpected = true; + if (gTestPath in GetExpectedTestFailSet()) + passExpected = false; + + var expectedStatus = passExpected ? 'pass' : 'fail'; + var text = 'Should ' + expectedStatus + ': ' + gTestPath; + + ok(passed == passExpected, text); SimpleTest.finish(); } @@ -137,8 +253,7 @@ do { break; } - gTestPath = arg; - RunTest(gTestPath); + RunTest(arg); } while (false); diff --git a/dom/canvas/test/webgl-conformance/mochitest-errata.ini b/dom/canvas/test/webgl-conformance/mochitest-errata.ini new file mode 100644 index 000000000000..06016ee3175e --- /dev/null +++ b/dom/canvas/test/webgl-conformance/mochitest-errata.ini @@ -0,0 +1,72 @@ +# See python/mozbuild/mozbuild/mozinfo.py for incoming data. + +[DEFAULT] +# No e10s yet. +# 'B2G Desktop Linux' fails to create WebGL contexts. +skip-if = e10s || ((os == 'linux') && (buildapp == 'b2g')) + +######################################################################## +# All +#[_wrappers/test_always-fail.html] +#fail-if = 1 +# We'll be able to use `fail-if` after bug 987849. + +######################################################################## +# "tst-linux{32,64}-spot-NNN" Slaves: +# Android 2.3, B2G Emu, Linux, and Mulet. +# Android: os == 'android'. (Not enough info to separate out 2.3) +# B2G Emu: os == 'b2g'. +# Linux: os == 'linux'. +# Mulet: os == 'b2g' && buildapp == 'mulet'. +[_wrappers/test_conformance__misc__invalid-passed-params.html] +# Causes consistent *blues*: "DMError: Remote Device Error: unable to +# connect to 127.0.0.1 after 5 attempts" on 'Android 2.3 Opt'. +skip-if = (os == 'android') || (os == 'b2g') || (os == 'linux') +[_wrappers/test_conformance__reading__read-pixels-test.html] +# Causes consistent *blues*: "DMError: Remote Device Error: unable to +# connect to 127.0.0.1 after 5 attempts" on 'Android 2.3 Opt'. +# Crashes near on B2G ICS Emulator. +skip-if = (os == 'android') || (os == 'b2g') || (os == 'linux') +[_wrappers/test_conformance__textures__gl-teximage.html] +# Crashes on Android 2.3. +# Crashes near on B2G ICS Emulator. +skip-if = (os == 'android') || (os == 'b2g') || (os == 'linux') +[_wrappers/test_conformance__misc__type-conversion-test.html] +# Resets device on Android 2.3. +# Crashes on B2G ICS Emulator, desktop Linux, and Mulet Linux x64. +skip-if = (os == 'android') || (os == 'b2g') || (os == 'linux') +[_wrappers/test_conformance__textures__tex-image-and-sub-image-2d-with-video.html] +# Random fail on Android 2.3. +# Crashes on Mulet Linux x64 and B2G ICS Emulator. +skip-if = (os == 'android') || (os == 'b2g') || (os == 'linux') +[_wrappers/test_conformance__textures__tex-image-with-format-and-type.html] +# Crashes or blues on Android 2.3 +skip-if = (os == 'android') || (os == 'b2g') || (os == 'linux') +[_wrappers/test_conformance__textures__tex-input-validation.html] +# Crashes on 'Android 2.3' +# Asserts on 'B2G ICS Emulator Debug'. +skip-if = (os == 'android') || (os == 'b2g') || (os == 'linux') + +######################################################################## +# Android +[_wrappers/test_conformance__misc__uninitialized-test.html] +# Crashes on Android. +skip-if = os == 'android' +[_wrappers/test_conformance__renderbuffers__framebuffer-object-attachment.html] +# Crashes on Android. +skip-if = os == 'android' +[_wrappers/test_conformance__textures__texture-size.html] +# Crashes on Android 4.0. +skip-if = os == 'android' +[_wrappers/test_conformance__textures__texture-size-cube-maps.html] +# Crashes on Android 4.0. +skip-if = os == 'android' + +######################################################################## +# B2G +[_wrappers/test_conformance__context__context-attributes-alpha-depth-stencil-antialias.html] +# Asserts on 'B2G ICS Emulator Debug'. +skip-if = (os == 'b2g') +[_wrappers/test_conformance__textures__tex-image-and-uniform-binding-bugs.html] +# Intermittently asserts on 'B2G ICS Emulator Debug'. +skip-if = (os == 'b2g') diff --git a/dom/canvas/test/webgl-conformance/mochitest.ini.template b/dom/canvas/test/webgl-conformance/mochitest.ini.template index c6d478aba0d8..f4c74cdd7dc9 100644 --- a/dom/canvas/test/webgl-conformance/mochitest.ini.template +++ b/dom/canvas/test/webgl-conformance/mochitest.ini.template @@ -1,5 +1,6 @@ [DEFAULT] -skip-if = e10s +%%DEFAULT_ERRATA%% + support-files = mochi-single.html mochi-wrapper.css %%SUPPORT_FILES%%