From e68b9adbc85c4ab0e30ad0f53d0d178240aaf6b1 Mon Sep 17 00:00:00 2001 From: Nathan Froyd Date: Mon, 30 Oct 2017 15:45:59 -0400 Subject: [PATCH 01/32] Bug 1412935 - remove some autospider settings that moz.configure knows how to derive; r=sfink --- js/src/devtools/automation/autospider.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/js/src/devtools/automation/autospider.py b/js/src/devtools/automation/autospider.py index 68f8302d24c8..465255078837 100755 --- a/js/src/devtools/automation/autospider.py +++ b/js/src/devtools/automation/autospider.py @@ -251,16 +251,6 @@ elif platform.system() == 'Windows': ['PATH', 'INCLUDE', 'LIB', 'LIBPATH', 'CC', 'CXX', 'WINDOWSSDKDIR']) -# Compiler flags, based on word length -if word_bits == 32: - if compiler == 'clang': - env['CC'] = '{CC} -arch i386'.format(**env) - env['CXX'] = '{CXX} -arch i386'.format(**env) - elif compiler == 'gcc': - env['CC'] = '{CC} -m32'.format(**env) - env['CXX'] = '{CXX} -m32'.format(**env) - env['AR'] = 'ar' - # Configure flags, based on word length and cross-compilation if word_bits == 32: if platform.system() == 'Windows': From 8c5655f380e67482941d49f1f3fd3b51f024285b Mon Sep 17 00:00:00 2001 From: Geoff Brown Date: Mon, 30 Oct 2017 15:13:40 -0600 Subject: [PATCH 02/32] Bug 1412953 - Increase number of test chunks for Android 4.3 opt plain reftests; r=jmaher --- taskcluster/ci/test/reftest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskcluster/ci/test/reftest.yml b/taskcluster/ci/test/reftest.yml index 230e991a6eb4..a45eeffcab22 100644 --- a/taskcluster/ci/test/reftest.yml +++ b/taskcluster/ci/test/reftest.yml @@ -90,7 +90,7 @@ reftest: chunks: by-test-platform: android-4.3-arm7-api-16/debug: 48 - android.*: 16 + android.*: 24 macosx64.*/opt: 1 macosx64.*/debug: 2 windows10-64.*/opt: 1 From 2d0571a5672c273e6114d52ec1efb69d3112e919 Mon Sep 17 00:00:00 2001 From: Geoff Brown Date: Mon, 30 Oct 2017 15:13:42 -0600 Subject: [PATCH 03/32] Bug 1412526 - Only verify first 10 modified tests in test-verify; r=jmaher Trying to verify an unlimited number of tests will likely run out of time or log space, so give up once 10 tests are verified. --- testing/mozharness/scripts/desktop_unittest.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/testing/mozharness/scripts/desktop_unittest.py b/testing/mozharness/scripts/desktop_unittest.py index 6f1b4a05eb57..b77c552c4bbe 100755 --- a/testing/mozharness/scripts/desktop_unittest.py +++ b/testing/mozharness/scripts/desktop_unittest.py @@ -685,6 +685,8 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix abs_res_dir = self.query_abs_res_dir() max_verify_time = timedelta(minutes=60) + max_verify_tests = 10 + verified_tests = 0 if suites: self.info('#### Running %s suites' % suite_category) @@ -782,6 +784,15 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix # Signal verify time exceeded, to break out of suites and # suite categories loops also. return False + if verified_tests >= max_verify_tests: + # When changesets are merged between trees or many tests are + # otherwise updated at once, there probably is not enough time + # to verify all tests, and attempting to do so may cause other + # problems, such as generating too much log output. + self.info("TinderboxPrint: Too many modified tests: Not all tests " + "were verified.
") + return False + verified_tests = verified_tests + 1 final_cmd = copy.copy(cmd) final_cmd.extend(verify_args) From 02944dc81718085f85cccebc70d5ea06b6c47e4e Mon Sep 17 00:00:00 2001 From: Kevin Jones Date: Mon, 30 Oct 2017 12:52:00 -0400 Subject: [PATCH 04/32] Bug 1322485 - Implement browser.tabs.discard API. r=mixedpuppy --- browser/base/content/tabbrowser.xml | 3 + browser/components/extensions/ext-tabs.js | 15 +++++ .../components/extensions/schemas/tabs.json | 16 +++++ .../test/browser/browser-common.ini | 2 + .../test/browser/browser_ext_tabs_discard.js | 62 +++++++++++++++++++ .../test/mochitest/test_ext_all_apis.html | 1 + 6 files changed, 99 insertions(+) create mode 100644 browser/components/extensions/test/browser/browser_ext_tabs_discard.js diff --git a/browser/base/content/tabbrowser.xml b/browser/base/content/tabbrowser.xml index 974b94085b41..ef7721bd12b9 100644 --- a/browser/base/content/tabbrowser.xml +++ b/browser/base/content/tabbrowser.xml @@ -2565,6 +2565,9 @@ tab.removeAttribute("linkedpanel"); this._createLazyBrowser(tab); + + let evt = new CustomEvent("TabBrowserDiscarded", { bubbles: true }); + tab.dispatchEvent(evt); ]]> diff --git a/browser/components/extensions/ext-tabs.js b/browser/components/extensions/ext-tabs.js index 57827a4dadce..8a0a5bf9c207 100644 --- a/browser/components/extensions/ext-tabs.js +++ b/browser/components/extensions/ext-tabs.js @@ -268,6 +268,8 @@ this.tabs = class extends ExtensionAPI { } else if (event.type == "TabBrowserInserted" && !event.detail.insertedOnTabCreation) { needed.push("discarded"); + } else if (event.type == "TabBrowserDiscarded") { + needed.push("discarded"); } let tab = tabManager.getWrapper(event.originalTarget); @@ -307,6 +309,7 @@ this.tabs = class extends ExtensionAPI { windowTracker.addListener("TabPinned", listener); windowTracker.addListener("TabUnpinned", listener); windowTracker.addListener("TabBrowserInserted", listener); + windowTracker.addListener("TabBrowserDiscarded", listener); tabTracker.on("tab-isarticle", isArticleChangeListener); @@ -316,6 +319,7 @@ this.tabs = class extends ExtensionAPI { windowTracker.removeListener("TabPinned", listener); windowTracker.removeListener("TabUnpinned", listener); windowTracker.removeListener("TabBrowserInserted", listener); + windowTracker.removeListener("TabBrowserDiscarded", listener); tabTracker.off("tab-isarticle", isArticleChangeListener); }; }).api(), @@ -446,6 +450,17 @@ this.tabs = class extends ExtensionAPI { } }, + async discard(tabIds) { + if (!Array.isArray(tabIds)) { + tabIds = [tabIds]; + } + let tabs = tabIds.map(tabId => tabTracker.getTab(tabId)); + + for (let tab of tabs) { + tab.ownerGlobal.gBrowser.discardBrowser(tab.linkedBrowser); + } + }, + async update(tabId, updateProperties) { let nativeTab = getTabOrActive(tabId); diff --git a/browser/components/extensions/schemas/tabs.json b/browser/components/extensions/schemas/tabs.json index e4d2c4f46fb2..db4c16c2f2e8 100644 --- a/browser/components/extensions/schemas/tabs.json +++ b/browser/components/extensions/schemas/tabs.json @@ -879,6 +879,22 @@ } ] }, + { + "name": "discard", + "type": "function", + "description": "discards one or more tabs.", + "async": true, + "parameters": [ + { + "name": "tabIds", + "description": "The tab or list of tabs to discard.", + "choices": [ + {"type": "integer", "minimum": 0}, + {"type": "array", "items": {"type": "integer", "minimum": 0}} + ] + } + ] + }, { "name": "detectLanguage", "type": "function", diff --git a/browser/components/extensions/test/browser/browser-common.ini b/browser/components/extensions/test/browser/browser-common.ini index cb5de1cbb92d..74bd592b7147 100644 --- a/browser/components/extensions/test/browser/browser-common.ini +++ b/browser/components/extensions/test/browser/browser-common.ini @@ -138,6 +138,8 @@ skip-if = !e10s || debug || asan skip-if = os == "linux" && debug && bits == 32 # Bug 1350189 [browser_ext_tabs_create_invalid_url.js] [browser_ext_tabs_detectLanguage.js] +[browser_ext_tabs_discard.js] +skip-if = !e10s [browser_ext_tabs_discarded.js] [browser_ext_tabs_duplicate.js] [browser_ext_tabs_events.js] diff --git a/browser/components/extensions/test/browser/browser_ext_tabs_discard.js b/browser/components/extensions/test/browser/browser_ext_tabs_discard.js new file mode 100644 index 000000000000..931de88a4447 --- /dev/null +++ b/browser/components/extensions/test/browser/browser_ext_tabs_discard.js @@ -0,0 +1,62 @@ +/* -*- Mode: indent-tabs-mode: nil; js-indent-level: 2 -*- */ +/* vim: set sts=2 sw=2 et tw=80: */ +/* global gBrowser SessionStore */ +"use strict"; + +add_task(async function test_discarded() { + let extension = ExtensionTestUtils.loadExtension({ + manifest: { + "permissions": ["tabs"], + }, + + background: async function() { + let tabs = await browser.tabs.query({currentWindow: true}); + tabs.sort((tab1, tab2) => tab1.index - tab2.index); + + async function finishTest() { + try { + await browser.tabs.discard(tabs[0].id); + await browser.tabs.discard(tabs[2].id); + browser.test.succeed("attempting to discard an already discarded tab or the active tab should not throw error"); + } catch (e) { + browser.test.fail("attempting to discard an already discarded tab or the active tab should not throw error"); + } + let discardedTab = await browser.tabs.get(tabs[2].id); + browser.test.assertEq(false, discardedTab.discarded, "attempting to discard the active tab should not have succeeded"); + + await browser.test.assertRejects(browser.tabs.discard(999999999), /Invalid tab ID/, "attempt to discard invalid tabId should throw"); + await browser.test.assertRejects(browser.tabs.discard([999999999, tabs[1].id]), /Invalid tab ID/, "attempt to discard a valid and invalid tabId should throw"); + discardedTab = await browser.tabs.get(tabs[1].id); + browser.test.assertEq(false, discardedTab.discarded, "tab is still not discarded"); + + browser.test.notifyPass("test-finished"); + } + + browser.tabs.onUpdated.addListener(async function(tabId, updatedInfo) { + if ("discarded" in updatedInfo) { + browser.test.assertEq(tabId, tabs[0].id, "discarding tab triggered onUpdated"); + let discardedTab = await browser.tabs.get(tabs[0].id); + browser.test.assertEq(true, discardedTab.discarded, "discarded tab discard property"); + + await finishTest(); + } + }); + + browser.tabs.discard(tabs[0].id); + }, + }); + + BrowserTestUtils.loadURI(gBrowser.browsers[0], "http://example.com"); + await BrowserTestUtils.browserLoaded(gBrowser.browsers[0]); + let tab1 = await BrowserTestUtils.openNewForegroundTab(gBrowser, "http://example.com"); + let tab2 = await BrowserTestUtils.openNewForegroundTab(gBrowser, "http://example.com"); + + await extension.startup(); + + await extension.awaitFinish("test-finished"); + await extension.unload(); + + await BrowserTestUtils.removeTab(tab1); + await BrowserTestUtils.removeTab(tab2); +}); + diff --git a/browser/components/extensions/test/mochitest/test_ext_all_apis.html b/browser/components/extensions/test/mochitest/test_ext_all_apis.html index 7ff86efb3bbb..9bb0fedbb5a9 100644 --- a/browser/components/extensions/test/mochitest/test_ext_all_apis.html +++ b/browser/components/extensions/test/mochitest/test_ext_all_apis.html @@ -26,6 +26,7 @@ let expectedBackgroundApisTargetSpecific = [ "tabs.create", "tabs.detectLanguage", "tabs.duplicate", + "tabs.discard", "tabs.executeScript", "tabs.get", "tabs.getCurrent", From 645d6562bd6ab34027b6d19d94e6ebba9ad4323a Mon Sep 17 00:00:00 2001 From: Myk Melez Date: Mon, 30 Oct 2017 11:12:50 -0700 Subject: [PATCH 05/32] Bug 1412445 - replace custom QI impl with XPCOMUtils.generateQI call; r=rickychien --- browser/components/preferences/in-content/main.js | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/browser/components/preferences/in-content/main.js b/browser/components/preferences/in-content/main.js index 48dcfa882398..c7ccbe8772c3 100644 --- a/browser/components/preferences/in-content/main.js +++ b/browser/components/preferences/in-content/main.js @@ -1341,15 +1341,7 @@ var gMainPane = { // nsISupports - QueryInterface(aIID) { - if (aIID.equals(Ci.nsIObserver) || - aIID.equals(Ci.nsIDOMEventListener || - aIID.equals(Ci.nsISupports))) - return this; - - throw Cr.NS_ERROR_NO_INTERFACE; - }, - + QueryInterface: XPCOMUtils.generateQI([Ci.nsIObserver, Ci.nsIDOMEventListener]), // nsIObserver From 76e70509690c0a5bcb43de9ed3877acecce0ecd8 Mon Sep 17 00:00:00 2001 From: Jeff Gilbert Date: Thu, 26 Oct 2017 18:32:35 -0700 Subject: [PATCH 06/32] Bug 1411626 - Make GLContextSymbols a pure aggregate POD. - r=andi MozReview-Commit-ID: CFRJqfcdXIJ --- gfx/gl/GLContext.cpp | 5 +++-- gfx/gl/GLContextSymbols.h | 10 ++-------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/gfx/gl/GLContext.cpp b/gfx/gl/GLContext.cpp index 2e8b379a1d6c..6b3b223d5dbf 100644 --- a/gfx/gl/GLContext.cpp +++ b/gfx/gl/GLContext.cpp @@ -278,6 +278,7 @@ GLContext::GLContext(CreateContextFlags flags, const SurfaceCaps& caps, mTopError(LOCAL_GL_NO_ERROR), mDebugFlags(ChooseDebugFlags(flags)), mSharedContext(sharedContext), + mSymbols{}, mCaps(caps), mScreen(nullptr), mLockedSurface(nullptr), @@ -348,7 +349,7 @@ GLContext::InitWithPrefix(const char* prefix, bool trygl) if (!InitWithPrefixImpl(prefix, trygl)) { // If initialization fails, zero the symbols to avoid hard-to-understand bugs. - mSymbols.Zero(); + mSymbols = {}; NS_WARNING("GLContext::InitWithPrefix failed!"); return false; } @@ -2140,7 +2141,7 @@ GLContext::MarkDestroyed() NS_WARNING("MakeCurrent() failed during MarkDestroyed! Skipping GL object teardown."); } - mSymbols.Zero(); + mSymbols = {}; } #ifdef MOZ_GL_DEBUG diff --git a/gfx/gl/GLContextSymbols.h b/gfx/gl/GLContextSymbols.h index 1b9cfada6c5a..08b987d984ba 100644 --- a/gfx/gl/GLContextSymbols.h +++ b/gfx/gl/GLContextSymbols.h @@ -25,15 +25,9 @@ namespace mozilla { namespace gl { -struct GLContextSymbols +struct GLContextSymbols final { - GLContextSymbols() { - Zero(); - } - - void Zero() { - memset(this, 0, sizeof(GLContextSymbols)); - } + GLContextSymbols() = delete; // Initialize with {}. void (GLAPIENTRY * fActiveTexture)(GLenum); void (GLAPIENTRY * fAttachShader)(GLuint, GLuint); From 316ad3a2577739b7dbba1e542a6002ba3209a4da Mon Sep 17 00:00:00 2001 From: Kris Maglione Date: Sun, 29 Oct 2017 19:12:37 -0700 Subject: [PATCH 07/32] Bug 1404743: Add better diagnostic crash reasons for URLPreloader failure. r=mccr8 data-r=rweiss The crash reports for this section of code suggest that in some cases, we're ending up with a null Omnijar archive when trying to pre-load files. Since the pre-load file list is determined in the previous session, and the startup cache (where the pre-load list is stored) is cleared whenever the build ID changes, it should never be possible for the given omnijar archive to be missing here. These crashes also tend to appear in conjunction with crashes due to failures in the GetSharedGlobal function, and may have a similar cause. This change adds better diagnostic information to these crashes, and should at least tell us which omnijar archive we're failing to get, and the archive path that caused us to load that archive. It may not tell us much, but it may point to data corruption, or provide some other clues. MozReview-Commit-ID: EKq3r4eG5ii --HG-- extra : rebase_source : 745f99b8d5d7fbbf8511b62082d224899cdff947 --- js/xpconnect/loader/URLPreloader.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/js/xpconnect/loader/URLPreloader.cpp b/js/xpconnect/loader/URLPreloader.cpp index b9876d86938b..27574400ebcb 100644 --- a/js/xpconnect/loader/URLPreloader.cpp +++ b/js/xpconnect/loader/URLPreloader.cpp @@ -376,6 +376,11 @@ URLPreloader::BackgroundReadFiles() } RefPtr zip = entry->Archive(); + if (!zip) { + MOZ_CRASH_UNSAFE_PRINTF( + "Failed to get Omnijar %s archive for entry (path: \"%s\")", + entry->TypeString(), entry->mPath.get()); + } auto item = zip->GetItem(entry->mPath.get()); if (!item) { From 1a399688572b8f7fa71a9195ad2065370ee17868 Mon Sep 17 00:00:00 2001 From: Eric Rahm Date: Fri, 27 Oct 2017 13:23:48 -0700 Subject: [PATCH 08/32] Bug 1412416 - Part 1: Support arena duplication of wide strings. r=froydnj This adds support for duplicating both raw `char*` strings and raw `char16_t*` strings by making the character type generic. --HG-- extra : rebase_source : bb5245ed71161c8c785684e5a56ac894f03874ea extra : histedit_source : e94eb738a3982f0cb63a894a0cdfbdf0be2b9cad --- xpcom/ds/ArenaAllocatorExtensions.h | 8 ++++---- xpcom/tests/gtest/TestArenaAllocator.cpp | 11 ++++++++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/xpcom/ds/ArenaAllocatorExtensions.h b/xpcom/ds/ArenaAllocatorExtensions.h index 8bdf25d6f5b5..22f79ce9a75f 100644 --- a/xpcom/ds/ArenaAllocatorExtensions.h +++ b/xpcom/ds/ArenaAllocatorExtensions.h @@ -32,11 +32,11 @@ T* DuplicateString(const T* aSrc, const CheckedInt& aLen, * @param aArena The arena to allocate the string copy out of. * @return An arena allocated null-terminated string. */ -template -char* ArenaStrdup(const char* aStr, - ArenaAllocator& aArena) +template +T* ArenaStrdup(const T* aStr, + ArenaAllocator& aArena) { - return detail::DuplicateString(aStr, strlen(aStr), aArena); + return detail::DuplicateString(aStr, nsCharTraits::length(aStr), aArena); } /** diff --git a/xpcom/tests/gtest/TestArenaAllocator.cpp b/xpcom/tests/gtest/TestArenaAllocator.cpp index d80cf4fe60b9..7413d9b20207 100644 --- a/xpcom/tests/gtest/TestArenaAllocator.cpp +++ b/xpcom/tests/gtest/TestArenaAllocator.cpp @@ -284,10 +284,15 @@ TEST(ArenaAllocator, Clear) TEST(ArenaAllocator, Extensions) { ArenaAllocator<4096, 8> a; - const char* const kTestStr = "This is a test string."; - char* dup = mozilla::ArenaStrdup(kTestStr, a); - EXPECT_STREQ(dup, kTestStr); + // Test with raw strings. + const char* const kTestCStr = "This is a test string."; + char* c_dup = mozilla::ArenaStrdup(kTestCStr, a); + EXPECT_STREQ(c_dup, kTestCStr); + + const char16_t* const kTestStr = u"This is a wide test string."; + char16_t* dup = mozilla::ArenaStrdup(kTestStr, a); + EXPECT_TRUE(nsString(dup).Equals(kTestStr)); NS_NAMED_LITERAL_STRING(wideStr, "A wide string."); nsLiteralString::char_type* wide = mozilla::ArenaStrdup(wideStr, a); EXPECT_TRUE(wideStr.Equals(wide)); From b8c8c56e4653fdfbb7e5bd14e20a12cc1b800c2c Mon Sep 17 00:00:00 2001 From: Eric Rahm Date: Fri, 27 Oct 2017 13:26:47 -0700 Subject: [PATCH 09/32] Bug 1412416 - Part 2: Simplify ArenaStrdup for xpcom strings. r=froydnj Now that xpcom strings use templates we can combine their `ArenaStrdup` implementations. `nsTStringRepr` is used so as to allow handling of both `nsTLiteralString` and `nsTSubstring` types. --HG-- extra : rebase_source : e77891da589f320507b45f524a3203b3dc9f38e6 extra : histedit_source : 79d3b9a1add191563e0985f0c0e416bd29f24351 --- xpcom/ds/ArenaAllocatorExtensions.h | 20 +++----------------- xpcom/tests/gtest/TestArenaAllocator.cpp | 11 +++++++++++ 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/xpcom/ds/ArenaAllocatorExtensions.h b/xpcom/ds/ArenaAllocatorExtensions.h index 22f79ce9a75f..47a3ab368286 100644 --- a/xpcom/ds/ArenaAllocatorExtensions.h +++ b/xpcom/ds/ArenaAllocatorExtensions.h @@ -46,23 +46,9 @@ T* ArenaStrdup(const T* aStr, * @param aArena The arena to allocate the string copy out of. * @return An arena allocated null-terminated string. */ -template -nsAString::char_type* ArenaStrdup( - const nsAString& aStr, ArenaAllocator& aArena) -{ - return detail::DuplicateString(aStr.BeginReading(), aStr.Length(), aArena); -} - -/** - * Makes an arena allocated null-terminated copy of the source string. - * - * @param aSrc String to copy. - * @param aArena The arena to allocate the string copy out of. - * @return An arena allocated null-terminated string. - */ -template -nsACString::char_type* ArenaStrdup( - const nsACString& aStr, ArenaAllocator& aArena) +template +T* ArenaStrdup(const detail::nsTStringRepr& aStr, + ArenaAllocator& aArena) { return detail::DuplicateString(aStr.BeginReading(), aStr.Length(), aArena); } diff --git a/xpcom/tests/gtest/TestArenaAllocator.cpp b/xpcom/tests/gtest/TestArenaAllocator.cpp index 7413d9b20207..7f5e4effc215 100644 --- a/xpcom/tests/gtest/TestArenaAllocator.cpp +++ b/xpcom/tests/gtest/TestArenaAllocator.cpp @@ -293,6 +293,8 @@ TEST(ArenaAllocator, Extensions) const char16_t* const kTestStr = u"This is a wide test string."; char16_t* dup = mozilla::ArenaStrdup(kTestStr, a); EXPECT_TRUE(nsString(dup).Equals(kTestStr)); + + // Make sure it works with literal strings. NS_NAMED_LITERAL_STRING(wideStr, "A wide string."); nsLiteralString::char_type* wide = mozilla::ArenaStrdup(wideStr, a); EXPECT_TRUE(wideStr.Equals(wide)); @@ -300,4 +302,13 @@ TEST(ArenaAllocator, Extensions) NS_NAMED_LITERAL_CSTRING(cStr, "A c-string."); nsLiteralCString::char_type* cstr = mozilla::ArenaStrdup(cStr, a); EXPECT_TRUE(cStr.Equals(cstr)); + + // Make sure it works with normal strings. + nsAutoString x(u"testing wide"); + nsAutoString::char_type* x_copy = mozilla::ArenaStrdup(x, a); + EXPECT_TRUE(x.Equals(x_copy)); + + nsAutoCString y("testing c-string"); + nsAutoCString::char_type* y_copy = mozilla::ArenaStrdup(y, a); + EXPECT_TRUE(y.Equals(y_copy)); } From b4e47f4f0f3176eff4fc308b75e9618dcf8cf29f Mon Sep 17 00:00:00 2001 From: Andrea Marchesini Date: Mon, 30 Oct 2017 23:33:18 +0100 Subject: [PATCH 10/32] Bug 1412894 - MutableBlobStorage should return errors as soon as the temporary file writing fails, r=smaug --- dom/file/MutableBlobStorage.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dom/file/MutableBlobStorage.cpp b/dom/file/MutableBlobStorage.cpp index 0b59efdc18f9..8561bf6c9606 100644 --- a/dom/file/MutableBlobStorage.cpp +++ b/dom/file/MutableBlobStorage.cpp @@ -464,6 +464,11 @@ MutableBlobStorage::Append(const void* aData, uint32_t aLength) // If we are already in the temporaryFile mode, we have to dispatch a // runnable. if (mStorageState == eInTemporaryFile) { + // If a previous operation failed, let's return that error now. + if (NS_FAILED(mErrorResult)) { + return mErrorResult; + } + RefPtr runnable = WriteRunnable::CopyBuffer(this, aData, aLength); if (NS_WARN_IF(!runnable)) { @@ -590,6 +595,7 @@ MutableBlobStorage::TemporaryFileCreated(PRFileDesc* aFD) } mFD = aFD; + MOZ_ASSERT(NS_SUCCEEDED(mErrorResult)); // This runnable takes the ownership of mData and it will write this buffer // into the temporary file. From ab5f3e7d54061c77b72bf4640bb80edbab266eb3 Mon Sep 17 00:00:00 2001 From: Joel Maher Date: Mon, 30 Oct 2017 19:07:17 -0400 Subject: [PATCH 11/32] Bug 1411980 - annotate newer directories with bugzilla compoents. r=overholt --- testing/web-platform/moz.build | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/testing/web-platform/moz.build b/testing/web-platform/moz.build index a1625acf27ee..95c014aeace3 100644 --- a/testing/web-platform/moz.build +++ b/testing/web-platform/moz.build @@ -273,6 +273,9 @@ with Files("tests/encoding/**"): with Files("tests/encrypted-media/**"): BUG_COMPONENT = ("Core", "Audio/Video: Playback") +with Files("tests/entries-api/**"): + BUG_COMPONENT = ("Core", "DOM") + with Files("tests/eventsource/**"): BUG_COMPONENT = ("Core", "DOM") @@ -428,9 +431,8 @@ with Files("tests/orientation-sensor/**"): with Files("tests/page-visibility/**"): BUG_COMPONENT = ("Core", "DOM") -# TODO: not sure what component to use here -#with Files("tests/paint-timing/**"): -# BUG_COMPONENT = ("", "") +with Files("tests/paint-timing/**"): + BUG_COMPONENT = ("Core", "DOM") # No tests in here with Files("tests/payment-handler/**"): @@ -538,6 +540,9 @@ with Files("tests/touch-events/**"): with Files("tests/tools/**"): BUG_COMPONENT = ("Testing", "web-platform-tests") +with Files("tests/trusted-types/**"): + BUG_COMPONENT = ("Core", "DOM: Security") + with Files("tests/uievents/**"): BUG_COMPONENT = ("Core", "DOM: Events") From ede9d9fd216c93a52383a1118ec5e5c7a8eb57b2 Mon Sep 17 00:00:00 2001 From: Joel Maher Date: Mon, 30 Oct 2017 19:07:20 -0400 Subject: [PATCH 12/32] Bug 1412916 - fill in missing bugzilla_components. r=gps --- moz.build | 3 +++ testing/moz.build | 3 +++ testing/web-platform/moz.build | 3 +++ 3 files changed, 9 insertions(+) diff --git a/moz.build b/moz.build index b53f40ebb972..cb88c0fa6e54 100644 --- a/moz.build +++ b/moz.build @@ -38,6 +38,9 @@ with Files('*gradle*'): BUG_COMPONENT = ('Firefox for Android', 'Build Config & IDE Support') SCHEDULES.exclusive = ['android'] +with Files('*.json'): + BUG_COMPONENT = ('Core', 'Build Config') + with Files('**/l10n.toml'): BUG_COMPONENT = ('Core', 'Localization') FINAL = True diff --git a/testing/moz.build b/testing/moz.build index 42ef0d86cdca..058a5a520018 100644 --- a/testing/moz.build +++ b/testing/moz.build @@ -1,3 +1,6 @@ +with Files("*"): + BUG_COMPONENT = ("Testing", "General") + with Files("awsy/**"): BUG_COMPONENT = ("Testing", "AWSY") SCHEDULES.exclusive = ["awsy"] diff --git a/testing/web-platform/moz.build b/testing/web-platform/moz.build index 95c014aeace3..11fdfd5ec80d 100644 --- a/testing/web-platform/moz.build +++ b/testing/web-platform/moz.build @@ -30,6 +30,9 @@ with Files("**"): 'web-platform-tests-wdspec', ] +with Files("moz.build"): + BUG_COMPONENT = ("Testing", "web-platform-tests") + with Files("README.md"): BUG_COMPONENT = ("Testing", "web-platform-tests") From ecf5ddbf21de65bf0bbde746c7422fd576a3f921 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Mon, 30 Oct 2017 18:49:05 +0000 Subject: [PATCH 13/32] Bug 1412981 - Remove nsIStatusReporter as it is unused. r=froydnj --HG-- extra : rebase_source : 9b917d55005e90abe26a61a58368ff82bf103c3f --- xpcom/base/moz.build | 2 - xpcom/base/nsIStatusReporter.idl | 90 ------- xpcom/base/nsStatusReporterManager.cpp | 321 ------------------------- xpcom/base/nsStatusReporterManager.h | 41 ---- xpcom/build/XPCOMInit.cpp | 4 - xpcom/build/XPCOMModule.inc | 1 - xpcom/build/nsXPCOMCID.h | 5 - 7 files changed, 464 deletions(-) delete mode 100644 xpcom/base/nsIStatusReporter.idl delete mode 100644 xpcom/base/nsStatusReporterManager.cpp delete mode 100644 xpcom/base/nsStatusReporterManager.h diff --git a/xpcom/base/moz.build b/xpcom/base/moz.build index 7b8f31d0154d..9e1c83b54fb0 100644 --- a/xpcom/base/moz.build +++ b/xpcom/base/moz.build @@ -21,7 +21,6 @@ XPIDL_SOURCES += [ 'nsIMessageLoop.idl', 'nsIMutable.idl', 'nsISecurityConsoleMessage.idl', - 'nsIStatusReporter.idl', 'nsISupports.idl', 'nsIUUIDGenerator.idl', 'nsIVersionComparator.idl', @@ -160,7 +159,6 @@ UNIFIED_SOURCES += [ 'nsMessageLoop.cpp', 'NSPRLogModulesParser.cpp', 'nsSecurityConsoleMessage.cpp', - 'nsStatusReporterManager.cpp', 'nsSystemInfo.cpp', 'nsTraceRefcnt.cpp', 'nsUUIDGenerator.cpp', diff --git a/xpcom/base/nsIStatusReporter.idl b/xpcom/base/nsIStatusReporter.idl deleted file mode 100644 index 9f9245f4983a..000000000000 --- a/xpcom/base/nsIStatusReporter.idl +++ /dev/null @@ -1,90 +0,0 @@ -/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=8 sts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#include "nsISupports.idl" - -interface nsISimpleEnumerator; - -/* - * Status reporters show Firefox's service status. - */ - -[scriptable, uuid(ffcb716c-deeb-44ea-9d9d-ab25dc6980a8)] -interface nsIStatusReporter : nsISupports -{ - readonly attribute ACString name; - /* - * The name of the process containing this reporter. Each reporter initially - * has "" in this field, indicating that it applies to the current process. - */ - readonly attribute ACString process; - /* - * A human-readable status description. - */ - readonly attribute AUTF8String description; -}; - -[scriptable, uuid(fd531273-3319-4fcd-90f2-9f53876c3828)] -interface nsIStatusReporterManager : nsISupports -{ - - /* - * Return an enumerator of nsIStatusReporters that are currently registered. - */ - nsISimpleEnumerator enumerateReporters(); - - /* - * Register the given nsIStatusReporter. After a reporter is registered, - * it will be available via enumerateReporters(). The Manager service - * will hold a strong reference to the given reporter. - */ - void registerReporter(in nsIStatusReporter reporter); - - /* - * Unregister the given status reporter. - */ - void unregisterReporter(in nsIStatusReporter reporter); - - /* - * Initialize. - */ - void init(); - - /* - * Dump service status as a json file - */ - void dumpReports(); -}; - -%{C++ - -/* - * Note that this defaults 'process' to "", which is usually what's desired. - */ -#define NS_STATUS_REPORTER_IMPLEMENT(_classname, _name, _desc_Function) \ - class StatusReporter_##_classname final : public nsIStatusReporter { \ - ~StatusReporter_##_classname() {} \ - public: \ - NS_DECL_ISUPPORTS \ - NS_IMETHOD GetName(nsACString &name) override \ - { name.AssignLiteral(_name); return NS_OK; } \ - NS_IMETHOD GetProcess(nsACString &process) override \ - { process.Truncate(); return NS_OK; } \ - NS_IMETHOD GetDescription(nsACString &desc) override \ - { _desc_Function(desc); return NS_OK; } \ - }; \ - NS_IMPL_ISUPPORTS(StatusReporter_##_classname, nsIStatusReporter) - -#define NS_STATUS_REPORTER_NAME(_classname) StatusReporter_##_classname - -nsresult NS_RegisterStatusReporter(nsIStatusReporter *reporter); -nsresult NS_UnregisterStatusReporter(nsIStatusReporter *reporter); -nsresult NS_DumpStatusReporter(); - -#define NS_STATUS_REPORTER_MANAGER_CID \ -{ 0xe8eb4e7e, 0xf2cf, 0x45e5, \ -{ 0xb8, 0xa4, 0x6a, 0x0f, 0x50, 0x18, 0x84, 0x63 } } -%} diff --git a/xpcom/base/nsStatusReporterManager.cpp b/xpcom/base/nsStatusReporterManager.cpp deleted file mode 100644 index ccd0a21e92d9..000000000000 --- a/xpcom/base/nsStatusReporterManager.cpp +++ /dev/null @@ -1,321 +0,0 @@ -/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=8 sts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#include "nsStatusReporterManager.h" -#include "nsCOMPtr.h" -#include "nsDirectoryServiceDefs.h" -#include "nsArrayEnumerator.h" -#include "nsISimpleEnumerator.h" -#include "nsIFile.h" -#include "nsDumpUtils.h" -#include "nsIFileStreams.h" -#include "nsPrintfCString.h" - -#ifdef XP_WIN -#include -#define getpid _getpid -#else -#include -#endif - -#ifdef XP_UNIX -#define DO_STATUS_REPORT 1 -#endif - -#ifdef DO_STATUS_REPORT // { -namespace { - -class DumpStatusInfoToTempDirRunnable : public mozilla::Runnable -{ -public: - DumpStatusInfoToTempDirRunnable() - : mozilla::Runnable("DumpStatusInfoToTempDirRunnable") - { - } - - NS_IMETHOD Run() override - { - nsCOMPtr mgr = - do_GetService("@mozilla.org/status-reporter-manager;1"); - mgr->DumpReports(); - return NS_OK; - } -}; - -void -doStatusReport(const nsCString& aInputStr) -{ - LOG("FifoWatcher(%s) dispatching status report runnable.", aInputStr.get()); - RefPtr runnable = - new DumpStatusInfoToTempDirRunnable(); - NS_DispatchToMainThread(runnable); -} - -} //anonymous namespace -#endif // DO_STATUS_REPORT } - -static bool gStatusReportProgress = 0; -static int gNumReporters = 0; - -nsresult -getStatus(nsACString& aDesc) -{ - if (!gStatusReportProgress) { - aDesc.AssignLiteral("Init"); - } else { - aDesc.AssignLiteral("Running: There are "); - aDesc.AppendInt(gNumReporters); - aDesc.AppendLiteral(" reporters"); - } - return NS_OK; -} - -NS_STATUS_REPORTER_IMPLEMENT(StatusReporter, "StatusReporter State", getStatus) - -#define DUMP(o, s) \ - do { \ - const char* s2 = (s); \ - uint32_t dummy; \ - nsresult rvDump = (o)->Write((s2), strlen(s2), &dummy); \ - if (NS_WARN_IF(NS_FAILED(rvDump))) \ - return rvDump; \ - } while (0) - -static nsresult -DumpReport(nsIFileOutputStream* aOStream, const nsCString& aProcess, - const nsCString& aName, const nsCString& aDescription) -{ - if (aProcess.IsEmpty()) { - int pid = getpid(); - nsPrintfCString pidStr("PID %u", pid); - DUMP(aOStream, "\n {\n \"Process\": \""); - DUMP(aOStream, pidStr.get()); - } else { - DUMP(aOStream, "\n { \"Unknown Process\": \""); - } - - DUMP(aOStream, "\",\n \"Reporter name\": \""); - DUMP(aOStream, aName.get()); - - DUMP(aOStream, "\",\n \"Status Description\": [\""); - nsCString desc = aDescription; - desc.ReplaceSubstring("|", "\",\""); - DUMP(aOStream, desc.get()); - - DUMP(aOStream, "\"]\n }"); - - return NS_OK; -} - -/** - ** nsStatusReporterManager implementation - **/ - -NS_IMPL_ISUPPORTS(nsStatusReporterManager, nsIStatusReporterManager) - -nsStatusReporterManager::nsStatusReporterManager() -{ -} - -nsStatusReporterManager::~nsStatusReporterManager() -{ -} - -NS_IMETHODIMP -nsStatusReporterManager::Init() -{ - RegisterReporter(new NS_STATUS_REPORTER_NAME(StatusReporter)); - gStatusReportProgress = 1; - -#ifdef DO_STATUS_REPORT - if (FifoWatcher::MaybeCreate()) { - FifoWatcher* fw = FifoWatcher::GetSingleton(); - fw->RegisterCallback(NS_LITERAL_CSTRING("status report"), doStatusReport); - } -#endif - - return NS_OK; -} - -NS_IMETHODIMP -nsStatusReporterManager::DumpReports() -{ - static unsigned number = 1; - nsresult rv; - - nsCString filename("status-reports-"); - filename.AppendInt((uint32_t)getpid()); - filename.Append('-'); - filename.AppendInt(number++); - filename.AppendLiteral(".json"); - - // Open a file in NS_OS_TEMP_DIR for writing. - // The file is initialized as "incomplete-status-reports-pid-number.json" in the - // begining, it will be rename as "status-reports-pid-number.json" in the end. - nsCOMPtr tmpFile; - rv = nsDumpUtils::OpenTempFile(NS_LITERAL_CSTRING("incomplete-") + - filename, - getter_AddRefs(tmpFile), - NS_LITERAL_CSTRING("status-reports")); - if (NS_WARN_IF(NS_FAILED(rv))) { - return rv; - } - - nsCOMPtr ostream = - do_CreateInstance("@mozilla.org/network/file-output-stream;1"); - rv = ostream->Init(tmpFile, -1, -1, 0); - if (NS_WARN_IF(NS_FAILED(rv))) { - return rv; - } - - //Write the reports to the file - - DUMP(ostream, "{\n\"subject\":\"about:service reports\",\n"); - DUMP(ostream, "\"reporters\": [ "); - - nsCOMPtr e; - bool more, first = true; - EnumerateReporters(getter_AddRefs(e)); - while (NS_SUCCEEDED(e->HasMoreElements(&more)) && more) { - nsCOMPtr supports; - e->GetNext(getter_AddRefs(supports)); - nsCOMPtr r = do_QueryInterface(supports); - - nsCString process; - rv = r->GetProcess(process); - if (NS_WARN_IF(NS_FAILED(rv))) { - return rv; - } - - nsCString name; - rv = r->GetName(name); - if (NS_WARN_IF(NS_FAILED(rv))) { - return rv; - } - - nsCString description; - rv = r->GetDescription(description); - if (NS_WARN_IF(NS_FAILED(rv))) { - return rv; - } - - if (first) { - first = false; - } else { - DUMP(ostream, ","); - } - - rv = DumpReport(ostream, process, name, description); - if (NS_WARN_IF(NS_FAILED(rv))) { - return rv; - } - } - DUMP(ostream, "\n]\n}\n"); - - rv = ostream->Close(); - if (NS_WARN_IF(NS_FAILED(rv))) { - return rv; - } - - // Rename the status reports file - nsCOMPtr srFinalFile; - rv = NS_GetSpecialDirectory(NS_OS_TEMP_DIR, getter_AddRefs(srFinalFile)); - if (NS_WARN_IF(NS_FAILED(rv))) { - return rv; - } - -#ifdef ANDROID - rv = srFinalFile->AppendNative(NS_LITERAL_CSTRING("status-reports")); - if (NS_WARN_IF(NS_FAILED(rv))) { - return rv; - } -#endif - - rv = srFinalFile->AppendNative(filename); - if (NS_WARN_IF(NS_FAILED(rv))) { - return rv; - } - - rv = srFinalFile->CreateUnique(nsIFile::NORMAL_FILE_TYPE, 0600); - if (NS_WARN_IF(NS_FAILED(rv))) { - return rv; - } - - nsAutoString srActualFinalFilename; - rv = srFinalFile->GetLeafName(srActualFinalFilename); - if (NS_WARN_IF(NS_FAILED(rv))) { - return rv; - } - - rv = tmpFile->MoveTo(/* directory */ nullptr, srActualFinalFilename); - - if (NS_WARN_IF(NS_FAILED(rv))) { - return rv; - } - - return NS_OK; -} - -NS_IMETHODIMP -nsStatusReporterManager::EnumerateReporters(nsISimpleEnumerator** aResult) -{ - return NS_NewArrayEnumerator(aResult, mReporters); -} - -NS_IMETHODIMP -nsStatusReporterManager::RegisterReporter(nsIStatusReporter* aReporter) -{ - if (mReporters.IndexOf(aReporter) != -1) { - return NS_ERROR_FAILURE; - } - - mReporters.AppendObject(aReporter); - gNumReporters++; - return NS_OK; -} - -NS_IMETHODIMP -nsStatusReporterManager::UnregisterReporter(nsIStatusReporter* aReporter) -{ - if (!mReporters.RemoveObject(aReporter)) { - return NS_ERROR_FAILURE; - } - gNumReporters--; - return NS_OK; -} - -nsresult -NS_RegisterStatusReporter(nsIStatusReporter* aReporter) -{ - nsCOMPtr mgr = - do_GetService("@mozilla.org/status-reporter-manager;1"); - if (!mgr) { - return NS_ERROR_FAILURE; - } - return mgr->RegisterReporter(aReporter); -} - -nsresult -NS_UnregisterStatusReporter(nsIStatusReporter* aReporter) -{ - nsCOMPtr mgr = - do_GetService("@mozilla.org/status-reporter-manager;1"); - if (!mgr) { - return NS_ERROR_FAILURE; - } - return mgr->UnregisterReporter(aReporter); -} - -nsresult -NS_DumpStatusReporter() -{ - nsCOMPtr mgr = - do_GetService("@mozilla.org/status-reporter-manager;1"); - if (!mgr) { - return NS_ERROR_FAILURE; - } - return mgr->DumpReports(); -} diff --git a/xpcom/base/nsStatusReporterManager.h b/xpcom/base/nsStatusReporterManager.h deleted file mode 100644 index 84427cfcf65e..000000000000 --- a/xpcom/base/nsStatusReporterManager.h +++ /dev/null @@ -1,41 +0,0 @@ -/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=8 sts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - - -#include "nsIStatusReporter.h" -#include "nsCOMArray.h" -#include "nsString.h" - -class nsStatusReporter final : public nsIStatusReporter -{ -public: - NS_DECL_ISUPPORTS - NS_DECL_NSISTATUSREPORTER - - nsStatusReporter(nsACString& aProcess, nsACString& aDesc); - -private: - nsCString sProcess; - nsCString sName; - nsCString sDesc; - - virtual ~nsStatusReporter(); -}; - - -class nsStatusReporterManager : public nsIStatusReporterManager -{ -public: - NS_DECL_ISUPPORTS - NS_DECL_NSISTATUSREPORTERMANAGER - - nsStatusReporterManager(); - -private: - nsCOMArray mReporters; - - virtual ~nsStatusReporterManager(); -}; diff --git a/xpcom/build/XPCOMInit.cpp b/xpcom/build/XPCOMInit.cpp index 37b80189ccdc..501a6b8bc15a 100644 --- a/xpcom/build/XPCOMInit.cpp +++ b/xpcom/build/XPCOMInit.cpp @@ -104,8 +104,6 @@ extern nsresult nsStringInputStreamConstructor(nsISupports*, REFNSIID, void**); #include "nsSecurityConsoleMessage.h" #include "nsMessageLoop.h" -#include "nsStatusReporterManager.h" - #include #include "mozilla/Services.h" #include "mozilla/Omnijar.h" @@ -228,8 +226,6 @@ NS_GENERIC_FACTORY_CONSTRUCTOR_INIT(nsMemoryReporterManager, Init) NS_GENERIC_FACTORY_CONSTRUCTOR(nsMemoryInfoDumper) -NS_GENERIC_FACTORY_CONSTRUCTOR_INIT(nsStatusReporterManager, Init) - NS_GENERIC_FACTORY_CONSTRUCTOR(nsIOUtil) NS_GENERIC_FACTORY_CONSTRUCTOR(nsSecurityConsoleMessage) diff --git a/xpcom/build/XPCOMModule.inc b/xpcom/build/XPCOMModule.inc index effa02270b00..faa9c4655c29 100644 --- a/xpcom/build/XPCOMModule.inc +++ b/xpcom/build/XPCOMModule.inc @@ -75,4 +75,3 @@ COMPONENT(IOUTIL, nsIOUtilConstructor) COMPONENT(CYCLE_COLLECTOR_LOGGER, nsCycleCollectorLoggerConstructor) COMPONENT(MESSAGE_LOOP, nsMessageLoopConstructor) - COMPONENT(STATUS_REPORTER_MANAGER, nsStatusReporterManagerConstructor) diff --git a/xpcom/build/nsXPCOMCID.h b/xpcom/build/nsXPCOMCID.h index ac006de1ac7a..0a4983d62e31 100644 --- a/xpcom/build/nsXPCOMCID.h +++ b/xpcom/build/nsXPCOMCID.h @@ -76,11 +76,6 @@ */ #define NS_MEMORY_INFO_DUMPER_CONTRACTID "@mozilla.org/memory-info-dumper;1" -/** - * Status reporter service CID - */ -#define NS_STATUS_REPORTER_MANAGER_CONTRACTID "@mozilla.org/status-reporter-manager;1" - /** * Cycle collector logger contract id */ From 4194dc0b2367821e93480c65ff14ed8f966205e6 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Mon, 30 Oct 2017 17:17:49 +0000 Subject: [PATCH 14/32] Bug 1412980 - Remove nsPackageKitService as it is unused. r=froydnj --HG-- extra : rebase_source : a172bec722747d8f942a953102a4ac686f46a6a8 --- toolkit/system/gnome/moz.build | 1 - toolkit/system/gnome/nsGnomeModule.cpp | 5 - toolkit/system/gnome/nsPackageKitService.cpp | 267 ------------------- toolkit/system/gnome/nsPackageKitService.h | 26 -- xpcom/system/moz.build | 1 - xpcom/system/nsIPackageKitService.idl | 46 ---- 6 files changed, 346 deletions(-) delete mode 100644 toolkit/system/gnome/nsPackageKitService.cpp delete mode 100644 toolkit/system/gnome/nsPackageKitService.h delete mode 100644 xpcom/system/nsIPackageKitService.idl diff --git a/toolkit/system/gnome/moz.build b/toolkit/system/gnome/moz.build index 52db5828556f..7f7f18a18a09 100644 --- a/toolkit/system/gnome/moz.build +++ b/toolkit/system/gnome/moz.build @@ -21,7 +21,6 @@ if CONFIG['MOZ_ENABLE_GCONF']: SOURCES += [ 'nsGIOService.cpp', 'nsGSettingsService.cpp', - 'nsPackageKitService.cpp' ] FINAL_LIBRARY = 'xul' diff --git a/toolkit/system/gnome/nsGnomeModule.cpp b/toolkit/system/gnome/nsGnomeModule.cpp index 12af5bd701ea..d7b109b53ead 100644 --- a/toolkit/system/gnome/nsGnomeModule.cpp +++ b/toolkit/system/gnome/nsGnomeModule.cpp @@ -14,10 +14,8 @@ NS_GENERIC_FACTORY_CONSTRUCTOR_INIT(nsGConfService, Init) #endif #include "nsGIOService.h" #include "nsGSettingsService.h" -#include "nsPackageKitService.h" NS_GENERIC_FACTORY_CONSTRUCTOR(nsGIOService) NS_GENERIC_FACTORY_CONSTRUCTOR_INIT(nsGSettingsService, Init) -NS_GENERIC_FACTORY_CONSTRUCTOR_INIT(nsPackageKitService, Init) #include "nsSystemAlertsService.h" NS_GENERIC_FACTORY_CONSTRUCTOR_INIT(nsSystemAlertsService, Init) @@ -26,7 +24,6 @@ NS_DEFINE_NAMED_CID(NS_GCONFSERVICE_CID); #endif NS_DEFINE_NAMED_CID(NS_GIOSERVICE_CID); NS_DEFINE_NAMED_CID(NS_GSETTINGSSERVICE_CID); -NS_DEFINE_NAMED_CID(NS_PACKAGEKITSERVICE_CID); NS_DEFINE_NAMED_CID(NS_SYSTEMALERTSSERVICE_CID); static const mozilla::Module::CIDEntry kGnomeCIDs[] = { @@ -35,7 +32,6 @@ static const mozilla::Module::CIDEntry kGnomeCIDs[] = { #endif { &kNS_GIOSERVICE_CID, false, nullptr, nsGIOServiceConstructor }, { &kNS_GSETTINGSSERVICE_CID, false, nullptr, nsGSettingsServiceConstructor }, - { &kNS_PACKAGEKITSERVICE_CID, false, nullptr, nsPackageKitServiceConstructor }, { &kNS_SYSTEMALERTSSERVICE_CID, false, nullptr, nsSystemAlertsServiceConstructor }, { nullptr } }; @@ -46,7 +42,6 @@ static const mozilla::Module::ContractIDEntry kGnomeContracts[] = { #endif { NS_GIOSERVICE_CONTRACTID, &kNS_GIOSERVICE_CID }, { NS_GSETTINGSSERVICE_CONTRACTID, &kNS_GSETTINGSSERVICE_CID }, - { NS_PACKAGEKITSERVICE_CONTRACTID, &kNS_PACKAGEKITSERVICE_CID }, { NS_SYSTEMALERTSERVICE_CONTRACTID, &kNS_SYSTEMALERTSSERVICE_CID }, { nullptr } }; diff --git a/toolkit/system/gnome/nsPackageKitService.cpp b/toolkit/system/gnome/nsPackageKitService.cpp deleted file mode 100644 index 9c14bc4e89f6..000000000000 --- a/toolkit/system/gnome/nsPackageKitService.cpp +++ /dev/null @@ -1,267 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#include "nsArrayUtils.h" -#include "nsIObserver.h" -#include "nsIObserverService.h" -#include "nsISupportsPrimitives.h" -#include "nsPackageKitService.h" -#include "nsString.h" -#include "prlink.h" -#include "mozilla/Unused.h" -#include "mozilla/UniquePtr.h" - -#include -#include -#include - -using namespace mozilla; - -typedef struct _GAsyncResult GAsyncResult; -typedef enum { - G_BUS_TYPE_STARTER = -1, - G_BUS_TYPE_NONE = 0, - G_BUS_TYPE_SYSTEM = 1, - G_BUS_TYPE_SESSION = 2 -} GBusType; -typedef struct _GCancellable GCancellable; -typedef enum { - G_DBUS_CALL_FLAGS_NONE = 0, - G_DBUS_CALL_FLAGS_NO_AUTO_START = (1<<0) -} GDBusCallFlags; -typedef struct _GDBusInterfaceInfo GDBusInterfaceInfo; -typedef struct _GDBusProxy GDBusProxy; -typedef enum { - G_DBUS_PROXY_FLAGS_NONE = 0, - G_DBUS_PROXY_FLAGS_DO_NOT_LOAD_PROPERTIES = (1<<0), - G_DBUS_PROXY_FLAGS_DO_NOT_CONNECT_SIGNALS = (1<<1), - G_DBUS_PROXY_FLAGS_DO_NOT_AUTO_START = (1<<2), - G_DBUS_PROXY_FLAGS_GET_INVALIDATED_PROPERTIES = (1<<3) -} GDBusProxyFlags; -typedef struct _GVariant GVariant; -typedef void (*GAsyncReadyCallback) (GObject *source_object, - GAsyncResult *res, - gpointer user_data); - -#define GDBUS_FUNCTIONS \ - FUNC(g_dbus_proxy_call, void, (GDBusProxy *proxy, const gchar *method_name, GVariant *parameters, GDBusCallFlags flags, gint timeout_msec, GCancellable *cancellable, GAsyncReadyCallback callback, gpointer user_data)) \ - FUNC(g_dbus_proxy_call_finish, GVariant*, (GDBusProxy *proxy, GAsyncResult *res, GError **error)) \ - FUNC(g_dbus_proxy_new_finish, GDBusProxy*, (GAsyncResult *res, GError **error)) \ - FUNC(g_dbus_proxy_new_for_bus, void, (GBusType bus_type, GDBusProxyFlags flags, GDBusInterfaceInfo *info, const gchar *name, const gchar *object_path, const gchar *interface_name, GCancellable *cancellable, GAsyncReadyCallback callback, gpointer user_data)) \ - FUNC(g_variant_is_floating, gboolean, (GVariant *value)) \ - FUNC(g_variant_new, GVariant*, (const gchar *format_string, ...)) \ - FUNC(g_variant_unref, void, (GVariant* value)) - -#define FUNC(name, type, params) \ - typedef type (*_##name##_fn) params; \ - static _##name##_fn _##name; - -GDBUS_FUNCTIONS - -#undef FUNC - -#define g_dbus_proxy_call _g_dbus_proxy_call -#define g_dbus_proxy_call_finish _g_dbus_proxy_call_finish -#define g_dbus_proxy_new_finish _g_dbus_proxy_new_finish -#define g_dbus_proxy_new_for_bus _g_dbus_proxy_new_for_bus -#define g_variant_is_floating _g_variant_is_floating -#define g_variant_new _g_variant_new -#define g_variant_unref _g_variant_unref - -static PRLibrary *gioLib = nullptr; - -typedef void (*nsGDBusFunc)(); -struct nsGDBusDynamicFunction { - const char *functionName; - nsGDBusFunc *function; -}; - -nsresult -nsPackageKitService::Init() -{ -#define FUNC(name, type, params) { #name, (nsGDBusFunc *)&_##name }, - const nsGDBusDynamicFunction kGDBusSymbols[] = { - GDBUS_FUNCTIONS - }; -#undef FUNC - - if (!gioLib) { - gioLib = PR_LoadLibrary("libgio-2.0.so.0"); - if (!gioLib) - return NS_ERROR_FAILURE; - } - - for (auto GDBusSymbol : kGDBusSymbols) { - *GDBusSymbol.function = - PR_FindFunctionSymbol(gioLib, GDBusSymbol.functionName); - if (!*GDBusSymbol.function) { - return NS_ERROR_FAILURE; - } - } - - return NS_OK; -} - -NS_IMPL_ISUPPORTS(nsPackageKitService, nsIPackageKitService) - -nsPackageKitService::~nsPackageKitService() -{ - if (gioLib) { - PR_UnloadLibrary(gioLib); - gioLib = nullptr; - } -} - -static const char* InstallPackagesMethods[] = { - "InstallPackageNames", - "InstallMimeTypes", - "InstallFontconfigResources", - "InstallGStreamerResources" -}; - -struct InstallPackagesProxyNewData { - nsCOMPtr observer; - uint32_t method; - GVariant* parameters; -}; - -static void -InstallPackagesNotifyObserver(nsIObserver* aObserver, - gchar* aErrorMessage) -{ - if (aObserver) { - aObserver->Observe(nullptr, "packagekit-install", - aErrorMessage ? - NS_ConvertUTF8toUTF16(aErrorMessage).get() : - nullptr); - } -} - -static void -InstallPackagesProxyCallCallback(GObject *aSourceObject, - GAsyncResult *aResult, - gpointer aUserData) -{ - nsCOMPtr observer = static_cast(aUserData); - GDBusProxy* proxy = reinterpret_cast(aSourceObject); - - GError* error = nullptr; - GVariant* result = g_dbus_proxy_call_finish(proxy, aResult, &error); - if (result) { - InstallPackagesNotifyObserver(observer, nullptr); - g_variant_unref(result); - } else { - NS_ASSERTION(error, "g_dbus_proxy_call_finish should set error when it returns NULL"); - InstallPackagesNotifyObserver(observer, error->message); - g_error_free(error); - } - - g_object_unref(proxy); - Unused << observer.forget().take(); -} - -static void -InstallPackagesProxyNewCallback(GObject *aSourceObject, - GAsyncResult *aResult, - gpointer aUserData) -{ - InstallPackagesProxyNewData* userData = - static_cast(aUserData); - - NS_ASSERTION(g_variant_is_floating(userData->parameters), - "userData->parameters should be a floating reference."); - - GError* error = nullptr; - GDBusProxy* proxy = g_dbus_proxy_new_finish(aResult, &error); - - if (proxy) { - // Send the asynchronous request to install the packages - // This call will consume the floating reference userData->parameters so we - // don't need to release it explicitly. - nsIObserver* observer; - userData->observer.forget(&observer); - g_dbus_proxy_call(proxy, - InstallPackagesMethods[userData->method], - userData->parameters, - G_DBUS_CALL_FLAGS_NONE, - G_MAXINT, - nullptr, - &InstallPackagesProxyCallCallback, - static_cast(observer)); - } else { - NS_ASSERTION(error, "g_dbus_proxy_new_finish should set error when it returns NULL"); - InstallPackagesNotifyObserver(userData->observer, error->message); - g_error_free(error); - g_variant_unref(userData->parameters); - } - delete userData; -} - -NS_IMETHODIMP -nsPackageKitService::InstallPackages(uint32_t aInstallMethod, - nsIArray* aPackageArray, - nsIObserver* aObserver) -{ - NS_ENSURE_ARG(aPackageArray); - - uint32_t arrayLength; - aPackageArray->GetLength(&arrayLength); - if (arrayLength == 0 || - arrayLength == std::numeric_limits::max() || - aInstallMethod >= PK_INSTALL_METHOD_COUNT) { - return NS_ERROR_INVALID_ARG; - } - - // Create the GVariant* parameter from the list of packages. - GVariant* parameters = nullptr; - auto packages = MakeUnique(arrayLength + 1); - - nsresult rv = NS_OK; - for (uint32_t i = 0; i < arrayLength; i++) { - nsCOMPtr package = - do_QueryElementAt(aPackageArray, i); - if (!package) { - rv = NS_ERROR_FAILURE; - break; - } - nsString data; - package->GetData(data); - packages[i] = g_strdup(NS_ConvertUTF16toUTF8(data).get()); - if (!packages[i]) { - rv = NS_ERROR_OUT_OF_MEMORY; - break; - } - } - packages[arrayLength] = nullptr; - - if (NS_SUCCEEDED(rv)) { - // We create a new GVariant object to send parameters to PackageKit. - parameters = g_variant_new("(u^ass)", static_cast(0), - packages.get(), "hide-finished"); - if (!parameters) { - rv = NS_ERROR_OUT_OF_MEMORY; - } - } - for (uint32_t i = 0; i < arrayLength; i++) { - g_free(packages[i]); - } - NS_ENSURE_SUCCESS(rv, rv); - - // Send the asynchronous request to load the bus proxy - InstallPackagesProxyNewData* data = new InstallPackagesProxyNewData; - data->observer = aObserver; - data->method = aInstallMethod; - data->parameters = parameters; - g_dbus_proxy_new_for_bus(G_BUS_TYPE_SESSION, - G_DBUS_PROXY_FLAGS_NONE, - nullptr, - "org.freedesktop.PackageKit", - "/org/freedesktop/PackageKit", - "org.freedesktop.PackageKit.Modify", - nullptr, - &InstallPackagesProxyNewCallback, - static_cast(data)); - return NS_OK; -} diff --git a/toolkit/system/gnome/nsPackageKitService.h b/toolkit/system/gnome/nsPackageKitService.h deleted file mode 100644 index b14c7c51fe12..000000000000 --- a/toolkit/system/gnome/nsPackageKitService.h +++ /dev/null @@ -1,26 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#ifndef nsPackageKitService_h_ -#define nsPackageKitService_h_ - -#include "nsIPackageKitService.h" - -#define NS_PACKAGEKITSERVICE_CID \ -{0x9c95515e, 0x611d, 0x11e4, {0xb9, 0x7e, 0x60, 0xa4, 0x4c, 0x71, 0x70, 0x42}} - -class nsPackageKitService final : public nsIPackageKitService -{ -public: - NS_DECL_ISUPPORTS - NS_DECL_NSIPACKAGEKITSERVICE - - nsresult Init(); - -private: - ~nsPackageKitService(); -}; - -#endif diff --git a/xpcom/system/moz.build b/xpcom/system/moz.build index 8a4f88efe553..2be77e7f0878 100644 --- a/xpcom/system/moz.build +++ b/xpcom/system/moz.build @@ -12,7 +12,6 @@ XPIDL_SOURCES += [ 'nsIGIOService.idl', 'nsIGSettingsService.idl', 'nsIHapticFeedback.idl', - 'nsIPackageKitService.idl', 'nsIPlatformInfo.idl', 'nsIXULAppInfo.idl', 'nsIXULRuntime.idl', diff --git a/xpcom/system/nsIPackageKitService.idl b/xpcom/system/nsIPackageKitService.idl deleted file mode 100644 index 5cd3494a3b83..000000000000 --- a/xpcom/system/nsIPackageKitService.idl +++ /dev/null @@ -1,46 +0,0 @@ -/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#include "nsISupports.idl" - -interface nsIArray; -interface nsIObserver; - -[scriptable, uuid(89bb04f6-ce2a-11e3-8f4f-60a44c717042)] -interface nsIPackageKitService : nsISupports -{ - - /* PackageKit installation methods */ - /* See https://github.com/nekohayo/gnome-packagekit/blob/master/src/org.freedesktop.PackageKit.xml */ - const unsigned long PK_INSTALL_PACKAGE_NAMES = 0; - const unsigned long PK_INSTALL_MIME_TYPES = 1; - const unsigned long PK_INSTALL_FONTCONFIG_RESOURCES = 2; - const unsigned long PK_INSTALL_GSTREAMER_RESOURCES = 3; - const unsigned long PK_INSTALL_METHOD_COUNT = 4; - - /* Ask to install a list of packages via PackageKit - * @param packageKitMethod - * The PackageKit installation method - * @param packageArray - * A nonempty array of strings describing the list of packages to - * install. - * @param An object implementing nsIObserver that will be notified with - * a message of topic "packagekit-install". The message data - * contains the error returned by PackageKit if the installation - * fails and is null otherwise. - * - * This function may raise an NS_ERROR_INVALID_ARG, NS_ERROR_FAILURE or - * NS_ERROR_OUT_OF_MEMORY exception. Otherwise, the observer will be notified - * when the operation completes. - * - */ - void installPackages(in unsigned long packageKitMethod, - in nsIArray packageArray, - in nsIObserver observer); -}; - -%{C++ -#define NS_PACKAGEKITSERVICE_CONTRACTID "@mozilla.org/packagekit-service;1" -%} From 113db6d333912e0575e4ba143799320ecf60e25e Mon Sep 17 00:00:00 2001 From: Aaron Klotz Date: Mon, 30 Oct 2017 14:52:00 -0600 Subject: [PATCH 15/32] Bug 1336971: Ensure that we always re-examine the length of the top-level remote doc array to pick up any changes due to mutation; r=Jamie MozReview-Commit-ID: BLq1zzyKs9e --- accessible/windows/msaa/AccessibleWrap.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/accessible/windows/msaa/AccessibleWrap.cpp b/accessible/windows/msaa/AccessibleWrap.cpp index 38ad3bcef9ae..e024d9e5eb44 100644 --- a/accessible/windows/msaa/AccessibleWrap.cpp +++ b/accessible/windows/msaa/AccessibleWrap.cpp @@ -1554,8 +1554,10 @@ AccessibleWrap::GetRemoteIAccessibleFor(const VARIANT& aVarChild) RefPtr result; - size_t docCount = remoteDocs->Length(); - for (size_t i = 0; i < docCount; i++) { + // We intentionally leave the call to remoteDocs->Length() inside the loop + // condition because it is possible for reentry to occur in the call to + // GetProxiedAccessibleInSubtree() such that remoteDocs->Length() is mutated. + for (size_t i = 0; i < remoteDocs->Length(); i++) { DocAccessibleParent* remoteDoc = remoteDocs->ElementAt(i); uint32_t remoteDocMsaaId = WrapperFor(remoteDoc)->GetExistingID(); From 89df96b8909bd8e63a459975f54a4e89b215544e Mon Sep 17 00:00:00 2001 From: Stephen A Pohl Date: Mon, 30 Oct 2017 21:33:59 -0400 Subject: [PATCH 16/32] Bug 1412130: Fix window levels on macOS to ensure that popups hide when the application is deactivated. r=mstange --- widget/cocoa/nsCocoaWindow.mm | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/widget/cocoa/nsCocoaWindow.mm b/widget/cocoa/nsCocoaWindow.mm index b9077e6dff56..545733e6a3f4 100644 --- a/widget/cocoa/nsCocoaWindow.mm +++ b/widget/cocoa/nsCocoaWindow.mm @@ -888,8 +888,8 @@ nsCocoaWindow::Show(bool bState) if (nativeParentWindow && mPopupLevel == ePopupLevelParent) { [nativeParentWindow addChildWindow:mWindow ordered:NSWindowAbove]; - [mWindow setLevel:NSPopUpMenuWindowLevel]; } + SetPopupWindowLevel(); } else { NS_OBJC_BEGIN_TRY_ABORT_BLOCK; @@ -1377,7 +1377,6 @@ nsCocoaWindow::HideWindowChrome(bool aShouldHide) enumerator = [childWindows objectEnumerator]; while ((child = [enumerator nextObject])) { [mWindow addChildWindow:child ordered:NSWindowAbove]; - [mWindow setLevel:NSPopUpMenuWindowLevel]; } // Show the new window. @@ -2505,14 +2504,12 @@ void nsCocoaWindow::SetPopupWindowLevel() // deactivated. if (mPopupLevel == ePopupLevelFloating) { [mWindow setLevel:NSFloatingWindowLevel]; - [mWindow setHidesOnDeactivate:YES]; - } - else { + } else { // Otherwise, this is a top-level or parent popup. Parent popups always // appear just above their parent and essentially ignore the level. [mWindow setLevel:NSPopUpMenuWindowLevel]; - [mWindow setHidesOnDeactivate:NO]; } + [mWindow setHidesOnDeactivate:YES]; } void From 6115852e16f173dc10db54b670fce5f27f2777c8 Mon Sep 17 00:00:00 2001 From: Bevis Tseng Date: Thu, 19 Oct 2017 18:20:31 +0800 Subject: [PATCH 17/32] Bug 1409985 - Don't fire DOMEvent during StableState. r=smaug --- dom/base/nsContentUtils.cpp | 8 ++++ dom/base/nsContentUtils.h | 5 +++ dom/events/EventDispatcher.cpp | 4 ++ dom/media/MediaDecoder.cpp | 26 ++++++++++++- dom/media/webspeech/synth/nsSpeechTask.cpp | 44 +++++++++++++++++----- xpcom/base/CycleCollectedJSContext.h | 5 +++ 6 files changed, 81 insertions(+), 11 deletions(-) diff --git a/dom/base/nsContentUtils.cpp b/dom/base/nsContentUtils.cpp index df33074086b8..181b454ca701 100644 --- a/dom/base/nsContentUtils.cpp +++ b/dom/base/nsContentUtils.cpp @@ -5829,6 +5829,14 @@ nsContentUtils::RunInMetastableState(already_AddRefed aRunnable) CycleCollectedJSContext::Get()->RunInMetastableState(Move(aRunnable)); } +/* static */ +bool +nsContentUtils::IsInStableOrMetaStableState() +{ + MOZ_ASSERT(CycleCollectedJSContext::Get(), "Must be on a script thread!"); + return CycleCollectedJSContext::Get()->IsInStableOrMetaStableState(); +} + /* static */ nsISerialEventTarget* nsContentUtils::GetStableStateEventTarget() diff --git a/dom/base/nsContentUtils.h b/dom/base/nsContentUtils.h index e866096c1bba..82deb33f805f 100644 --- a/dom/base/nsContentUtils.h +++ b/dom/base/nsContentUtils.h @@ -1972,6 +1972,11 @@ public: */ static void RunInMetastableState(already_AddRefed aRunnable); + /** + * Returns true if we are doing StableState/MetastableState. + */ + static bool IsInStableOrMetaStableState(); + /** * Returns a nsISerialEventTarget which will run any event dispatched to it * once the event loop has reached a "stable state". Runnables dispatched to diff --git a/dom/events/EventDispatcher.cpp b/dom/events/EventDispatcher.cpp index eaa114c4d014..27702689cb04 100644 --- a/dom/events/EventDispatcher.cpp +++ b/dom/events/EventDispatcher.cpp @@ -601,6 +601,10 @@ EventDispatcher::Dispatch(nsISupports* aTarget, NS_ENSURE_TRUE(aEvent->mMessage || !aDOMEvent || aTargets, NS_ERROR_DOM_INVALID_STATE_ERR); + // Events shall not be fired while we are in stable state to prevent anything + // visible from the scripts. + MOZ_ASSERT(!nsContentUtils::IsInStableOrMetaStableState()); + #ifdef MOZ_TASK_TRACER if (MOZ_UNLIKELY(mozilla::tasktracer::IsStartLogging())) { nsAutoCString eventType; diff --git a/dom/media/MediaDecoder.cpp b/dom/media/MediaDecoder.cpp index 4c19889c912a..04b1ae4d2e3c 100644 --- a/dom/media/MediaDecoder.cpp +++ b/dom/media/MediaDecoder.cpp @@ -158,7 +158,18 @@ class MediaDecoder::BackgroundVideoDecodingPermissionObserver final : if (observerService) { observerService->AddObserver(this, "unselected-tab-hover", false); mIsRegisteredForEvent = true; - EnableEvent(); + if (nsContentUtils::IsInStableOrMetaStableState()) { + // Events shall not be fired synchronously to prevent anything visible + // from the scripts while we are in stable state. + if (nsCOMPtr doc = GetOwnerDoc()) { + doc->Dispatch(TaskCategory::Other, + NewRunnableMethod( + "MediaDecoder::BackgroundVideoDecodingPermissionObserver::EnableEvent", + this, &MediaDecoder::BackgroundVideoDecodingPermissionObserver::EnableEvent)); + } + } else { + EnableEvent(); + } } } @@ -170,7 +181,18 @@ class MediaDecoder::BackgroundVideoDecodingPermissionObserver final : mIsRegisteredForEvent = false; mDecoder->mIsBackgroundVideoDecodingAllowed = false; mDecoder->UpdateVideoDecodeMode(); - DisableEvent(); + if (nsContentUtils::IsInStableOrMetaStableState()) { + // Events shall not be fired synchronously to prevent anything visible + // from the scripts while we are in stable state. + if (nsCOMPtr doc = GetOwnerDoc()) { + doc->Dispatch(TaskCategory::Other, + NewRunnableMethod( + "MediaDecoder::BackgroundVideoDecodingPermissionObserver::DisableEvent", + this, &MediaDecoder::BackgroundVideoDecodingPermissionObserver::DisableEvent)); + } + } else { + DisableEvent(); + } } } private: diff --git a/dom/media/webspeech/synth/nsSpeechTask.cpp b/dom/media/webspeech/synth/nsSpeechTask.cpp index 4d8d70e674d5..921e280a4068 100644 --- a/dom/media/webspeech/synth/nsSpeechTask.cpp +++ b/dom/media/webspeech/synth/nsSpeechTask.cpp @@ -61,18 +61,35 @@ public: switch (event) { case MediaStreamGraphEvent::EVENT_FINISHED: { + RefPtr self = this; if (!mStarted) { mStarted = true; aGraph->DispatchToMainThreadAfterStreamStateUpdate( - NewRunnableMethod("dom::SynthStreamListener::DoNotifyStarted", - this, - &SynthStreamListener::DoNotifyStarted)); + NS_NewRunnableFunction( + "dom::SynthStreamListener::NotifyEvent", + [self] { + // "start" event will be fired in DoNotifyStarted() which is + // not allowed in stable state, so we do it asynchronously in + // next run. + NS_DispatchToMainThread(NewRunnableMethod( + "dom::SynthStreamListener::DoNotifyStarted", + self, + &SynthStreamListener::DoNotifyStarted)); + })); } aGraph->DispatchToMainThreadAfterStreamStateUpdate( - NewRunnableMethod("dom::SynthStreamListener::DoNotifyFinished", - this, - &SynthStreamListener::DoNotifyFinished)); + NS_NewRunnableFunction( + "dom::SynthStreamListener::NotifyEvent", + [self] { + // "end" event will be fired in DoNotifyFinished() which is + // not allowed in stable state, so we do it asynchronously in + // next run. + NS_DispatchToMainThread(NewRunnableMethod( + "dom::SynthStreamListener::DoNotifyFinished", + self, + &SynthStreamListener::DoNotifyFinished)); + })); } break; case MediaStreamGraphEvent::EVENT_REMOVED: @@ -89,10 +106,19 @@ public: { if (aBlocked == MediaStreamListener::UNBLOCKED && !mStarted) { mStarted = true; + RefPtr self = this; aGraph->DispatchToMainThreadAfterStreamStateUpdate( - NewRunnableMethod("dom::SynthStreamListener::DoNotifyStarted", - this, - &SynthStreamListener::DoNotifyStarted)); + NS_NewRunnableFunction( + "dom::SynthStreamListener::NotifyBlockingChanged", + [self] { + // "start" event will be fired in DoNotifyStarted() which is + // not allowed in stable state, so we do it asynchronously in + // next run. + NS_DispatchToMainThread(NewRunnableMethod( + "dom::SynthStreamListener::DoNotifyStarted", + self, + &SynthStreamListener::DoNotifyStarted)); + })); } } diff --git a/xpcom/base/CycleCollectedJSContext.h b/xpcom/base/CycleCollectedJSContext.h index e3485b581346..1f49513eb4dc 100644 --- a/xpcom/base/CycleCollectedJSContext.h +++ b/xpcom/base/CycleCollectedJSContext.h @@ -242,6 +242,11 @@ public: void DispatchMicroTaskRunnable(already_AddRefed aRunnable); + bool IsInStableOrMetaStableState() + { + return mDoingStableStates; + } + // Storage for watching rejected promises waiting for some client to // consume their rejection. // Promises in this list have been rejected in the last turn of the From 341a10bb1b0de05a0452bfe8bc6e52dc627dfc7c Mon Sep 17 00:00:00 2001 From: Lee Salzman Date: Mon, 30 Oct 2017 22:31:11 -0400 Subject: [PATCH 18/32] Bug 1412545 - AddRef user data before passing to cairo_font_face_set_user_data. r=jrmuizel MozReview-Commit-ID: FcPTjCWh9wu --- gfx/2d/ScaledFontFontconfig.cpp | 3 ++- gfx/thebes/gfxFcPlatformFontList.cpp | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/gfx/2d/ScaledFontFontconfig.cpp b/gfx/2d/ScaledFontFontconfig.cpp index 0258816ac2b0..47571e76ed12 100644 --- a/gfx/2d/ScaledFontFontconfig.cpp +++ b/gfx/2d/ScaledFontFontconfig.cpp @@ -418,16 +418,17 @@ ScaledFontFontconfig::CreateFromInstanceData(const InstanceData& aInstanceData, // Bug 1362117 - Cairo may keep the font face alive after the owning NativeFontResource // was freed. To prevent this, we must bind the NativeFontResource to the font face so that // it stays alive at least as long as the font face. + aNativeFontResource->AddRef(); if (cairo_font_face_set_user_data(font, &sNativeFontResourceKey, aNativeFontResource, ReleaseNativeFontResource) != CAIRO_STATUS_SUCCESS) { gfxWarning() << "Failed binding NativeFontResource to Cairo font face"; + aNativeFontResource->Release(); cairo_font_face_destroy(font); FcPatternDestroy(pattern); return nullptr; } - aNativeFontResource->AddRef(); } cairo_matrix_t sizeMatrix; diff --git a/gfx/thebes/gfxFcPlatformFontList.cpp b/gfx/thebes/gfxFcPlatformFontList.cpp index 6c2be694befa..5f844aac9ae9 100644 --- a/gfx/thebes/gfxFcPlatformFontList.cpp +++ b/gfx/thebes/gfxFcPlatformFontList.cpp @@ -690,15 +690,16 @@ gfxFontconfigFontEntry::CreateScaledFont(FcPattern* aRenderPattern, // so that it gets deleted whenever cairo decides NS_ASSERTION(mFTFace, "FT_Face is null when setting user data"); NS_ASSERTION(mUserFontData, "user font data is null when setting user data"); + mUserFontData.get()->AddRef(); if (cairo_font_face_set_user_data(face, &sFcFontlistUserFontDataKey, mUserFontData, ReleaseFTUserFontData) != CAIRO_STATUS_SUCCESS) { NS_WARNING("Failed binding FTUserFontData to Cairo font face"); + mUserFontData.get()->Release(); cairo_font_face_destroy(face); return nullptr; } - mUserFontData.get()->AddRef(); } cairo_scaled_font_t *scaledFont = nullptr; From a1e75219591e5ba89b1a97546d40fd203740d5d4 Mon Sep 17 00:00:00 2001 From: Jeff Muizelaar Date: Mon, 30 Oct 2017 22:35:40 -0400 Subject: [PATCH 19/32] Bug 1412565. Release the vec before deleting the item. r=lsalzman I think I caught and fixed this earlier but somehow lost the change. Oh well. --- gfx/webrender_bindings/Moz2DImageRenderer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gfx/webrender_bindings/Moz2DImageRenderer.cpp b/gfx/webrender_bindings/Moz2DImageRenderer.cpp index 32bd68beefb9..57d7f8a7ce69 100644 --- a/gfx/webrender_bindings/Moz2DImageRenderer.cpp +++ b/gfx/webrender_bindings/Moz2DImageRenderer.cpp @@ -71,8 +71,8 @@ void DeleteFontData(WrFontKey aKey) { auto i = sFontDataTable.find(aKey); if (i != sFontDataTable.end()) { - sFontDataTable.erase(i); wr_dec_ref_arc(i->second.mVec); + sFontDataTable.erase(i); } } } From e51e2066e0d50e657877a587d7cfae451d36d38f Mon Sep 17 00:00:00 2001 From: Jim Chen Date: Mon, 30 Oct 2017 22:48:58 -0400 Subject: [PATCH 20/32] Bug 1412681 - Fix NSS dialogs; r=snorp The context object for NSS dialogs apparently won't give us a window, so we have to fall back to the active window. MozReview-Commit-ID: 4cHpcE8oggL --- mobile/android/components/NSSDialogService.js | 13 +++++++++---- mobile/android/modules/Prompt.jsm | 14 +++----------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/mobile/android/components/NSSDialogService.js b/mobile/android/components/NSSDialogService.js index b87698743d7b..f71a617659cb 100644 --- a/mobile/android/components/NSSDialogService.js +++ b/mobile/android/components/NSSDialogService.js @@ -56,9 +56,14 @@ NSSDialogs.prototype = { escapedArgList.length); }, - getPrompt: function(aTitle, aText, aButtons, aWindow) { + getPrompt: function(aTitle, aText, aButtons, aCtx) { + let win = null; + try { + win = aCtx.getInterface(Ci.nsIDOMWindow); + } catch (e) { + } return new Prompt({ - window: aWindow, + window: win, title: aTitle, text: aText, buttons: aButtons, @@ -203,10 +208,10 @@ NSSDialogs.prototype = { return detailLines.join("
"); }, - viewCertDetails: function(details, window) { + viewCertDetails: function(details, ctx) { let p = this.getPrompt(this.getString("clientAuthAsk.message3"), "", - [ this.getString("nssdialogs.ok.label") ], window); + [ this.getString("nssdialogs.ok.label") ], ctx); p.addLabel({ label: details }); this.showPrompt(p); }, diff --git a/mobile/android/modules/Prompt.jsm b/mobile/android/modules/Prompt.jsm index ef32dd68826b..97b1d7c6db67 100644 --- a/mobile/android/modules/Prompt.jsm +++ b/mobile/android/modules/Prompt.jsm @@ -20,21 +20,13 @@ function log(msg) { Services.console.logStringMessage(msg); } -function getRootWindow(win) { - // Get the root xul window. - return win.QueryInterface(Ci.nsIInterfaceRequestor) - .getInterface(Ci.nsIDocShell).QueryInterface(Ci.nsIDocShellTreeItem) - .rootTreeItem.QueryInterface(Ci.nsIInterfaceRequestor) - .getInterface(Ci.nsIDOMWindow); -} - function Prompt(aOptions) { this.window = "window" in aOptions ? aOptions.window : null; this.msg = { async: true }; if (this.window) { - let window = getRootWindow(this.window); + let window = GeckoViewUtils.getChromeWindow(this.window); let tab = window && window.document.documentElement .getAttribute("windowtype") === "navigator:browser" && @@ -260,7 +252,7 @@ var DoorHanger = { }, show: function(aWindow, aMessage, aValue, aButtons, aOptions, aCategory) { - let chromeWin = getRootWindow(aWindow); + let chromeWin = GeckoViewUtils.getChromeWindow(aWindow); if (chromeWin.NativeWindow && chromeWin.NativeWindow.doorhanger) { // We're dealing with browser.js. return chromeWin.NativeWindow.doorhanger.show( @@ -299,7 +291,7 @@ var DoorHanger = { }, hide: function(aWindow, aValue) { - let chromeWin = getRootWindow(aWindow); + let chromeWin = GeckoViewUtils.getChromeWindow(aWindow); if (chromeWin.NativeWindow && chromeWin.NativeWindow.doorhanger) { // We're dealing with browser.js. return chromeWin.NativeWindow.doorhanger.hide( From bb8ebfb3ddbfbb8a5a93578b81bd2beed8d2aa75 Mon Sep 17 00:00:00 2001 From: Steve Armand Date: Sat, 28 Oct 2017 22:43:19 -0400 Subject: [PATCH 21/32] Bug 1403131 - Run linters against mozharness scripts and configs. r=rail MozReview-Commit-ID: vCOGNkXdEB --HG-- extra : rebase_source : ae13f1a7db351173a2ffebad5695706711bc0217 extra : amend_source : 17c34ccb9e603b5219ec898c5ef767ab541ca6f7 --- .../mozharness/mozilla/blob_upload.py | 25 +++--- .../mozharness/mozharness/mozilla/buildbot.py | 33 ++++--- .../mozharness/mozilla/building/buildbase.py | 63 +++++++------- .../mozharness/mozilla/building/hazards.py | 60 +++++++------ .../mozharness/mozilla/checksums.py | 6 +- .../mozharness/mozilla/l10n/locales.py | 1 - .../mozilla/l10n/multi_locale_build.py | 5 +- .../mozharness/mozharness/mozilla/mapper.py | 6 +- .../mozharness/mozharness/mozilla/merkle.py | 8 +- testing/mozharness/mozharness/mozilla/mock.py | 1 - .../mozharness/mozharness/mozilla/purge.py | 4 +- .../mozharness/mozharness/mozilla/release.py | 2 - .../mozharness/mozilla/repo_manifest.py | 3 +- .../mozharness/mozilla/repo_manipulation.py | 2 - .../mozharness/mozharness/mozilla/secrets.py | 3 - .../mozharness/mozilla/selfserve.py | 1 + .../mozilla/signed_certificate_timestamp.py | 5 +- .../mozharness/mozilla/taskcluster_helper.py | 7 +- .../mozilla/testing/codecoverage.py | 30 ++++--- .../mozharness/mozilla/testing/device.py | 17 ++-- .../mozharness/mozilla/testing/errors.py | 40 ++++++--- .../mozilla/testing/firefox_ui_tests.py | 9 +- .../mozharness/mozilla/testing/mozpool.py | 33 ++++--- .../mozharness/mozilla/testing/talos.py | 55 ++++++++---- .../mozharness/mozilla/testing/testbase.py | 87 ++++++++++--------- .../mozharness/mozilla/testing/try_tools.py | 14 +-- .../mozharness/mozilla/testing/unittest.py | 10 +-- .../mozilla/testing/verify_tools.py | 45 +++++----- .../mozharness/mozilla/updates/balrog.py | 13 +-- tools/lint/flake8.yml | 1 + 30 files changed, 334 insertions(+), 255 deletions(-) diff --git a/testing/mozharness/mozharness/mozilla/blob_upload.py b/testing/mozharness/mozharness/mozilla/blob_upload.py index 1607ddf994ad..71e9a5472157 100644 --- a/testing/mozharness/mozharness/mozilla/blob_upload.py +++ b/testing/mozharness/mozharness/mozilla/blob_upload.py @@ -12,14 +12,12 @@ from mozharness.base.script import PostScriptRun blobupload_config_options = [ [["--blob-upload-branch"], - {"dest": "blob_upload_branch", - "help": "Branch for blob server's metadata", - }], + {"dest": "blob_upload_branch", + "help": "Branch for blob server's metadata", }], [["--blob-upload-server"], - {"dest": "blob_upload_servers", - "action": "extend", - "help": "Blob servers's location", - }] + {"dest": "blob_upload_servers", + "action": "extend", + "help": "Blob servers's location", }] ] @@ -46,7 +44,7 @@ class BlobUploadMixin(VirtualenvMixin): if self.config.get('blob_upload_branch') and \ (self.config.get('blob_upload_servers') or self.config.get('default_blob_upload_servers')) and \ - self.config.get('blob_uploader_auth_file'): + self.config.get('blob_uploader_auth_file'): self.info("Blob upload gear active.") upload = [self.query_python_path(), self.query_python_path("blobberc.py")] @@ -75,7 +73,7 @@ class BlobUploadMixin(VirtualenvMixin): return blob_branch = self.config.get('blob_upload_branch') blob_servers_list = self.config.get('blob_upload_servers', - self.config.get('default_blob_upload_servers')) + self.config.get('default_blob_upload_servers')) servers = [] for server in blob_servers_list: @@ -87,11 +85,12 @@ class BlobUploadMixin(VirtualenvMixin): manifest_path = os.path.join(dirs['abs_work_dir'], 'uploaded_files.json') record_uploaded_files = ['--output-manifest', manifest_path] self.info("Files from %s are to be uploaded with <%s> branch at " - "the following location(s): %s" % (blob_dir, blob_branch, - ", ".join(["%s" % s for s in blob_servers_list]))) + "the following location(s): %s" % + (blob_dir, blob_branch, ", ".join(["%s" % s for s in blob_servers_list]))) # call blob client to upload files to server - self.run_command(upload + servers + auth + branch + dir_to_upload + record_uploaded_files) + self.run_command(upload + servers + auth + branch + + dir_to_upload + record_uploaded_files) uploaded_files = '{}' if os.path.isfile(manifest_path): @@ -100,7 +99,7 @@ class BlobUploadMixin(VirtualenvMixin): self.rmtree(manifest_path) self.set_buildbot_property(prop_name='blobber_files', - prop_value=uploaded_files, write_to_file=True) + prop_value=uploaded_files, write_to_file=True) else: self.warning("Blob upload gear skipped. Missing cmdline options.") diff --git a/testing/mozharness/mozharness/mozilla/buildbot.py b/testing/mozharness/mozharness/mozilla/buildbot.py index e17343633b27..79e8aacc4dcc 100755 --- a/testing/mozharness/mozharness/mozilla/buildbot.py +++ b/testing/mozharness/mozharness/mozilla/buildbot.py @@ -88,13 +88,17 @@ class BuildbotMixin(object): if os.path.exists(log_file): file_size = os.path.getsize(log_file) if file_size > self.config['buildbot_max_log_size']: - self.error("Log file size %d is greater than max allowed %d! Setting TBPL_FAILURE (was %s)..." % (file_size, self.config['buildbot_max_log_size'], tbpl_status)) + self.error("Log file size %d is greater than max allowed %d! Setting " + "TBPL_FAILURE (was %s)..." % + (file_size, self.config['buildbot_max_log_size'], tbpl_status)) tbpl_status = TBPL_FAILURE if not level: level = TBPL_STATUS_DICT[tbpl_status] - self.worst_buildbot_status = self.worst_level(tbpl_status, self.worst_buildbot_status, TBPL_WORST_LEVEL_TUPLE) + self.worst_buildbot_status = self.worst_level(tbpl_status, self.worst_buildbot_status, + TBPL_WORST_LEVEL_TUPLE) if self.worst_buildbot_status != tbpl_status: - self.info("Current worst status %s is worse; keeping it." % self.worst_buildbot_status) + self.info("Current worst status %s is worse; keeping it." % + self.worst_buildbot_status) self.add_summary("# TBPL %s #" % self.worst_buildbot_status, level=level) if set_return_code: self.return_code = EXIT_STATUS_DICT[self.worst_buildbot_status] @@ -137,7 +141,8 @@ class BuildbotMixin(object): self.info("Writing buildbot properties to %s" % file_name) else: if not isinstance(prop_list, (list, tuple)): - self.log("dump_buildbot_properties: Can't dump non-list prop_list %s!" % str(prop_list), level=error_level) + self.log("dump_buildbot_properties: Can't dump non-list prop_list %s!" % + str(prop_list), level=error_level) return self.info("Writing buildbot properties %s to %s" % (str(prop_list), file_name)) contents = "" @@ -146,7 +151,7 @@ class BuildbotMixin(object): return self.write_to_file(file_name, contents) def invoke_sendchange(self, downloadables=None, branch=None, - username="sendchange-unittest", sendchange_props=None): + username="sendchange-unittest", sendchange_props=None): """ Generic sendchange, currently b2g- and unittest-specific. """ c = self.config @@ -154,9 +159,11 @@ class BuildbotMixin(object): if branch is None: if c.get("debug_build"): platform = re.sub('[_-]debug', '', self.buildbot_config["properties"]["platform"]) - branch = '%s-%s-debug-unittest' % (self.buildbot_config["properties"]["branch"], platform) + branch = '%s-%s-debug-unittest' % (self.buildbot_config["properties"]["branch"], + platform) else: - branch = '%s-%s-opt-unittest' % (self.buildbot_config["properties"]["branch"], self.buildbot_config["properties"]["platform"]) + branch = '%s-%s-opt-unittest' % (self.buildbot_config["properties"]["branch"], + self.buildbot_config["properties"]["platform"]) sendchange = [ 'sendchange', '--master', c.get("sendchange_masters")[0], @@ -167,15 +174,18 @@ class BuildbotMixin(object): sendchange += ['-r', self.buildbot_config['sourcestamp']["revision"]] if len(self.buildbot_config['sourcestamp']['changes']) > 0: if self.buildbot_config['sourcestamp']['changes'][0].get('who'): - sendchange += ['--username', self.buildbot_config['sourcestamp']['changes'][0]['who']] + sendchange += ['--username', + self.buildbot_config['sourcestamp']['changes'][0]['who']] if self.buildbot_config['sourcestamp']['changes'][0].get('comments'): - sendchange += ['--comments', self.buildbot_config['sourcestamp']['changes'][0]['comments'].encode('ascii', 'ignore')] + sendchange += ['--comments', self.buildbot_config['sourcestamp'] + ['changes'][0]['comments'].encode('ascii', 'ignore')] if sendchange_props: for key, value in sendchange_props.iteritems(): sendchange.extend(['--property', '%s:%s' % (key, value)]) else: if self.buildbot_config["properties"].get("builduid"): - sendchange += ['--property', "builduid:%s" % self.buildbot_config["properties"]["builduid"]] + sendchange += ['--property', "builduid:%s" % + self.buildbot_config["properties"]["builduid"]] sendchange += [ '--property', "buildid:%s" % self.query_buildid(), '--property', 'pgo_build:False', @@ -186,7 +196,8 @@ class BuildbotMixin(object): retcode = self.run_command(buildbot + sendchange) if retcode != 0: - self.info("The sendchange failed but we don't want to turn the build orange: %s" % retcode) + self.info("The sendchange failed but we don't want to turn the build orange: %s" % + retcode) def query_build_name(self): build_name = self.config.get('platform') diff --git a/testing/mozharness/mozharness/mozilla/building/buildbase.py b/testing/mozharness/mozharness/mozilla/building/buildbase.py index abb8e0d41e14..f55b310bff5a 100755 --- a/testing/mozharness/mozharness/mozilla/building/buildbase.py +++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py @@ -34,7 +34,6 @@ from mozharness.mozilla.buildbot import ( BuildbotMixin, EXIT_STATUS_DICT, TBPL_STATUS_DICT, - TBPL_EXCEPTION, TBPL_FAILURE, TBPL_RETRY, TBPL_WARNING, @@ -73,7 +72,7 @@ Skipping run_tooltool...', ERROR_MSGS.update(MOCK_ERROR_MSGS) -### Output Parsers +# Output Parsers TBPL_UPLOAD_ERRORS = [ { @@ -141,7 +140,8 @@ class MakeUploadOutputParser(OutputParser): self.info("Skipping wrong packageUrl: %s" % m) else: if 'completeMarUrl' in self.matches: - self.fatal("Found multiple package URLs. Please update buildbase.py") + self.fatal("Found multiple package URLs. " + "Please update buildbase.py") self.info("Using package as mar file: %s" % m) self.matches['completeMarUrl'] = m u, self.package_filename = os.path.split(m) @@ -155,7 +155,8 @@ class MakeUploadOutputParser(OutputParser): if m: self.matches['completeMarHash'] = m.group(1) self.matches['completeMarSize'] = m.group(2) - self.info("Using package as mar file and found package hash=%s size=%s" % (m.group(1), m.group(2))) + self.info("Using package as mar file and found package hash=%s size=%s" % + (m.group(1), m.group(2))) # now let's check for retry errors which will give log levels: # tbpl status as RETRY and mozharness status as WARNING @@ -259,21 +260,20 @@ class BuildingConfig(BaseConfig): # not matter. ie: you can supply --branch before --build-pool # or vice versa and the hierarchy will not be different - #### The order from highest precedence to lowest is: - ## There can only be one of these... + # The order from highest precedence to lowest is: + # There can only be one of these... # 1) build_pool: this can be either staging, pre-prod, and prod cfgs # 2) branch: eg: mozilla-central, cedar, cypress, etc # 3) build_variant: these could be known like asan and debug # or a custom config - ## - ## There can be many of these + # + # There can be many of these: # 4) all other configs: these are any configs that are passed with # --cfg and --opt-cfg. There order is kept in # which they were passed on the cmd line. This # behaviour is maintains what happens by default # in mozharness - ## - #### + # # so, let's first assign the configs that hold a known position of # importance (1 through 3) @@ -360,7 +360,8 @@ class BuildOptionParser(object): 'code-coverage': 'builds/releng_sub_%s_configs/%s_code_coverage.py', 'source': 'builds/releng_sub_%s_configs/%s_source.py', 'noopt-debug': 'builds/releng_sub_%s_configs/%s_noopt_debug.py', - 'api-16-gradle-dependencies': 'builds/releng_sub_%s_configs/%s_api_16_gradle_dependencies.py', + 'api-16-gradle-dependencies': + 'builds/releng_sub_%s_configs/%s_api_16_gradle_dependencies.py', 'api-16': 'builds/releng_sub_%s_configs/%s_api_16.py', 'api-16-old-id': 'builds/releng_sub_%s_configs/%s_api_16_old_id.py', 'api-16-artifact': 'builds/releng_sub_%s_configs/%s_api_16_artifact.py', @@ -379,7 +380,7 @@ class BuildOptionParser(object): 'android-checkstyle': 'builds/releng_sub_%s_configs/%s_checkstyle.py', 'android-lint': 'builds/releng_sub_%s_configs/%s_lint.py', 'android-findbugs': 'builds/releng_sub_%s_configs/%s_findbugs.py', - 'valgrind' : 'builds/releng_sub_%s_configs/%s_valgrind.py', + 'valgrind': 'builds/releng_sub_%s_configs/%s_valgrind.py', 'artifact': 'builds/releng_sub_%s_configs/%s_artifact.py', 'debug-artifact': 'builds/releng_sub_%s_configs/%s_debug_artifact.py', 'devedition': 'builds/releng_sub_%s_configs/%s_devedition.py', @@ -761,8 +762,7 @@ or run without that action (ie: --no-{action})" # dirs['abs_obj_dir'] can be different from env['MOZ_OBJDIR'] on # mac, and that confuses mach. del env['MOZ_OBJDIR'] - return self.get_output_from_command_m(cmd, - cwd=dirs['abs_obj_dir'], env=env) + return self.get_output_from_command_m(cmd, cwd=dirs['abs_obj_dir'], env=env) else: return None @@ -1165,7 +1165,7 @@ or run without that action (ie: --no-{action})" if 'revision' in self.buildbot_properties: revision = self.buildbot_properties['revision'] elif (self.buildbot_config and - self.buildbot_config.get('sourcestamp', {}).get('revision')): + self.buildbot_config.get('sourcestamp', {}).get('revision')): revision = self.buildbot_config['sourcestamp']['revision'] elif self.buildbot_config and self.buildbot_config.get('revision'): revision = self.buildbot_config['revision'] @@ -1331,7 +1331,7 @@ or run without that action (ie: --no-{action})" 'buildid from application.ini: "%s". buildid from buildbot ' 'properties: "%s"' % (app_ini_buildid, buildbot_buildid) ) - if app_ini_buildid == buildbot_buildid != None: + if (app_ini_buildid == buildbot_buildid) is not None: self.info('buildids match.') else: self.error( @@ -1409,7 +1409,7 @@ or run without that action (ie: --no-{action})" taskid = self.buildbot_config['properties'].get('upload_to_task_id') tc = Taskcluster( branch=self.branch, - rank=pushinfo.pushdate, # Use pushdate as the rank + rank=pushinfo.pushdate, # Use pushdate as the rank client_id=self.client_id, access_token=self.access_token, log_obj=self.log_obj, @@ -1480,7 +1480,8 @@ or run without that action (ie: --no-{action})" # which means we should have uploadFiles. files = self.query_buildbot_property('uploadFiles') or [] if not files: - self.warning('No files from the build system to upload to S3: uploadFiles property is missing or empty.') + self.warning('No files from the build system to upload to S3: uploadFiles' + 'property is missing or empty.') packageName = self.query_buildbot_property('packageFilename') self.info('packageFilename is: %s' % packageName) @@ -1506,7 +1507,7 @@ or run without that action (ie: --no-{action})" property_conditions = [ # key: property name, value: condition ('symbolsUrl', lambda m: m.endswith('crashreporter-symbols.zip') or - m.endswith('crashreporter-symbols-full.zip')), + m.endswith('crashreporter-symbols-full.zip')), ('testsUrl', lambda m: m.endswith(('tests.tar.bz2', 'tests.zip'))), ('robocopApkUrl', lambda m: m.endswith('apk') and 'robocop' in m), ('jsshellUrl', lambda m: 'jsshell-' in m and m.endswith('.zip')), @@ -1523,7 +1524,8 @@ or run without that action (ie: --no-{action})" ] # Also upload our mozharness log files - files.extend([os.path.join(self.log_obj.abs_log_dir, x) for x in self.log_obj.log_files.values()]) + files.extend([os.path.join(self.log_obj.abs_log_dir, x) + for x in self.log_obj.log_files.values()]) # Also upload our buildprops.json file. files.extend([os.path.join(dirs['base_work_dir'], 'buildprops.json')]) @@ -1629,8 +1631,8 @@ or run without that action (ie: --no-{action})" os.path.join(dirs['abs_work_dir'], 'buildprops.json')) if 'MOZILLABUILD' in os.environ: - # We found many issues with intermittent build failures when not invoking mach via bash. - # See bug 1364651 before considering changing. + # We found many issues with intermittent build failures when not invoking + # mach via bash. See bug 1364651 before considering changing. mach = [ os.path.join(os.environ['MOZILLABUILD'], 'msys', 'bin', 'bash.exe'), os.path.join(dirs['abs_src_dir'], 'mach') @@ -1715,7 +1717,8 @@ or run without that action (ie: --no-{action})" cwd=objdir, ) if not package_filename: - self.fatal("Unable to determine the package filename for the multi-l10n build. Was trying to run: %s" % package_cmd) + self.fatal("Unable to determine the package filename for the multi-l10n build." + "Was trying to run: %s" % package_cmd) self.info('Multi-l10n package filename is: %s' % package_filename) @@ -1918,7 +1921,6 @@ or run without that action (ie: --no-{action})" return data - def _load_sccache_stats(self): stats_file = os.path.join( self.query_abs_dirs()['abs_obj_dir'], 'sccache-stats.json' @@ -2034,7 +2036,7 @@ or run without that action (ie: --no-{action})" return alert - if installer.endswith('.apk'): # Android + if installer.endswith('.apk'): # Android yield filter_alert({ "name": "installer size", "value": installer_size, @@ -2185,9 +2187,9 @@ or run without that action (ie: --no-{action})" build_type, 'talos') self.invoke_sendchange(downloadables=[installer_url], - branch=talos_branch, - username='sendchange', - sendchange_props=sendchange_props) + branch=talos_branch, + username='sendchange', + sendchange_props=sendchange_props) elif test_type == 'unittest': # do unittest sendchange if c.get('debug_build'): @@ -2237,11 +2239,10 @@ or run without that action (ie: --no-{action})" if self.config.get('taskcluster_nightly'): env = self.query_mach_build_env(multiLocale=False) props_path = os.path.join(env["UPLOAD_PATH"], - 'balrog_props.json') + 'balrog_props.json') self.generate_balrog_props(props_path) return - def valgrind_test(self): '''Execute mach's valgrind-test for memory leaks''' env = self.query_build_env() @@ -2260,8 +2261,6 @@ or run without that action (ie: --no-{action})" self.fatal("'mach valgrind-test' did not run successfully. Please check " "log for errors.") - - def _post_fatal(self, message=None, exit_code=None): if not self.return_code: # only overwrite return_code if it's 0 self.error('setting return code to 2 because fatal was called') diff --git a/testing/mozharness/mozharness/mozilla/building/hazards.py b/testing/mozharness/mozharness/mozilla/building/hazards.py index 6de235f8929b..ece46edd2a30 100644 --- a/testing/mozharness/mozharness/mozilla/building/hazards.py +++ b/testing/mozharness/mozharness/mozilla/building/hazards.py @@ -17,6 +17,7 @@ class HazardError(Exception): def splitlines(self): return str(self).splitlines() + class HazardAnalysis(object): def clobber_shell(self, builder): """Clobber the specially-built JS shell used to run the analysis""" @@ -32,9 +33,9 @@ class HazardAnalysis(object): js_src_dir = os.path.join(dirs['gecko_src'], 'js', 'src') rc = builder.run_command(['autoconf-2.13'], - cwd=js_src_dir, - env=builder.env, - error_list=MakefileErrorList) + cwd=js_src_dir, + env=builder.env, + error_list=MakefileErrorList) if rc != 0: rc = builder.run_command(['autoconf2.13'], cwd=js_src_dir, @@ -44,14 +45,14 @@ class HazardAnalysis(object): raise HazardError("autoconf failed, can't continue.") rc = builder.run_command([os.path.join(js_src_dir, 'configure'), - '--enable-optimize', - '--disable-debug', - '--enable-ctypes', - '--with-system-nspr', - '--without-intl-api'], - cwd=dirs['shell_objdir'], - env=builder.env, - error_list=MakefileErrorList) + '--enable-optimize', + '--disable-debug', + '--enable-ctypes', + '--with-system-nspr', + '--without-intl-api'], + cwd=dirs['shell_objdir'], + env=builder.env, + error_list=MakefileErrorList) if rc != 0: raise HazardError("Configure failed, can't continue.") @@ -60,9 +61,9 @@ class HazardAnalysis(object): dirs = builder.query_abs_dirs() rc = builder.run_command(['make', '-j', str(builder.config.get('concurrency', 4)), '-s'], - cwd=dirs['shell_objdir'], - env=builder.env, - error_list=MakefileErrorList) + cwd=dirs['shell_objdir'], + env=builder.env, + error_list=MakefileErrorList) if rc != 0: raise HazardError("Build failed, can't continue.") @@ -109,9 +110,8 @@ jobs = 4 build_script = builder.config['build_command'] builder.copyfile(os.path.join(dirs['mozharness_scriptdir'], - os.path.join('spidermonkey', build_script)), - os.path.join(analysis_dir, build_script), - copystat=True) + os.path.join('spidermonkey', build_script)), + os.path.join(analysis_dir, build_script), copystat=True) def run(self, builder, env, error_list): """Execute the analysis, which consists of building all analyzed @@ -132,9 +132,9 @@ jobs = 4 "--buildcommand", build_script, ] retval = builder.run_command(cmd, - cwd=analysis_dir, - env=env, - error_list=error_list) + cwd=analysis_dir, + env=env, + error_list=error_list) if retval != 0: raise HazardError("failed to build") @@ -171,7 +171,8 @@ jobs = 4 long_desc=long, compress=False, # blobber will compress upload_dir=upload_dir) - print("== Hazards (temporarily inline here, beware weirdly interleaved output, see bug 1211402) ==") + print("== Hazards (temporarily inline here, beware weirdly interleaved " + "output, see bug 1211402) ==") print(file(os.path.join(analysis_dir, "hazards.txt")).read()) def upload_results(self, builder): @@ -186,7 +187,8 @@ jobs = 4 dirs = builder.query_abs_dirs() analysis_dir = dirs['abs_analysis_dir'] - analysis_scriptdir = os.path.join(dirs['gecko_src'], 'js', 'src', 'devtools', 'rootAnalysis') + analysis_scriptdir = os.path.join(dirs['gecko_src'], 'js', 'src', + 'devtools', 'rootAnalysis') expect_file = os.path.join(analysis_scriptdir, builder.config['expect_file']) expect = builder.read_from_file(expect_file) if expect is None: @@ -216,12 +218,13 @@ jobs = 4 if expect_hazards is not None and expect_hazards != num_hazards: if expect_hazards < num_hazards: - builder.warning("TEST-UNEXPECTED-FAIL %d more hazards than expected (expected %d, saw %d)" % - (num_hazards - expect_hazards, expect_hazards, num_hazards)) + builder.warning("TEST-UNEXPECTED-FAIL %d more hazards than expected " + "(expected %d, saw %d)" % + (num_hazards - expect_hazards, expect_hazards, num_hazards)) builder.buildbot_status(TBPL_WARNING) else: builder.info("%d fewer hazards than expected! (expected %d, saw %d)" % - (expect_hazards - num_hazards, expect_hazards, num_hazards)) + (expect_hazards - num_hazards, expect_hazards, num_hazards)) expect_refs = data.get('expect-refs') if expect_refs is None: @@ -231,11 +234,12 @@ jobs = 4 if expect_refs is not None and expect_refs != num_refs: if expect_refs < num_refs: - builder.warning("TEST-UNEXPECTED-FAIL %d more unsafe refs than expected (expected %d, saw %d)" % - (num_refs - expect_refs, expect_refs, num_refs)) + builder.warning("TEST-UNEXPECTED-FAIL %d more unsafe refs than expected " + "(expected %d, saw %d)" % + (num_refs - expect_refs, expect_refs, num_refs)) builder.buildbot_status(TBPL_WARNING) else: builder.info("%d fewer unsafe refs than expected! (expected %d, saw %d)" % - (expect_refs - num_refs, expect_refs, num_refs)) + (expect_refs - num_refs, expect_refs, num_refs)) builder.info("TinderboxPrint: " + ", ".join(status)) diff --git a/testing/mozharness/mozharness/mozilla/checksums.py b/testing/mozharness/mozharness/mozilla/checksums.py index 6b899737567c..718db9d19a5e 100644 --- a/testing/mozharness/mozharness/mozilla/checksums.py +++ b/testing/mozharness/mozharness/mozilla/checksums.py @@ -12,10 +12,12 @@ def parse_checksums_file(checksums): # If the file already exists, make sure that the size matches the # previous entry. elif fileInfo[file_]['size'] != size: - raise ValueError("Found different sizes for same file %s (%s and %s)" % (file_, fileInfo[file_]['size'], size)) + raise ValueError("Found different sizes for same file %s (%s and %s)" % + (file_, fileInfo[file_]['size'], size)) # Same goes for the hash. elif type_ in fileInfo[file_]['hashes'] and fileInfo[file_]['hashes'][type_] != hash_: - raise ValueError("Found different %s hashes for same file %s (%s and %s)" % (type_, file_, fileInfo[file_]['hashes'][type_], hash_)) + raise ValueError("Found different %s hashes for same file %s (%s and %s)" % + (type_, file_, fileInfo[file_]['hashes'][type_], hash_)) fileInfo[file_]['size'] = size fileInfo[file_]['hashes'][type_] = hash_ return fileInfo diff --git a/testing/mozharness/mozharness/mozilla/l10n/locales.py b/testing/mozharness/mozharness/mozilla/l10n/locales.py index b06084309744..dc94374a6bc5 100755 --- a/testing/mozharness/mozharness/mozilla/l10n/locales.py +++ b/testing/mozharness/mozharness/mozilla/l10n/locales.py @@ -8,7 +8,6 @@ """ import os -from urlparse import urljoin import pprint import sys from copy import deepcopy diff --git a/testing/mozharness/mozharness/mozilla/l10n/multi_locale_build.py b/testing/mozharness/mozharness/mozilla/l10n/multi_locale_build.py index aa476ce7941f..34c974f6a615 100755 --- a/testing/mozharness/mozharness/mozilla/l10n/multi_locale_build.py +++ b/testing/mozharness/mozharness/mozilla/l10n/multi_locale_build.py @@ -135,7 +135,6 @@ class MultiLocaleBuild(LocalesMixin, MercurialScript): self.fatal("Erroring out after the build failed.") def add_locales(self): - c = self.config dirs = self.query_abs_dirs() locales = self.query_locales() @@ -215,7 +214,8 @@ class MultiLocaleBuild(LocalesMixin, MercurialScript): rsync = self.query_exe('rsync') backup_dir = '%s-bak' % dirs['abs_objdir'] if not os.path.isdir(dirs['abs_objdir']) or not os.path.isdir(backup_dir): - self.warning("Both %s and %s need to exist to restore the objdir! Skipping..." % (dirs['abs_objdir'], backup_dir)) + self.warning("Both %s and %s need to exist to restore the objdir! Skipping..." % + (dirs['abs_objdir'], backup_dir)) return self.run_command([rsync, '-a', '--delete', '--partial', '%s/' % backup_dir, @@ -233,6 +233,7 @@ class MultiLocaleBuild(LocalesMixin, MercurialScript): """ return self.run_command(**kwargs) + # __main__ {{{1 if __name__ == '__main__': pass diff --git a/testing/mozharness/mozharness/mozilla/mapper.py b/testing/mozharness/mozharness/mozilla/mapper.py index c5a2d48959e1..855e077a1164 100644 --- a/testing/mozharness/mozharness/mozilla/mapper.py +++ b/testing/mozharness/mozharness/mozilla/mapper.py @@ -51,9 +51,11 @@ class MapperMixin: j = json.loads(r.readline()) if j['%s_rev' % vcs] is None: if require_answer: - raise Exception("Mapper returned a revision of None; maybe it needs more time.") + raise Exception("Mapper returned a revision of None; " + "maybe it needs more time.") else: - self.warning("Mapper returned a revision of None. Accepting because require_answer is False.") + self.warning("Mapper returned a revision of None. " + "Accepting because require_answer is False.") return j['%s_rev' % vcs] except Exception, err: self.warning('Error: %s' % str(err)) diff --git a/testing/mozharness/mozharness/mozilla/merkle.py b/testing/mozharness/mozharness/mozilla/merkle.py index ed53f8cd798c..0ff9d6cf5003 100644 --- a/testing/mozharness/mozharness/mozilla/merkle.py +++ b/testing/mozharness/mozharness/mozilla/merkle.py @@ -2,18 +2,22 @@ import struct + def _round2(n): k = 1 while k < n: k <<= 1 return k >> 1 + def _leaf_hash(hash_fn, leaf): return hash_fn(b'\x00' + leaf).digest() + def _pair_hash(hash_fn, left, right): return hash_fn(b'\x01' + left + right).digest() + class InclusionProof: """ Represents a Merkle inclusion proof for purposes of serialization, @@ -56,7 +60,7 @@ class InclusionProof: raise Exception('Inclusion proof too short for log ID header') log_id_len, = struct.unpack('B', serialized[start:start+read]) start += read - start += log_id_len # Ignore the log ID itself + start += log_id_len # Ignore the log ID itself read = 8 + 8 + 2 if len(serialized) < start + read: @@ -118,10 +122,10 @@ class InclusionProof: return node - def verify(self, hash_fn, leaf, leaf_index, tree_size, tree_head): return self._expected_head(hash_fn, leaf, leaf_index, tree_size) == tree_head + class MerkleTree: """ Implements a Merkle tree on a set of data items following the diff --git a/testing/mozharness/mozharness/mozilla/mock.py b/testing/mozharness/mozharness/mozilla/mock.py index f8587c0d66a9..35f45ddb6202 100644 --- a/testing/mozharness/mozharness/mozilla/mock.py +++ b/testing/mozharness/mozharness/mozilla/mock.py @@ -18,7 +18,6 @@ Nothing to remove.' } - # MockMixin {{{1 class MockMixin(object): """Provides methods to setup and interact with mock environments. diff --git a/testing/mozharness/mozharness/mozilla/purge.py b/testing/mozharness/mozharness/mozilla/purge.py index ca80bd4ff81b..84db900fb3c6 100644 --- a/testing/mozharness/mozharness/mozilla/purge.py +++ b/testing/mozharness/mozharness/mozilla/purge.py @@ -68,8 +68,8 @@ class PurgeMixin(object): }] retval = self.retry(self.run_command, attempts=3, good_statuses=(0,), args=[cmd], - kwargs={'cwd':os.path.dirname(dirs['base_work_dir']), - 'error_list':error_list}) + kwargs={'cwd': os.path.dirname(dirs['base_work_dir']), + 'error_list': error_list}) if retval != 0: self.fatal("failed to clobber build", exit_code=2) diff --git a/testing/mozharness/mozharness/mozilla/release.py b/testing/mozharness/mozharness/mozilla/release.py index 52a84cdba31e..9463f7ad4458 100755 --- a/testing/mozharness/mozharness/mozilla/release.py +++ b/testing/mozharness/mozharness/mozilla/release.py @@ -68,5 +68,3 @@ def get_previous_version(version, partial_versions): composed = sorted([(v, StrictVersion(v)) for v in partial_versions if v != version], key=lambda x: x[1], reverse=True) return composed[0][0] - - diff --git a/testing/mozharness/mozharness/mozilla/repo_manifest.py b/testing/mozharness/mozharness/mozilla/repo_manifest.py index 2ffb34fe9707..a5b7915f83a0 100644 --- a/testing/mozharness/mozharness/mozilla/repo_manifest.py +++ b/testing/mozharness/mozharness/mozilla/repo_manifest.py @@ -24,7 +24,8 @@ def load_manifest(filename): if node.nodeType in (node.TEXT_NODE, node.COMMENT_NODE): continue - if node.tagName not in ('include', 'project', 'remote', 'default', 'manifest', 'copyfile', 'remove-project'): + if node.tagName not in ('include', 'project', 'remote', 'default', 'manifest', 'copyfile', + 'remove-project'): raise ValueError("Unsupported tag: %s" % node.tagName) to_visit.extend(node.childNodes) diff --git a/testing/mozharness/mozharness/mozilla/repo_manipulation.py b/testing/mozharness/mozharness/mozilla/repo_manipulation.py index a2dfc46a2f43..0408aeffda73 100644 --- a/testing/mozharness/mozharness/mozilla/repo_manipulation.py +++ b/testing/mozharness/mozharness/mozilla/repo_manipulation.py @@ -160,5 +160,3 @@ the script (--clean-repos --pull --migrate). The second run will be faster.""" self.warning("No changes for %s!" % cwd) elif status: self.fatal(error_message) - - diff --git a/testing/mozharness/mozharness/mozilla/secrets.py b/testing/mozharness/mozharness/mozilla/secrets.py index 215884354708..ab045f7b0f57 100644 --- a/testing/mozharness/mozharness/mozilla/secrets.py +++ b/testing/mozharness/mozharness/mozilla/secrets.py @@ -7,11 +7,8 @@ """Support for fetching secrets from the secrets API """ -import os -import mozharness import urllib2 import json -from mozharness.base.log import ERROR class SecretsMixin(object): diff --git a/testing/mozharness/mozharness/mozilla/selfserve.py b/testing/mozharness/mozharness/mozilla/selfserve.py index 69e243059b1d..266401f53251 100644 --- a/testing/mozharness/mozharness/mozilla/selfserve.py +++ b/testing/mozharness/mozharness/mozilla/selfserve.py @@ -1,6 +1,7 @@ import json import site + # SelfServeMixin {{{1 class SelfServeMixin(object): def _get_session(self): diff --git a/testing/mozharness/mozharness/mozilla/signed_certificate_timestamp.py b/testing/mozharness/mozharness/mozilla/signed_certificate_timestamp.py index b3b309717404..633191741daf 100644 --- a/testing/mozharness/mozharness/mozilla/signed_certificate_timestamp.py +++ b/testing/mozharness/mozharness/mozilla/signed_certificate_timestamp.py @@ -3,6 +3,7 @@ import struct import base64 + class SignedCertificateTimestamp: """ Represents a Signed Certificate Timestamp from a Certificate Transparency @@ -75,7 +76,6 @@ class SignedCertificateTimestamp: if 'extensions' in response_json: self.extensions = base64.b64decode(response_json['extensions']) - @staticmethod def from_rfc6962(serialized): start = 0 @@ -120,11 +120,10 @@ class SignedCertificateTimestamp: sct.signature = struct.pack('!HH', alg, sig_len) + sig return sct - def to_rfc6962(self): version = struct.pack("B", self.version) timestamp = struct.pack("!Q", self.timestamp) ext_len = struct.pack("!H", len(self.extensions)) return version + self.id + timestamp + \ - ext_len + self.extensions + self.signature + ext_len + self.extensions + self.signature diff --git a/testing/mozharness/mozharness/mozilla/taskcluster_helper.py b/testing/mozharness/mozharness/mozilla/taskcluster_helper.py index 3a750d243ac8..9ef1ee889db4 100644 --- a/testing/mozharness/mozharness/mozilla/taskcluster_helper.py +++ b/testing/mozharness/mozharness/mozilla/taskcluster_helper.py @@ -4,7 +4,6 @@ import os from datetime import datetime, timedelta from urlparse import urljoin -from mozharness.base.log import INFO from mozharness.base.log import LogMixin @@ -251,7 +250,8 @@ class TaskClusterArtifactFinderMixin(object): self.set_artifacts( self.url_to_artifact(parent_id, installer_path), self.url_to_artifact(parent_id, 'public/build/target.test_packages.json'), - self.url_to_artifact(parent_id, 'public/build/target.crashreporter-symbols.zip') + self.url_to_artifact(parent_id, + 'public/build/target.crashreporter-symbols.zip') ) else: # Case 2: The parent task has an associated BBB task @@ -272,5 +272,6 @@ class TaskClusterArtifactFinderMixin(object): # Use the signed installer if it's set if 'signed_installer_url' in properties: signed_installer_url = properties['signed_installer_url'] - self.info('Overriding installer_url with signed_installer_url: %s' % signed_installer_url) + self.info('Overriding installer_url with signed_installer_url: %s' % + signed_installer_url) self.installer_url = signed_installer_url diff --git a/testing/mozharness/mozharness/mozilla/testing/codecoverage.py b/testing/mozharness/mozharness/mozilla/testing/codecoverage.py index 6beb2850c743..2a9afb136478 100644 --- a/testing/mozharness/mozharness/mozilla/testing/codecoverage.py +++ b/testing/mozharness/mozharness/mozilla/testing/codecoverage.py @@ -11,7 +11,6 @@ from mozharness.base.script import ( PreScriptAction, PostScriptAction, ) -from mozharness.mozilla.tooltool import TooltoolMixin code_coverage_config_options = [ [["--code-coverage"], @@ -77,15 +76,17 @@ class CodeCoverageMixin(object): # Create the grcov directory, get the tooltool manifest, and finally # download and unpack the grcov binary. self.grcov_dir = tempfile.mkdtemp() - manifest = os.path.join(dirs.get('abs_test_install_dir', os.path.join(dirs['abs_work_dir'], 'tests')), \ - 'config/tooltool-manifests/linux64/ccov.manifest') + manifest = os.path.join(dirs.get('abs_test_install_dir', + os.path.join(dirs['abs_work_dir'], 'tests')), + 'config/tooltool-manifests/linux64/ccov.manifest') tooltool_path = self._fetch_tooltool_py() - cmd = [tooltool_path, '--url', 'https://tooltool.mozilla-releng.net/', 'fetch', \ - '-m', manifest, '-o', '-c', '/builds/worker/tooltool-cache'] + cmd = [tooltool_path, '--url', 'https://tooltool.mozilla-releng.net/', 'fetch', + '-m', manifest, '-o', '-c', '/builds/worker/tooltool-cache'] self.run_command(cmd, cwd=self.grcov_dir) - self.run_command(['tar', '-jxvf', os.path.join(self.grcov_dir, 'grcov-linux-standalone-x86_64.tar.bz2'), \ - '-C', self.grcov_dir], cwd=self.grcov_dir) + self.run_command(['tar', '-jxvf', + os.path.join(self.grcov_dir, 'grcov-linux-standalone-x86_64.tar.bz2'), + '-C', self.grcov_dir], cwd=self.grcov_dir) @PostScriptAction('run-tests') def _package_coverage_data(self, action, success=None): @@ -98,7 +99,9 @@ class CodeCoverageMixin(object): # TODO This is fragile, find rel_topsrcdir properly somehow # We need to find the path relative to the gecko topsrcdir. Use # some known gecko directories as a test. - canary_dirs = ['browser', 'docshell', 'dom', 'js', 'layout', 'toolkit', 'xpcom', 'xpfe'] + canary_dirs = [ + 'browser', 'docshell', 'dom', 'js', 'layout', 'toolkit', 'xpcom', 'xpfe' + ] rel_topsrcdir = None for root, dirs, files in os.walk(self.gcov_dir): # need to use 'any' in case no gcda data was generated in that subdir. @@ -140,14 +143,17 @@ class CodeCoverageMixin(object): # 'grcov_output' will be a tuple, the first variable is the path to the lcov output, # the other is the path to the standard error output. - grcov_output = self.get_output_from_command(grcov_command, cwd=self.grcov_dir, \ - silent=True, tmpfile_base_path=os.path.join(self.grcov_dir, 'grcov_lcov_output'), \ - save_tmpfiles=True, return_type='files') + grcov_output = self.get_output_from_command( + grcov_command, cwd=self.grcov_dir, + silent=True, tmpfile_base_path=os.path.join(self.grcov_dir, 'grcov_lcov_output'), + save_tmpfiles=True, return_type='files' + ) new_output_name = grcov_output[0] + '.info' os.rename(grcov_output[0], new_output_name) # Zip the grcov output and upload it. - command = ['zip', os.path.join(dirs['abs_blob_upload_dir'], 'code-coverage-grcov.zip'), new_output_name] + command = ['zip', os.path.join(dirs['abs_blob_upload_dir'], + 'code-coverage-grcov.zip'), new_output_name] self.run_command(command, cwd=self.grcov_dir) shutil.rmtree(self.gcov_dir) shutil.rmtree(self.jsvm_dir) diff --git a/testing/mozharness/mozharness/mozilla/testing/device.py b/testing/mozharness/mozharness/mozilla/testing/device.py index 12c7a55ae18b..b909b8df4a46 100644 --- a/testing/mozharness/mozharness/mozilla/testing/device.py +++ b/testing/mozharness/mozharness/mozilla/testing/device.py @@ -10,11 +10,8 @@ This code is largely from https://hg.mozilla.org/build/tools/file/default/sut_tools ''' -import datetime -import os import re import subprocess -import sys import time from mozharness.base.errors import ADBErrorList @@ -117,9 +114,11 @@ class ADBDeviceHandler(BaseDeviceHandler): devices = self._query_attached_devices() if not devices: self.add_device_flag(DEVICE_NOT_CONNECTED) - self.fatal("No device connected via adb!\nUse 'adb connect' or specify a device_id or device_ip in config!") + self.fatal("No device connected via adb!\nUse 'adb connect' or specify a " + "device_id or device_ip in config!") elif len(devices) > 1: - self.warning("""More than one device detected; specify 'device_id' or\n'device_ip' to target a specific device!""") + self.warning("""More than one device detected; specify 'device_id' " + "or\n'device_ip' to target a specific device!""") device_id = devices[0] self.info("Found %s." % device_id) self.device_id = device_id @@ -362,9 +361,8 @@ class ADBDeviceHandler(BaseDeviceHandler): retries = 0 while retries < 6: output = self.get_output_from_command([adb, "-s", device_id, - "install", '-r', - file_path], - ignore_errors=True) + "install", '-r', file_path], + ignore_errors=True) if output and output.lower().find("success") >= 0: install_complete = True break @@ -479,7 +477,8 @@ class DeviceMixin(object): device_protocol = c.get('device_protocol') device_class = DEVICE_PROTOCOL_DICT.get(device_protocol) if not device_class: - self.fatal("Unknown device_protocol %s; set via --device-protocol!" % str(device_protocol)) + self.fatal("Unknown device_protocol %s; set via --device-protocol!" % + str(device_protocol)) self.device_handler = device_class( log_obj=self.log_obj, config=self.config, diff --git a/testing/mozharness/mozharness/mozilla/testing/errors.py b/testing/mozharness/mozharness/mozilla/testing/errors.py index ab2a8a3d0751..0615efe65e9f 100644 --- a/testing/mozharness/mozharness/mozilla/testing/errors.py +++ b/testing/mozharness/mozharness/mozilla/testing/errors.py @@ -19,7 +19,8 @@ from mozharness.base.log import INFO, WARNING, ERROR # ErrorLists {{{1 _mochitest_summary = { - 'regex': re.compile(r'''(\d+ INFO (Passed|Failed|Todo):\ +(\d+)|\t(Passed|Failed|Todo): (\d+))'''), + 'regex': ( + re.compile(r'''(\d+ INFO (Passed|Failed|Todo):\ +(\d+)|\t(Passed|Failed|Todo): (\d+))''')), 'pass_group': "Passed", 'fail_group': "Failed", 'known_fail_group': "Todo", @@ -39,13 +40,15 @@ TinderBoxPrintRe = { 'known_fail_group': "todo", }, "reftest_summary": { - 'regex': re.compile(r'''REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \('''), + 'regex': ( + re.compile(r'''REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \(''')), 'pass_group': "Successful", 'fail_group': "Unexpected", 'known_fail_group': "Known problems", }, "crashtest_summary": { - 'regex': re.compile(r'''REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \('''), + 'regex': ( + re.compile(r'''REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \(''')), 'pass_group': "Successful", 'fail_group': "Unexpected", 'known_fail_group': "Known problems", @@ -57,7 +60,8 @@ TinderBoxPrintRe = { 'known_fail_group': None, }, "jsreftest_summary": { - 'regex': re.compile(r'''REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \('''), + 'regex': ( + re.compile(r'''REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \(''')), 'pass_group': "Successful", 'fail_group': "Unexpected", 'known_fail_group': "Known problems", @@ -102,9 +106,16 @@ TinderBoxPrintRe = { }, "harness_error": { - 'full_regex': re.compile(r"(?:TEST-UNEXPECTED-FAIL|PROCESS-CRASH) \| .* \| (application crashed|missing output line for total leaks!|negative leaks caught!|\d+ bytes leaked)"), + 'full_regex': + re.compile(r"(?:TEST-UNEXPECTED-FAIL|PROCESS-CRASH) \| .* \| " + "(application crashed|missing output line for total leaks!|negative leaks " + "caught!|\d+ bytes leaked)"), 'minimum_regex': re.compile(r'''(TEST-UNEXPECTED|PROCESS-CRASH)'''), - 'retry_regex': re.compile(r'''(FAIL-SHOULD-RETRY|No space left on device|DMError|Connection to the other side was lost in a non-clean fashion|program finished with exit code 80|INFRA-ERROR|twisted.spread.pb.PBConnectionLost|_dl_open: Assertion|Timeout exceeded for _runCmd call)''') + 'retry_regex': + re.compile(r'''(FAIL-SHOULD-RETRY|No space left on device|DMError|Connection to the " + "other side was lost in a non-clean fashion|program finished with exit " + "code 80|INFRA-ERROR|twisted.spread.pb.PBConnectionLost|_dl_open: " + "Assertion|Timeout exceeded for _runCmd call)''') }, } @@ -120,8 +131,17 @@ HarnessErrorList = [ ] LogcatErrorList = [ - {'substr': 'Fatal signal 11 (SIGSEGV)', 'level': ERROR, 'explanation': 'This usually indicates the B2G process has crashed'}, - {'substr': 'Fatal signal 7 (SIGBUS)', 'level': ERROR, 'explanation': 'This usually indicates the B2G process has crashed'}, - {'substr': '[JavaScript Error:', 'level': WARNING}, - {'substr': 'seccomp sandbox violation', 'level': ERROR, 'explanation': 'A content process has violated the system call sandbox (bug 790923)'}, + {'substr': + 'Fatal signal 11 (SIGSEGV)', + 'level': ERROR, 'explanation': 'This usually indicates the B2G process has crashed'}, + {'substr': + 'Fatal signal 7 (SIGBUS)', + 'level': ERROR, 'explanation': 'This usually indicates the B2G process has crashed'}, + {'substr': + '[JavaScript Error:', + 'level': WARNING}, + {'substr': + 'seccomp sandbox violation', + 'level': ERROR, + 'explanation': 'A content process has violated the system call sandbox (bug 790923)'}, ] diff --git a/testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py b/testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py index f4df25b6837d..96c33d7cc03a 100644 --- a/testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py +++ b/testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py @@ -10,8 +10,7 @@ import copy import os import sys -from mozharness.base.log import FATAL, WARNING -from mozharness.base.python import PostScriptRun, PreScriptAction +from mozharness.base.python import PreScriptAction from mozharness.mozilla.structuredlog import StructuredOutputParser from mozharness.mozilla.testing.testbase import ( TestingMixin, @@ -30,7 +29,8 @@ firefox_ui_tests_config_options = [ "action": "store_true", "dest": "allow_software_gl_layers", "default": False, - "help": "Permits a software GL implementation (such as LLVMPipe) to use the GL compositor.", + "help": "Permits a software GL implementation (such as LLVMPipe) to use " + "the GL compositor.", }], [["--enable-webrender"], { "action": "store_true", @@ -254,7 +254,8 @@ class FirefoxUITests(TestingMixin, VCSToolsScript, CodeCoverageMixin): env.update({'MINIDUMP_STACKWALK': self.minidump_stackwalk_path}) env['RUST_BACKTRACE'] = 'full' - # If code coverage is enabled, set GCOV_PREFIX and JS_CODE_COVERAGE_OUTPUT_DIR env variables + # If code coverage is enabled, set GCOV_PREFIX and JS_CODE_COVERAGE_OUTPUT_DIR + # env variables if self.config.get('code_coverage'): env['GCOV_PREFIX'] = self.gcov_dir env['JS_CODE_COVERAGE_OUTPUT_DIR'] = self.jsvm_dir diff --git a/testing/mozharness/mozharness/mozilla/testing/mozpool.py b/testing/mozharness/mozharness/mozilla/testing/mozpool.py index 7e49f33ba913..6ecc710258ac 100644 --- a/testing/mozharness/mozharness/mozilla/testing/mozpool.py +++ b/testing/mozharness/mozharness/mozilla/testing/mozpool.py @@ -14,14 +14,15 @@ import sys from time import sleep from mozharness.mozilla.buildbot import TBPL_RETRY, TBPL_EXCEPTION -#TODO - adjust these values +# TODO - adjust these values MAX_RETRIES = 20 RETRY_INTERVAL = 60 + # MozpoolMixin {{{1 class MozpoolMixin(object): mozpool_handler = None - mobile_imaging_format= "http://mobile-imaging" + mobile_imaging_format = "http://mobile-imaging" def determine_mozpool_host(self, device): if "mobile_imaging_format" in self.config: @@ -32,11 +33,13 @@ class MozpoolMixin(object): return imaging_server_fqdn def query_mozpool_handler(self, device=None, mozpool_api_url=None): - if self.mozpool_handler != None: + if self.mozpool_handler is not None: return self.mozpool_handler else: - self.mozpool_api_url = self.determine_mozpool_host(device) if device else mozpool_api_url - assert self.mozpool_api_url != None, \ + self.mozpool_api_url = ( + self.determine_mozpool_host(device) if device else mozpool_api_url + ) + assert self.mozpool_api_url is not None, \ "query_mozpool_handler() requires either a device or mozpool_api_url!" site_packages_path = self.query_python_site_packages_path() @@ -44,7 +47,8 @@ class MozpoolMixin(object): sys.path.append(mph_path) sys.path.append(site_packages_path) try: - from mozpoolclient import MozpoolHandler, MozpoolException, MozpoolConflictException + from mozpoolclient import (MozpoolHandler, MozpoolException, + MozpoolConflictException) self.MozpoolException = MozpoolException self.MozpoolConflictException = MozpoolConflictException self.mozpool_handler = MozpoolHandler(self.mozpool_api_url, log_obj=self) @@ -60,9 +64,10 @@ class MozpoolMixin(object): tbpl_status=TBPL_RETRY): try: image = 'panda-android-4.0.4_v3.3' - duration = 4 * 60 * 60 # request valid for 14400 seconds == 4 hours - response = mph.request_device(self.mozpool_device, image, assignee=self.mozpool_assignee, \ - b2gbase=b2gbase, pxe_config=None, duration=duration) + duration = 4 * 60 * 60 # request valid for 14400 seconds == 4 hours + response = mph.request_device(self.mozpool_device, image, + assignee=self.mozpool_assignee, b2gbase=b2gbase, + pxe_config=None, duration=duration) break except self.MozpoolConflictException: self.warning("Device unavailable. Retry#%i.." % retry) @@ -76,7 +81,9 @@ class MozpoolMixin(object): def _retry_job_and_close_request(self, message, exception=None): mph = self.query_mozpool_handler(self.mozpool_device) - exception_message = str(exception) if exception!=None and str(exception) != None else "" + exception_message = ( + str(exception) if exception is not None and str(exception) is not None else "" + ) self.error("%s -> %s" % (message, exception_message)) if self.request_url: mph.close_request(self.request_url) @@ -99,13 +106,15 @@ class MozpoolMixin(object): def _wait_for_request_ready(self): mph = self.query_mozpool_handler(self.mozpool_device) + def on_fail(): # Device is not ready after retries... self.info("Aborting mozpool request.") self.close_request() for retry in self._retry_sleep(sleep_time=RETRY_INTERVAL, max_retries=MAX_RETRIES, - error_message="INFRA-ERROR: Request did not become ready in time", - tbpl_status=TBPL_EXCEPTION, fail_cb=on_fail): + error_message="INFRA-ERROR: Request did not become " + "ready in time", + tbpl_status=TBPL_EXCEPTION, fail_cb=on_fail): response = mph.query_request_status(self.request_url) state = response['state'] if state == 'ready': diff --git a/testing/mozharness/mozharness/mozilla/testing/talos.py b/testing/mozharness/mozharness/mozilla/testing/talos.py index be0c51e6bcd3..41ff61b2fff5 100755 --- a/testing/mozharness/mozharness/mozilla/testing/talos.py +++ b/testing/mozharness/mozharness/mozilla/testing/talos.py @@ -48,11 +48,13 @@ TalosErrorList = PythonErrorList + [ {'regex': re.compile(r'''No machine_name called '.*' can be found'''), 'level': CRITICAL}, {'substr': r"""No such file or directory: 'browser_output.txt'""", 'level': CRITICAL, - 'explanation': r"""Most likely the browser failed to launch, or the test was otherwise unsuccessful in even starting."""}, + 'explanation': r"""Most likely the browser failed to launch, or the test was otherwise " + "unsuccessful in even starting."""}, ] # TODO: check for running processes on script invocation + class TalosOutputParser(OutputParser): minidump_regex = re.compile(r'''talosError: "error executing: '(\S+) (\S+) (\S+)'"''') RE_PERF_DATA = re.compile(r'.*PERFHERDER_DATA:\s+(\{.*\})') @@ -198,12 +200,17 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin, self.gecko_profile = self.config.get('gecko_profile') self.gecko_profile_interval = self.config.get('gecko_profile_interval') self.pagesets_name = None - self.mitmproxy_rel_bin = None # some platforms download a mitmproxy release binary - self.mitmproxy_recording_set = None # zip file found on tooltool that contains all of the mitmproxy recordings - self.mitmproxy_recordings_file_list = self.config.get('mitmproxy', None) # files inside the recording set - self.mitmdump = None # path to mitdump tool itself, in py3 venv + # some platforms download a mitmproxy release binary + self.mitmproxy_rel_bin = None + # zip file found on tooltool that contains all of the mitmproxy recordings + self.mitmproxy_recording_set = None + # files inside the recording set + self.mitmproxy_recordings_file_list = self.config.get('mitmproxy', None) + # path to mitdump tool itself, in py3 venv + self.mitmdump = None - # We accept some configuration options from the try commit message in the format mozharness: + # We accept some configuration options from the try commit message + # in the format mozharness: # Example try commit message: # mozharness: --geckoProfile try: def query_gecko_profile_options(self): @@ -212,7 +219,9 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin, # this is inside automation # now let's see if we added GeckoProfile specs in the commit message try: - junk, junk, opts = self.buildbot_config['sourcestamp']['changes'][-1]['comments'].partition('mozharness:') + junk, junk, opts = self.buildbot_config( + ['sourcestamp']['changes'][-1]['comments'].partition('mozharness:') + ) except IndexError: # when we don't have comments on changes (bug 1255187) opts = None @@ -249,7 +258,8 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin, if self.abs_dirs: return self.abs_dirs abs_dirs = super(Talos, self).query_abs_dirs() - abs_dirs['abs_blob_upload_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'blobber_upload_dir') + abs_dirs['abs_blob_upload_dir'] = os.path.join(abs_dirs['abs_work_dir'], + 'blobber_upload_dir') self.abs_dirs = abs_dirs return self.abs_dirs @@ -288,7 +298,8 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin, def get_suite_from_test(self): """ Retrieve the talos suite name from a given talos test name.""" - # running locally, single test name provided instead of suite; go through tests and find suite name + # running locally, single test name provided instead of suite; + # go through tests and find suite name suite_name = None if self.query_talos_json_config(): if '-a' in self.config['talos_extra_options']: @@ -310,7 +321,7 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin, def validate_suite(self): """ Ensure suite name is a valid talos suite. """ if self.query_talos_json_config() and self.suite is not None: - if not self.suite in self.talos_json_config.get('suites'): + if self.suite not in self.talos_json_config.get('suites'): self.fatal("Suite '%s' is not valid (not found in talos json config)" % self.suite) def talos_options(self, args=None, **kw): @@ -318,7 +329,8 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin, # binary path binary_path = self.binary_path or self.config.get('binary_path') if not binary_path: - self.fatal("Talos requires a path to the binary. You can specify binary_path or add download-and-extract to your action list.") + self.fatal("Talos requires a path to the binary. You can specify binary_path or " + "add download-and-extract to your action list.") # talos options options = [] @@ -374,7 +386,8 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin, if self.config.get('run_local'): # talos initiated locally, get and verify test/suite from cmd line self.talos_path = os.path.dirname(self.talos_json) - if '-a' in self.config['talos_extra_options'] or '--activeTests' in self.config['talos_extra_options']: + if '-a' in (self.config['talos_extra_options'] or + '--activeTests' in self.config['talos_extra_options']): # test name (-a or --activeTests) specified, find out what suite it is a part of self.suite = self.get_suite_from_test() elif '--suite' in self.config['talos_extra_options']: @@ -442,7 +455,8 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin, # now create the py3 venv self.py3_venv_configuration(python_path=self.py3_path, venv_path='py3venv') self.py3_create_venv() - requirements = [os.path.join(self.talos_path, 'talos', 'mitmproxy', 'mitmproxy_requirements.txt')] + requirements = [os.path.join(self.talos_path, 'talos', 'mitmproxy', + 'mitmproxy_requirements.txt')] self.py3_install_requirement_files(requirements) # add py3 executables path to system path sys.path.insert(1, self.py3_path_to_executables()) @@ -469,7 +483,8 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin, self.fatal("Aborting: mitmproxy_release_bin_osx not found in talos.json") self.download_mitmproxy_binary(_platform) else: - self.info("Not downloading mitmproxy rel binary because no-download was specified") + self.info("Not downloading mitmproxy rel binary because " + "no-download was specified") self.info('The mitmdump macosx binary is found at: %s' % self.mitmdump) self.run_command([self.mitmdump, '--version'], env=self.query_env()) @@ -479,7 +494,9 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin, return self.mitmproxy_rel_bin if self.query_talos_json_config() and self.suite is not None: config_key = "mitmproxy_release_bin_" + platform - self.mitmproxy_rel_bin = self.talos_json_config['suites'][self.suite].get(config_key, False) + self.mitmproxy_rel_bin = ( + self.talos_json_config['suites'][self.suite].get(config_key, False) + ) return self.mitmproxy_rel_bin def download_mitmproxy_binary(self, platform): @@ -506,14 +523,17 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin, if self.mitmproxy_recording_set: return self.mitmproxy_recording_set if self.query_talos_json_config() and self.suite is not None: - self.mitmproxy_recording_set = self.talos_json_config['suites'][self.suite].get('mitmproxy_recording_set', False) + self.mitmproxy_recording_set = ( + self.talos_json_config['suites'][self.suite].get('mitmproxy_recording_set', False) + ) return self.mitmproxy_recording_set def download_mitmproxy_recording_set(self): """Download the set of mitmproxy recording files that will be played back""" self.info("Downloading the mitmproxy recording set using tooltool") dest = os.path.join(self.talos_path, 'talos', 'mitmproxy') - manifest_file = os.path.join(self.talos_path, 'talos', 'mitmproxy', 'mitmproxy-playback-set.manifest') + manifest_file = os.path.join(self.talos_path, 'talos', 'mitmproxy', + 'mitmproxy-playback-set.manifest') self.tooltool_fetch( manifest_file, output_dir=dest, @@ -597,7 +617,6 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin, shutil.copyfile(src, dest) except: self.critical("Error copying results %s to upload dir %s" % (src, dest)) - parser.update_worst_log_and_tbpl_levels(CRITICAL, TBPL_FAILURE) def run_tests(self, args=None, **kw): """run Talos tests""" diff --git a/testing/mozharness/mozharness/mozilla/testing/testbase.py b/testing/mozharness/mozharness/mozilla/testing/testbase.py index be5e9256858e..65cfae5e7654 100755 --- a/testing/mozharness/mozharness/mozilla/testing/testbase.py +++ b/testing/mozharness/mozharness/mozilla/testing/testbase.py @@ -9,10 +9,8 @@ import copy import os import platform import pprint -import re import urllib2 import json -import socket from urlparse import urlparse, ParseResult from mozharness.base.errors import BaseErrorList @@ -52,52 +50,53 @@ TOOLTOOL_PLATFORM_DIR = { testing_config_options = [ [["--installer-url"], {"action": "store", - "dest": "installer_url", - "default": None, - "help": "URL to the installer to install", + "dest": "installer_url", + "default": None, + "help": "URL to the installer to install", }], [["--installer-path"], {"action": "store", - "dest": "installer_path", - "default": None, - "help": "Path to the installer to install. This is set automatically if run with --download-and-extract.", + "dest": "installer_path", + "default": None, + "help": "Path to the installer to install. This is set automatically if run with " + "--download-and-extract.", }], [["--binary-path"], {"action": "store", - "dest": "binary_path", - "default": None, - "help": "Path to installed binary. This is set automatically if run with --install.", + "dest": "binary_path", + "default": None, + "help": "Path to installed binary. This is set automatically if run with --install.", }], [["--exe-suffix"], {"action": "store", - "dest": "exe_suffix", - "default": None, - "help": "Executable suffix for binaries on this platform", + "dest": "exe_suffix", + "default": None, + "help": "Executable suffix for binaries on this platform", }], [["--test-url"], {"action": "store", - "dest": "test_url", - "default": None, - "help": "URL to the zip file containing the actual tests", + "dest": "test_url", + "default": None, + "help": "URL to the zip file containing the actual tests", }], [["--test-packages-url"], {"action": "store", - "dest": "test_packages_url", - "default": None, - "help": "URL to a json file describing which tests archives to download", + "dest": "test_packages_url", + "default": None, + "help": "URL to a json file describing which tests archives to download", }], [["--jsshell-url"], {"action": "store", - "dest": "jsshell_url", - "default": None, - "help": "URL to the jsshell to install", + "dest": "jsshell_url", + "default": None, + "help": "URL to the jsshell to install", }], [["--download-symbols"], {"action": "store", - "dest": "download_symbols", - "type": "choice", - "choices": ['ondemand', 'true'], - "help": "Download and extract crash reporter symbols.", + "dest": "download_symbols", + "type": "choice", + "choices": ['ondemand', 'true'], + "help": "Download and extract crash reporter symbols.", }], ] + copy.deepcopy(virtualenv_config_options) \ + copy.deepcopy(try_config_options) \ @@ -212,7 +211,7 @@ class TestingMixin(VirtualenvMixin, BuildbotMixin, ResourceMonitoringMixin, self.symbols_url = symbols_url except Exception as ex: self.warning("Cannot open symbols url %s (installer url: %s): %s" % - (symbols_url, self.installer_url, ex)) + (symbols_url, self.installer_url, ex)) if raise_on_failure: raise @@ -243,7 +242,7 @@ class TestingMixin(VirtualenvMixin, BuildbotMixin, ResourceMonitoringMixin, c = self.config orig_config = copy.deepcopy(c) self.warning("When you use developer_config.py, we drop " - "'read-buildbot-config' from the list of actions.") + "'read-buildbot-config' from the list of actions.") if "read-buildbot-config" in rw_config.actions: rw_config.actions.remove("read-buildbot-config") self.actions = tuple(rw_config.actions) @@ -260,7 +259,8 @@ class TestingMixin(VirtualenvMixin, BuildbotMixin, ResourceMonitoringMixin, self.exception("You must use --installer-url with developer_config.py") if c.get("require_test_zip"): if not c.get('test_url') and not c.get('test_packages_url'): - self.exception("You must use --test-url or --test-packages-url with developer_config.py") + self.exception("You must use --test-url or " + "--test-packages-url with developer_config.py") c["installer_url"] = _replace_url(c["installer_url"], c["replace_urls"]) if c.get("test_url"): @@ -292,7 +292,8 @@ class TestingMixin(VirtualenvMixin, BuildbotMixin, ResourceMonitoringMixin, self.https_username, self.https_password = get_credentials() # This creates a password manager passman = urllib2.HTTPPasswordMgrWithDefaultRealm() - # Because we have put None at the start it will use this username/password combination from here on + # Because we have put None at the start it will use this username/password combination + # from here on passman.add_password(None, url, self.https_username, self.https_password) authhandler = urllib2.HTTPBasicAuthHandler(passman) @@ -311,7 +312,6 @@ class TestingMixin(VirtualenvMixin, BuildbotMixin, ResourceMonitoringMixin, c = self.config try: files = self.buildbot_config['sourcestamp']['changes'][-1]['files'] - buildbot_prop_branch = self.buildbot_config['properties']['branch'] # Bug 868490 - Only require exactly two files if require_test_zip; # otherwise accept either 1 or 2, since we'll be getting a @@ -321,7 +321,8 @@ class TestingMixin(VirtualenvMixin, BuildbotMixin, ResourceMonitoringMixin, expected_length = [2, 3] actual_length = len(files) if actual_length not in expected_length: - self.fatal("Unexpected number of files in buildbot config %s.\nExpected these number(s) of files: %s, but got: %d" % + self.fatal("Unexpected number of files in buildbot config %s.\n" + "Expected these number(s) of files: %s, but got: %d" % (c['buildbot_json_path'], str(expected_length), actual_length)) for f in files: if f['name'].endswith('tests.zip'): # yuk @@ -335,7 +336,8 @@ class TestingMixin(VirtualenvMixin, BuildbotMixin, ResourceMonitoringMixin, elif f['name'].endswith('test_packages.json'): self.test_packages_url = str(f['name']) self.info("Found a test packages url %s." % self.test_packages_url) - elif not any(f['name'].endswith(s) for s in ('code-coverage-gcno.zip', 'stylo-bindings.zip')): + elif not any(f['name'].endswith(s) for s in ('code-coverage-gcno.zip', + 'stylo-bindings.zip')): if not self.installer_url: self.installer_url = str(f['name']) self.info("Found installer url %s." % self.installer_url) @@ -401,7 +403,8 @@ You can set this by: 2. running via buildbot and running the read-buildbot-config action """ - if self.config.get("require_test_zip") and not self.test_url and not self.test_packages_url: + if (self.config.get("require_test_zip") + and not self.test_url and not self.test_packages_url): message += """test_url isn't set! You can set this by: @@ -526,10 +529,8 @@ You can set this by: if self.installer_path: file_name = self.installer_path dirs = self.query_abs_dirs() - source = self.download_file(self.installer_url, - file_name=file_name, - parent_dir=dirs['abs_work_dir'], - error_level=FATAL) + source = self.download_file(self.installer_url, file_name=file_name, + parent_dir=dirs['abs_work_dir'], error_level=FATAL) self.installer_path = os.path.realpath(source) self.set_buildbot_property("build_url", self.installer_url, write_to_file=True) @@ -619,7 +620,8 @@ Did you run with --create-virtualenv? Is mozinstall in virtualenv_modules?""") def install_app(self, app=None, target_dir=None, installer_path=None): """ Dependent on mozinstall """ # install the application - cmd = self.query_exe("mozinstall", default=self.query_python_path("mozinstall"), return_type="list") + cmd = self.query_exe("mozinstall", default=self.query_python_path("mozinstall"), + return_type="list") if app: cmd.extend(['--app', app]) # Remove the below when we no longer need to support mozinstall 0.3 @@ -720,7 +722,7 @@ Did you run with --create-virtualenv? Is mozinstall in virtualenv_modules?""") return self.nodejs_path c = self.config - dirs = self.query_abs_dirs(); + dirs = self.query_abs_dirs() nodejs_path = self.query_nodejs_filename() if not self.config.get('download_nodejs'): @@ -750,7 +752,8 @@ Did you run with --create-virtualenv? Is mozinstall in virtualenv_modules?""") self.chmod(abs_nodejs_path, 0755) self.nodejs_path = abs_nodejs_path else: - self.warning("nodejs path was given but couldn't be found. Tried looking in '%s'" % abs_nodejs_path) + self.warning("nodejs path was given but couldn't be found. " + "Tried looking in '%s'" % abs_nodejs_path) self.buildbot_status(TBPL_WARNING, WARNING) return self.nodejs_path diff --git a/testing/mozharness/mozharness/mozilla/testing/try_tools.py b/testing/mozharness/mozharness/mozilla/testing/try_tools.py index 583ba5b217ed..296c98441a46 100644 --- a/testing/mozharness/mozharness/mozilla/testing/try_tools.py +++ b/testing/mozharness/mozharness/mozilla/testing/try_tools.py @@ -16,9 +16,9 @@ from mozharness.base.transfer import TransferMixin try_config_options = [ [["--try-message"], {"action": "store", - "dest": "try_message", - "default": None, - "help": "try syntax string to select tests to run", + "dest": "try_message", + "default": None, + "help": "try syntax string to select tests to run", }], ] @@ -27,7 +27,7 @@ test_flavors = { 'chrome': {}, 'devtools-chrome': {}, 'mochitest': {}, - 'xpcshell' :{}, + 'xpcshell': {}, 'reftest': { "path": lambda x: os.path.join("tests", "reftest", "tests", x) }, @@ -45,6 +45,7 @@ test_flavors = { }, } + class TryToolsMixin(TransferMixin): """Utility functions for an interface between try syntax and out test harnesses. Requires log and script mixins.""" @@ -143,7 +144,7 @@ class TryToolsMixin(TransferMixin): all_try_args = re.findall(r'(?:\[.*?\]|\S)+', try_message[1]) break if not all_try_args: - self.warning('Try syntax not found in: %s.' % msg ) + self.warning('Try syntax not found in: %s.' % msg) return all_try_args def try_message_has_flag(self, flag, message=None): @@ -195,6 +196,7 @@ class TryToolsMixin(TransferMixin): ' and forward them to the underlying test harness command.')) label_dict = {} + def label_from_val(val): if val in label_dict: return label_dict[val] @@ -257,7 +259,7 @@ class TryToolsMixin(TransferMixin): 'files: %s.' % ','.join(self.try_test_paths[flavor])) args.extend(['--this-chunk=1', '--total-chunks=1']) - path_func = test_flavors[flavor].get("path", lambda x:x) + path_func = test_flavors[flavor].get("path", lambda x: x) tests = [path_func(os.path.normpath(item)) for item in self.try_test_paths[flavor]] else: tests = [] diff --git a/testing/mozharness/mozharness/mozilla/testing/unittest.py b/testing/mozharness/mozharness/mozilla/testing/unittest.py index d935ff699a8e..50c450acd7f0 100755 --- a/testing/mozharness/mozharness/mozilla/testing/unittest.py +++ b/testing/mozharness/mozharness/mozilla/testing/unittest.py @@ -186,13 +186,11 @@ class DesktopUnittestOutputParser(OutputParser): levels=TBPL_WORST_LEVEL_TUPLE) # Account for the possibility that no test summary was output. - if self.pass_count <= 0 and self.fail_count <= 0 and \ - (self.known_fail_count is None or self.known_fail_count <= 0): + if self.pass_count <= 0 and self.fail_count <= 0 and (self.known_fail_count is None + or self.known_fail_count <= 0): self.error('No tests run or test summary not found') - self.worst_log_level = self.worst_level(WARNING, - self.worst_log_level) - self.tbpl_status = self.worst_level(TBPL_WARNING, - self.tbpl_status, + self.worst_log_level = self.worst_level(WARNING, self.worst_log_level) + self.tbpl_status = self.worst_level(TBPL_WARNING, self.tbpl_status, levels=TBPL_WORST_LEVEL_TUPLE) if return_code not in success_codes: diff --git a/testing/mozharness/mozharness/mozilla/testing/verify_tools.py b/testing/mozharness/mozharness/mozilla/testing/verify_tools.py index 27521c125e00..5775fae8e15c 100644 --- a/testing/mozharness/mozharness/mozilla/testing/verify_tools.py +++ b/testing/mozharness/mozharness/mozilla/testing/verify_tools.py @@ -5,7 +5,6 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. # ***** END LICENSE BLOCK ***** -import argparse import os import posixpath import re @@ -43,7 +42,7 @@ class VerifyToolsMixin(object): been downloaded and extracted. """ - if self.config.get('verify') != True: + if not self.config.get('verify'): return repository = os.environ.get("GECKO_HEAD_REPOSITORY") @@ -62,7 +61,8 @@ class VerifyToolsMixin(object): manifests = [ (os.path.join(dirs['abs_mochitest_dir'], 'tests', 'mochitest.ini'), 'plain'), (os.path.join(dirs['abs_mochitest_dir'], 'chrome', 'chrome.ini'), 'chrome'), - (os.path.join(dirs['abs_mochitest_dir'], 'browser', 'browser-chrome.ini'), 'browser-chrome'), + (os.path.join(dirs['abs_mochitest_dir'], 'browser', + 'browser-chrome.ini'), 'browser-chrome'), (os.path.join(dirs['abs_mochitest_dir'], 'a11y', 'a11y.ini'), 'a11y'), (os.path.join(dirs['abs_xpcshell_dir'], 'tests', 'xpcshell.ini'), 'xpcshell'), ] @@ -71,13 +71,16 @@ class VerifyToolsMixin(object): if os.path.exists(path): man = TestManifest([path], strict=False) active = man.active_tests(exists=False, disabled=False, filters=[], **mozinfo.info) - tests_by_path.update({t['relpath']:(suite,t.get('subsuite')) for t in active}) + tests_by_path.update({t['relpath']: (suite, t.get('subsuite')) for t in active}) self.info("Verification updated with manifest %s" % path) ref_manifests = [ - (os.path.join(dirs['abs_reftest_dir'], 'tests', 'layout', 'reftests', 'reftest.list'), 'reftest'), - (os.path.join(dirs['abs_reftest_dir'], 'tests', 'testing', 'crashtest', 'crashtests.list'), 'crashtest'), - # TODO (os.path.join(dirs['abs_test_install_dir'], 'jsreftest', 'tests', 'jstests.list'), 'jstestbrowser'), + (os.path.join(dirs['abs_reftest_dir'], 'tests', 'layout', 'reftests', + 'reftest.list'), 'reftest'), + (os.path.join(dirs['abs_reftest_dir'], 'tests', 'testing', 'crashtest', + 'crashtests.list'), 'crashtest'), + # TODO (os.path.join(dirs['abs_test_install_dir'], 'jsreftest', 'tests', + # 'jstests.list'), 'jstestbrowser'), ] sys.path.append(dirs['abs_reftest_dir']) import manifest @@ -86,7 +89,8 @@ class VerifyToolsMixin(object): if os.path.exists(path): man = manifest.ReftestManifest() man.load(path) - tests_by_path.update({os.path.relpath(t,self.reftest_test_dir):(suite,None) for t in man.files}) + tests_by_path.update({os.path.relpath(t, self.reftest_test_dir): (suite, None) + for t in man.files}) self.info("Verification updated with manifest %s" % path) # determine which files were changed on this push @@ -108,15 +112,15 @@ class VerifyToolsMixin(object): if entry: self.info("Verification found test %s" % file) subsuite_mapping = { - ('browser-chrome', 'clipboard') : 'browser-chrome-clipboard', - ('chrome', 'clipboard') : 'chrome-clipboard', - ('plain', 'clipboard') : 'plain-clipboard', - ('browser-chrome', 'devtools') : 'mochitest-devtools-chrome', - ('browser-chrome', 'gpu') : 'browser-chrome-gpu', - ('chrome', 'gpu') : 'chrome-gpu', - ('plain', 'gpu') : 'plain-gpu', - ('plain', 'media') : 'mochitest-media', - ('plain', 'webgl') : 'mochitest-gl', + ('browser-chrome', 'clipboard'): 'browser-chrome-clipboard', + ('chrome', 'clipboard'): 'chrome-clipboard', + ('plain', 'clipboard'): 'plain-clipboard', + ('browser-chrome', 'devtools'): 'mochitest-devtools-chrome', + ('browser-chrome', 'gpu'): 'browser-chrome-gpu', + ('chrome', 'gpu'): 'chrome-gpu', + ('plain', 'gpu'): 'plain-gpu', + ('plain', 'media'): 'mochitest-media', + ('plain', 'webgl'): 'mochitest-gl', } if entry in subsuite_mapping: suite = subsuite_mapping[entry] @@ -142,7 +146,7 @@ class VerifyToolsMixin(object): # when verifying long-running tests. MAX_TIME_PER_TEST = 900 - if self.config.get('verify') != True: + if self.config.get('verify'): # not in verify mode: run once, with no additional args args = [[]] else: @@ -174,10 +178,10 @@ class VerifyToolsMixin(object): suite category. """ suites = None - if self.config.get('verify') == True: + if self.config.get('verify'): if all_suites and self.verify_downloaded: suites = dict((key, all_suites.get(key)) for key in - self.verify_suites if key in all_suites.keys()) + self.verify_suites if key in all_suites.keys()) else: # Until test zips are downloaded, manifests are not available, # so it is not possible to determine which suites are active/ @@ -208,4 +212,3 @@ class VerifyToolsMixin(object): test_name = test_name.rstrip(os.path.sep) self.log("TinderboxPrint: Verification of %s
: %s" % (test_name, tbpl_status), level=log_level) - diff --git a/testing/mozharness/mozharness/mozilla/updates/balrog.py b/testing/mozharness/mozharness/mozilla/updates/balrog.py index ec8a1d2c0792..7082eae385e5 100644 --- a/testing/mozharness/mozharness/mozilla/updates/balrog.py +++ b/testing/mozharness/mozharness/mozilla/updates/balrog.py @@ -24,7 +24,6 @@ class BalrogMixin(object): python = 'python2.7' return python - def generate_balrog_props(self, props_path): self.set_buildbot_property( "hashType", self.config.get("hash_type", "sha512"), write_to_file=True @@ -107,12 +106,16 @@ class BalrogMixin(object): product = self.buildbot_config["properties"]["product"] cmd = [ self.query_python(), - os.path.join(os.path.join(dirs['abs_tools_dir'], "scripts/updates/balrog-release-pusher.py")) + os.path.join(os.path.join(dirs['abs_tools_dir'], + "scripts/updates/balrog-release-pusher.py")) ] - cmd.extend(["--build-properties", os.path.join(dirs["base_work_dir"], "balrog_props.json")]) + cmd.extend(["--build-properties", + os.path.join(dirs["base_work_dir"], "balrog_props.json")]) cmd.extend(["--buildbot-configs", "https://hg.mozilla.org/build/buildbot-configs"]) - cmd.extend(["--release-config", os.path.join(dirs['build_dir'], self.config.get("release_config_file"))]) - cmd.extend(["--credentials-file", os.path.join(dirs['base_work_dir'], self.config.get("balrog_credentials_file"))]) + cmd.extend(["--release-config", os.path.join(dirs['build_dir'], + self.config.get("release_config_file"))]) + cmd.extend(["--credentials-file", os.path.join(dirs['base_work_dir'], + self.config.get("balrog_credentials_file"))]) cmd.extend(["--release-channel", self.query_release_config()['release_channel']]) return_codes = [] diff --git a/tools/lint/flake8.yml b/tools/lint/flake8.yml index 46fd1dbaf1c6..f4bd052cc860 100644 --- a/tools/lint/flake8.yml +++ b/tools/lint/flake8.yml @@ -23,6 +23,7 @@ flake8: - testing/mochitest - testing/mozbase - testing/mozharness/mozfile + - testing/mozharness/mozharness/mozilla - testing/mozharness/mozinfo - testing/mozharness/scripts - testing/remotecppunittests.py From a2d687ae35b8bcc3f8107675617989a9ed5720c0 Mon Sep 17 00:00:00 2001 From: Paul Bone Date: Wed, 18 Oct 2017 09:17:04 +1100 Subject: [PATCH 22/32] Bug 1298018 - Part 1: Clarify emptyChunks_ comment. r=jonco --- js/src/gc/GCRuntime.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/js/src/gc/GCRuntime.h b/js/src/gc/GCRuntime.h index 33e46f8d324f..2e94138d9513 100644 --- a/js/src/gc/GCRuntime.h +++ b/js/src/gc/GCRuntime.h @@ -1210,10 +1210,11 @@ class GCRuntime AtomMarkingRuntime atomMarking; private: - // When empty, chunks reside in the emptyChunks pool and are re-used as - // needed or eventually expired if not re-used. The emptyChunks pool gets - // refilled from the background allocation task heuristically so that empty - // chunks should always available for immediate allocation without syscalls. + // When chunks are empty, they reside in the emptyChunks pool and are + // re-used as needed or eventually expired if not re-used. The emptyChunks + // pool gets refilled from the background allocation task heuristically so + // that empty chunks should always be available for immediate allocation + // without syscalls. GCLockData emptyChunks_; // Chunks which have had some, but not all, of their arenas allocated live From 5d894541075c3654933f30bc44538fdb110470b3 Mon Sep 17 00:00:00 2001 From: Paul Bone Date: Mon, 16 Oct 2017 21:29:07 +1100 Subject: [PATCH 23/32] Bug 1298018 - Part 2: Remove some unused methods. r=jonco --- js/src/gc/Nursery.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/js/src/gc/Nursery.h b/js/src/gc/Nursery.h index 08ecbe099880..8d20f193d2b9 100644 --- a/js/src/gc/Nursery.h +++ b/js/src/gc/Nursery.h @@ -145,7 +145,6 @@ class Nursery unsigned numChunks() const { return chunks_.length(); } bool exists() const { return maxChunks() != 0; } - size_t nurserySize() const { return maxChunks() << ChunkShift; } void enable(); void disable(); @@ -411,8 +410,6 @@ class Nursery Canary* lastCanary_; #endif - NurseryChunk* allocChunk(); - NurseryChunk& chunk(unsigned index) const { return *chunks_[index]; } From df8aa00bd737c43a6ebc9533b318106e26828dc1 Mon Sep 17 00:00:00 2001 From: Paul Bone Date: Mon, 30 Oct 2017 13:17:11 +1100 Subject: [PATCH 24/32] Bug 1298018 - Part 3: Replace updateNumChunks(). r=jonco This method both shrinks and grows the nursery. This change replaces it with specialised code to either shrink or grow the nursery. --- js/src/gc/Nursery.cpp | 151 ++++++++++++++++++++++++------------------ js/src/gc/Nursery.h | 17 +++-- 2 files changed, 100 insertions(+), 68 deletions(-) diff --git a/js/src/gc/Nursery.cpp b/js/src/gc/Nursery.cpp index 1718e3c6b747..a0ab5471466d 100644 --- a/js/src/gc/Nursery.cpp +++ b/js/src/gc/Nursery.cpp @@ -147,8 +147,7 @@ js::Nursery::init(uint32_t maxNurseryBytes, AutoLockGCBgAlloc& lock) if (maxNurseryChunks_ == 0) return true; - updateNumChunksLocked(1, lock); - if (numChunks() == 0) + if (!allocateFirstChunk(lock)) return false; setCurrentChunk(0); @@ -196,9 +195,11 @@ js::Nursery::enable() if (isEnabled() || !maxChunks()) return; - updateNumChunks(1); - if (numChunks() == 0) - return; + { + AutoLockGCBgAlloc lock(runtime()); + if (!allocateFirstChunk(lock)) + return; + } setCurrentChunk(0); setStartPosition(); @@ -216,7 +217,8 @@ js::Nursery::disable() MOZ_ASSERT(isEmpty()); if (!isEnabled()) return; - updateNumChunks(0); + + freeChunksFrom(0); currentEnd_ = 0; runtime()->gc.storeBuffer().disable(); } @@ -238,7 +240,7 @@ js::Nursery::isEmpty() const void js::Nursery::enterZealMode() { if (isEnabled()) - updateNumChunks(maxNurseryChunks_); + growAllocableSpace(maxChunks()); } void @@ -991,93 +993,116 @@ js::Nursery::maybeResizeNursery(JS::gcreason::Reason reason) if (newMaxNurseryChunks != maxNurseryChunks_) { maxNurseryChunks_ = newMaxNurseryChunks; /* The configured maximum nursery size is changing */ - const int extraChunks = numChunks() - newMaxNurseryChunks; - if (extraChunks > 0) { + if (numChunks() > newMaxNurseryChunks) { /* We need to shrink the nursery */ - shrinkAllocableSpace(extraChunks); + shrinkAllocableSpace(newMaxNurseryChunks); previousPromotionRate_ = promotionRate; return; } } - if (promotionRate > GrowThreshold) + if (promotionRate > GrowThreshold) { + // The GC nursery is an optimization and so if we fail to allocate + // nursery chunks we do not report an error. growAllocableSpace(); - else if (promotionRate < ShrinkThreshold && previousPromotionRate_ < ShrinkThreshold) - shrinkAllocableSpace(1); + } else if (promotionRate < ShrinkThreshold && previousPromotionRate_ < ShrinkThreshold) { + shrinkAllocableSpace(numChunks() - 1); + } previousPromotionRate_ = promotionRate; } -void +bool js::Nursery::growAllocableSpace() { - updateNumChunks(Min(numChunks() * 2, maxNurseryChunks_)); + return growAllocableSpace(Min(numChunks() * 2, maxChunks())); } -void -js::Nursery::shrinkAllocableSpace(unsigned removeNumChunks) +bool +js::Nursery::growAllocableSpace(unsigned newCount) { -#ifdef JS_GC_ZEAL - if (runtime()->hasZealMode(ZealMode::GenerationalGC)) - return; -#endif - updateNumChunks(Max(numChunks() - removeNumChunks, 1u)); -} - -void -js::Nursery::minimizeAllocableSpace() -{ -#ifdef JS_GC_ZEAL - if (runtime()->hasZealMode(ZealMode::GenerationalGC)) - return; -#endif - updateNumChunks(1); -} - -void -js::Nursery::updateNumChunks(unsigned newCount) -{ - if (numChunks() != newCount) { - AutoLockGCBgAlloc lock(runtime()); - updateNumChunksLocked(newCount, lock); - } -} - -void -js::Nursery::updateNumChunksLocked(unsigned newCount, - AutoLockGCBgAlloc& lock) -{ - // The GC nursery is an optimization and so if we fail to allocate nursery - // chunks we do not report an error. - - MOZ_ASSERT(newCount <= maxChunks()); - unsigned priorCount = numChunks(); - MOZ_ASSERT(priorCount != newCount); - if (newCount < priorCount) { - // Shrink the nursery and free unused chunks. - for (unsigned i = newCount; i < priorCount; i++) - runtime()->gc.recycleChunk(chunk(i).toChunk(runtime()), lock); - chunks_.shrinkTo(newCount); - return; - } + MOZ_ASSERT(newCount > priorCount); + MOZ_ASSERT(newCount <= maxChunks()); + MOZ_ASSERT(priorCount >= 1); - // Grow the nursery and allocate new chunks. if (!chunks_.resize(newCount)) - return; + return false; + AutoLockGCBgAlloc lock(runtime()); for (unsigned i = priorCount; i < newCount; i++) { auto newChunk = runtime()->gc.getOrAllocChunk(lock); if (!newChunk) { chunks_.shrinkTo(i); - return; + return false; } chunks_[i] = NurseryChunk::fromChunk(newChunk); chunk(i).poisonAndInit(runtime(), JS_FRESH_NURSERY_PATTERN); } + + return true; +} + +void +js::Nursery::freeChunksFrom(unsigned firstFreeChunk) +{ + MOZ_ASSERT(firstFreeChunk < chunks_.length()); + { + AutoLockGC lock(runtime()); + for (unsigned i = firstFreeChunk; i < chunks_.length(); i++) + runtime()->gc.recycleChunk(chunk(i).toChunk(runtime()), lock); + } + chunks_.shrinkTo(firstFreeChunk); +} + +void +js::Nursery::shrinkAllocableSpace(unsigned newCount) +{ +#ifdef JS_GC_ZEAL + if (runtime()->hasZealMode(ZealMode::GenerationalGC)) + return; +#endif + + // Don't shrink the nursery to zero (use Nursery::disable() instead) and + // don't attempt to shrink it to the same size. + if ((newCount == 0) || (newCount == numChunks())) + return; + + MOZ_ASSERT(newCount < numChunks()); + + freeChunksFrom(newCount); +} + +void +js::Nursery::minimizeAllocableSpace() +{ + shrinkAllocableSpace(1); +} + +bool +js::Nursery::allocateFirstChunk(AutoLockGCBgAlloc& lock) +{ + // This assertion isn't required for correctness, but we do assume this + // is only called to initialize or re-enable the nursery. + MOZ_ASSERT(numChunks() == 0); + + MOZ_ASSERT(maxChunks() > 0); + + if (!chunks_.resize(1)) + return false; + + auto chunk = runtime()->gc.getOrAllocChunk(lock); + if (!chunk) { + chunks_.shrinkTo(0); + return false; + } + + chunks_[0] = NurseryChunk::fromChunk(chunk); + + return true; } bool diff --git a/js/src/gc/Nursery.h b/js/src/gc/Nursery.h index 8d20f193d2b9..96ef0ffa9074 100644 --- a/js/src/gc/Nursery.h +++ b/js/src/gc/Nursery.h @@ -417,9 +417,11 @@ class Nursery void setCurrentChunk(unsigned chunkno); void setStartPosition(); - void updateNumChunks(unsigned newCount); - void updateNumChunksLocked(unsigned newCount, - AutoLockGCBgAlloc& lock); + /* + * Ensure that the first chunk has been allocated. Callers will probably + * want to call setCurrentChunk(0) next. + */ + MOZ_MUST_USE bool allocateFirstChunk(AutoLockGCBgAlloc& lock); MOZ_ALWAYS_INLINE uintptr_t currentEnd() const; @@ -471,10 +473,15 @@ class Nursery /* Change the allocable space provided by the nursery. */ void maybeResizeNursery(JS::gcreason::Reason reason); - void growAllocableSpace(); - void shrinkAllocableSpace(unsigned removeNumChunks); + bool growAllocableSpace(); + bool growAllocableSpace(unsigned newSize); + void shrinkAllocableSpace(unsigned newCount); void minimizeAllocableSpace(); + // Free the chunks starting at firstFreeChunk until the end of the chunks + // vector. Shrinks the vector but does not update maxChunksAlloc(). + void freeChunksFrom(unsigned firstFreeChunk); + /* Profile recording and printing. */ void maybeClearProfileDurations(); void startProfile(ProfileKey key); From f2c7ee541b035890298ee5f3504ec5a9918ca92b Mon Sep 17 00:00:00 2001 From: Paul Bone Date: Mon, 30 Oct 2017 14:44:12 +1100 Subject: [PATCH 25/32] Bug 1298018 - Part 4: Lazily allocate nursery chunks. r=jonco --- js/src/gc/Nursery.cpp | 135 ++++++++++++++++++++++++------------------ js/src/gc/Nursery.h | 41 +++++++++---- 2 files changed, 109 insertions(+), 67 deletions(-) diff --git a/js/src/gc/Nursery.cpp b/js/src/gc/Nursery.cpp index a0ab5471466d..e256e85a55ea 100644 --- a/js/src/gc/Nursery.cpp +++ b/js/src/gc/Nursery.cpp @@ -117,7 +117,8 @@ js::Nursery::Nursery(JSRuntime* rt) , currentStartPosition_(0) , currentEnd_(0) , currentChunk_(0) - , maxNurseryChunks_(0) + , maxChunkCount_(0) + , chunkCountLimit_(0) , previousPromotionRate_(0) , profileThreshold_(0) , enableProfiling_(false) @@ -141,10 +142,10 @@ js::Nursery::init(uint32_t maxNurseryBytes, AutoLockGCBgAlloc& lock) return false; /* maxNurseryBytes parameter is rounded down to a multiple of chunk size. */ - maxNurseryChunks_ = maxNurseryBytes >> ChunkShift; + chunkCountLimit_ = maxNurseryBytes >> ChunkShift; /* If no chunks are specified then the nursery is permanently disabled. */ - if (maxNurseryChunks_ == 0) + if (chunkCountLimit_ == 0) return true; if (!allocateFirstChunk(lock)) @@ -192,7 +193,7 @@ js::Nursery::enable() { MOZ_ASSERT(isEmpty()); MOZ_ASSERT(!runtime()->gc.isVerifyPreBarriersEnabled()); - if (isEnabled() || !maxChunks()) + if (isEnabled() || !chunkCountLimit()) return; { @@ -219,7 +220,10 @@ js::Nursery::disable() return; freeChunksFrom(0); + maxChunkCount_ = 0; + currentEnd_ = 0; + runtime()->gc.storeBuffer().disable(); } @@ -240,7 +244,7 @@ js::Nursery::isEmpty() const void js::Nursery::enterZealMode() { if (isEnabled()) - growAllocableSpace(maxChunks()); + maxChunkCount_ = chunkCountLimit(); } void @@ -306,9 +310,18 @@ js::Nursery::allocate(size_t size) #endif if (currentEnd() < position() + size) { - if (currentChunk_ + 1 == numChunks()) + unsigned chunkno = currentChunk_ + 1; + MOZ_ASSERT(chunkno <= chunkCountLimit()); + MOZ_ASSERT(chunkno <= maxChunkCount()); + MOZ_ASSERT(chunkno <= allocatedChunkCount()); + if (chunkno == maxChunkCount()) return nullptr; - setCurrentChunk(currentChunk_ + 1); + if (MOZ_UNLIKELY(chunkno == allocatedChunkCount())) { + if (!allocateNextChunk(chunkno)) + return nullptr; + MOZ_ASSERT(chunkno < allocatedChunkCount()); + } + setCurrentChunk(chunkno); } void* thing = (void*)position(); @@ -515,7 +528,8 @@ js::Nursery::renderProfileJSON(JSONPrinter& json) const json.property("bytes_tenured", previousGC.tenuredBytes); json.floatProperty("promotion_rate", calcPromotionRate(nullptr), 0); json.property("nursery_bytes", previousGC.nurseryUsedBytes); - json.property("new_nursery_bytes", numChunks() * ChunkSize); + json.property("new_nursery_bytes", maxChunkCount() * ChunkSize); + json.property("new_nursery_bytes_alloc", allocatedChunkCount() * ChunkSize); json.beginObjectProperty("timings"); @@ -678,7 +692,7 @@ js::Nursery::collect(JS::gcreason::Reason reason) // Disable the nursery if the user changed the configuration setting. The // nursery can only be re-enabled by resetting the configurationa and // restarting firefox. - if (maxNurseryChunks_ == 0) + if (chunkCountLimit_ == 0) disable(); endProfile(ProfileKey::Total); @@ -701,7 +715,7 @@ js::Nursery::collect(JS::gcreason::Reason reason) fprintf(stderr, "MinorGC: %20s %5.1f%% %4u ", JS::gcreason::ExplainReason(reason), promotionRate * 100, - numChunks()); + maxChunkCount()); printProfileDurations(profileDurations_); if (reportTenurings_) { @@ -906,18 +920,18 @@ js::Nursery::clear() { #ifdef JS_GC_ZEAL /* Poison the nursery contents so touching a freed object will crash. */ - for (unsigned i = 0; i < numChunks(); i++) + for (unsigned i = 0; i < allocatedChunkCount(); i++) chunk(i).poisonAndInit(runtime(), JS_SWEPT_NURSERY_PATTERN); if (runtime()->hasZealMode(ZealMode::GenerationalGC)) { /* Only reset the alloc point when we are close to the end. */ - if (currentChunk_ + 1 == numChunks()) + if (currentChunk_ + 1 == maxChunkCount()) setCurrentChunk(0); } else #endif { #ifdef JS_CRASH_DIAGNOSTICS - for (unsigned i = 0; i < numChunks(); ++i) + for (unsigned i = 0; i < allocatedChunkCount(); ++i) chunk(i).poisonAndInit(runtime(), JS_SWEPT_NURSERY_PATTERN); #endif setCurrentChunk(0); @@ -930,7 +944,7 @@ js::Nursery::clear() size_t js::Nursery::spaceToEnd() const { - unsigned lastChunk = numChunks() - 1; + unsigned lastChunk = maxChunkCount() - 1; MOZ_ASSERT(lastChunk >= currentStartChunk_); MOZ_ASSERT(currentStartPosition_ - chunk(currentStartChunk_).start() <= NurseryChunkUsableSize); @@ -938,7 +952,7 @@ js::Nursery::spaceToEnd() const size_t bytes = (chunk(currentStartChunk_).end() - currentStartPosition_) + ((lastChunk - currentStartChunk_) * NurseryChunkUsableSize); - MOZ_ASSERT(bytes <= numChunks() * NurseryChunkUsableSize); + MOZ_ASSERT(bytes <= maxChunkCount() * NurseryChunkUsableSize); return bytes; } @@ -946,14 +960,45 @@ js::Nursery::spaceToEnd() const MOZ_ALWAYS_INLINE void js::Nursery::setCurrentChunk(unsigned chunkno) { - MOZ_ASSERT(chunkno < maxChunks()); - MOZ_ASSERT(chunkno < numChunks()); + MOZ_ASSERT(chunkno < chunkCountLimit()); + MOZ_ASSERT(chunkno < allocatedChunkCount()); currentChunk_ = chunkno; position_ = chunk(chunkno).start(); currentEnd_ = chunk(chunkno).end(); chunk(chunkno).poisonAndInit(runtime(), JS_FRESH_NURSERY_PATTERN); } +bool +js::Nursery::allocateNextChunk(const unsigned chunkno) +{ + if (chunkno == maxChunkCount()) + return false; + + const unsigned priorCount = allocatedChunkCount(); + const unsigned newCount = priorCount + 1; + MOZ_ASSERT(chunkno == currentChunk_ + 1); + MOZ_ASSERT(chunkno == allocatedChunkCount()); + + if (!chunks_.resize(newCount)) + return false; + + MOZ_ASSERT(chunkno < chunkCountLimit()); + MOZ_ASSERT(chunkno < maxChunkCount()); + + Chunk* newChunk; + { + AutoLockGCBgAlloc lock(runtime()); + newChunk = runtime()->gc.getOrAllocChunk(lock); + } + if (!newChunk) { + chunks_.shrinkTo(priorCount); + return false; + } + + chunks_[chunkno] = NurseryChunk::fromChunk(newChunk); + return true; +} + MOZ_ALWAYS_INLINE void js::Nursery::setStartPosition() { @@ -990,10 +1035,10 @@ js::Nursery::maybeResizeNursery(JS::gcreason::Reason reason) float(previousGC.tenuredBytes) / float(previousGC.nurseryCapacity); newMaxNurseryChunks = runtime()->gc.tunables.gcMaxNurseryBytes() >> ChunkShift; - if (newMaxNurseryChunks != maxNurseryChunks_) { - maxNurseryChunks_ = newMaxNurseryChunks; + if (newMaxNurseryChunks != chunkCountLimit_) { + chunkCountLimit_ = newMaxNurseryChunks; /* The configured maximum nursery size is changing */ - if (numChunks() > newMaxNurseryChunks) { + if (maxChunkCount() > newMaxNurseryChunks) { /* We need to shrink the nursery */ shrinkAllocableSpace(newMaxNurseryChunks); @@ -1007,43 +1052,16 @@ js::Nursery::maybeResizeNursery(JS::gcreason::Reason reason) // nursery chunks we do not report an error. growAllocableSpace(); } else if (promotionRate < ShrinkThreshold && previousPromotionRate_ < ShrinkThreshold) { - shrinkAllocableSpace(numChunks() - 1); + shrinkAllocableSpace(maxChunkCount() - 1); } previousPromotionRate_ = promotionRate; } -bool +void js::Nursery::growAllocableSpace() { - return growAllocableSpace(Min(numChunks() * 2, maxChunks())); -} - -bool -js::Nursery::growAllocableSpace(unsigned newCount) -{ - unsigned priorCount = numChunks(); - - MOZ_ASSERT(newCount > priorCount); - MOZ_ASSERT(newCount <= maxChunks()); - MOZ_ASSERT(priorCount >= 1); - - if (!chunks_.resize(newCount)) - return false; - - AutoLockGCBgAlloc lock(runtime()); - for (unsigned i = priorCount; i < newCount; i++) { - auto newChunk = runtime()->gc.getOrAllocChunk(lock); - if (!newChunk) { - chunks_.shrinkTo(i); - return false; - } - - chunks_[i] = NurseryChunk::fromChunk(newChunk); - chunk(i).poisonAndInit(runtime(), JS_FRESH_NURSERY_PATTERN); - } - - return true; + maxChunkCount_ = Min(maxChunkCount() * 2, chunkCountLimit()); } void @@ -1068,12 +1086,15 @@ js::Nursery::shrinkAllocableSpace(unsigned newCount) // Don't shrink the nursery to zero (use Nursery::disable() instead) and // don't attempt to shrink it to the same size. - if ((newCount == 0) || (newCount == numChunks())) + if ((newCount == 0) || (newCount == maxChunkCount())) return; - MOZ_ASSERT(newCount < numChunks()); + MOZ_ASSERT(newCount < maxChunkCount()); - freeChunksFrom(newCount); + if (newCount < allocatedChunkCount()) + freeChunksFrom(newCount); + + maxChunkCount_ = newCount; } void @@ -1085,11 +1106,12 @@ js::Nursery::minimizeAllocableSpace() bool js::Nursery::allocateFirstChunk(AutoLockGCBgAlloc& lock) { - // This assertion isn't required for correctness, but we do assume this + // These assertions aren't required for correctness, but we do assume this // is only called to initialize or re-enable the nursery. - MOZ_ASSERT(numChunks() == 0); + MOZ_ASSERT(allocatedChunkCount() == 0); + MOZ_ASSERT(maxChunkCount() == 0); - MOZ_ASSERT(maxChunks() > 0); + MOZ_ASSERT(chunkCountLimit() > 0); if (!chunks_.resize(1)) return false; @@ -1101,6 +1123,7 @@ js::Nursery::allocateFirstChunk(AutoLockGCBgAlloc& lock) } chunks_[0] = NurseryChunk::fromChunk(chunk); + maxChunkCount_ = 1; return true; } diff --git a/js/src/gc/Nursery.h b/js/src/gc/Nursery.h index 96ef0ffa9074..7d07c6adcb74 100644 --- a/js/src/gc/Nursery.h +++ b/js/src/gc/Nursery.h @@ -141,14 +141,22 @@ class Nursery MOZ_MUST_USE bool init(uint32_t maxNurseryBytes, AutoLockGCBgAlloc& lock); - unsigned maxChunks() const { return maxNurseryChunks_; } - unsigned numChunks() const { return chunks_.length(); } + unsigned chunkCountLimit() const { return chunkCountLimit_; } - bool exists() const { return maxChunks() != 0; } + // Number of allocated (ready to use) chunks. + unsigned allocatedChunkCount() const { return chunks_.length(); } + + // Total number of chunks and the capacity of the nursery. Chunks will be + // lazilly allocated and added to the chunks array up to this limit, after + // that the nursery must be collected, this limit may be raised during + // collection. + unsigned maxChunkCount() const { return maxChunkCount_; } + + bool exists() const { return chunkCountLimit() != 0; } void enable(); void disable(); - bool isEnabled() const { return numChunks() != 0; } + bool isEnabled() const { return maxChunkCount() != 0; } /* Return true if no allocations have been made since the last collection. */ bool isEmpty() const; @@ -233,7 +241,7 @@ class Nursery MOZ_MUST_USE bool queueDictionaryModeObjectToSweep(NativeObject* obj); size_t sizeOfHeapCommitted() const { - return numChunks() * gc::ChunkSize; + return allocatedChunkCount() * gc::ChunkSize; } size_t sizeOfMallocedBuffers(mozilla::MallocSizeOf mallocSizeOf) const { if (!mallocedBuffers.initialized()) @@ -252,7 +260,7 @@ class Nursery MOZ_ALWAYS_INLINE size_t freeSpace() const { MOZ_ASSERT(currentEnd_ - position_ <= NurseryChunkUsableSize); return (currentEnd_ - position_) + - (numChunks() - currentChunk_ - 1) * NurseryChunkUsableSize; + (maxChunkCount() - currentChunk_ - 1) * NurseryChunkUsableSize; } #ifdef JS_GC_ZEAL @@ -302,8 +310,18 @@ class Nursery /* The index of the chunk that is currently being allocated from. */ unsigned currentChunk_; - /* Maximum number of chunks to allocate for the nursery. */ - unsigned maxNurseryChunks_; + /* + * The nursery may grow the chunks_ vector up to this size without a + * collection. This allows the nursery to grow lazilly. This limit may + * change during maybeResizeNursery() each collection. + */ + unsigned maxChunkCount_; + + /* + * This limit is fixed by configuration. It represents the maximum size + * the nursery is permitted to tune itself to in maybeResizeNursery(); + */ + unsigned chunkCountLimit_; /* Promotion rate for the previous minor collection. */ float previousPromotionRate_; @@ -423,6 +441,8 @@ class Nursery */ MOZ_MUST_USE bool allocateFirstChunk(AutoLockGCBgAlloc& lock); + MOZ_MUST_USE bool allocateNextChunk(unsigned chunkno); + MOZ_ALWAYS_INLINE uintptr_t currentEnd() const; uintptr_t position() const { return position_; } @@ -473,13 +493,12 @@ class Nursery /* Change the allocable space provided by the nursery. */ void maybeResizeNursery(JS::gcreason::Reason reason); - bool growAllocableSpace(); - bool growAllocableSpace(unsigned newSize); + void growAllocableSpace(); void shrinkAllocableSpace(unsigned newCount); void minimizeAllocableSpace(); // Free the chunks starting at firstFreeChunk until the end of the chunks - // vector. Shrinks the vector but does not update maxChunksAlloc(). + // vector. Shrinks the vector but does not update maxChunkCount(). void freeChunksFrom(unsigned firstFreeChunk); /* Profile recording and printing. */ From a48b9d6d4b74a0e2c855a4a5a7e349fac42a2188 Mon Sep 17 00:00:00 2001 From: Paul Bone Date: Mon, 30 Oct 2017 17:46:23 +1100 Subject: [PATCH 26/32] Bug 1298018 - Part 5: Remove allocateFirstChunk(). r=jonco --- js/src/gc/Nursery.cpp | 44 ++++++++++--------------------------------- js/src/gc/Nursery.h | 9 ++++----- 2 files changed, 14 insertions(+), 39 deletions(-) diff --git a/js/src/gc/Nursery.cpp b/js/src/gc/Nursery.cpp index e256e85a55ea..45f9ffc81bdf 100644 --- a/js/src/gc/Nursery.cpp +++ b/js/src/gc/Nursery.cpp @@ -148,7 +148,8 @@ js::Nursery::init(uint32_t maxNurseryBytes, AutoLockGCBgAlloc& lock) if (chunkCountLimit_ == 0) return true; - if (!allocateFirstChunk(lock)) + maxChunkCount_ = 1; + if (!allocateNextChunk(0, lock)) return false; setCurrentChunk(0); @@ -198,7 +199,8 @@ js::Nursery::enable() { AutoLockGCBgAlloc lock(runtime()); - if (!allocateFirstChunk(lock)) + maxChunkCount_ = 1; + if (!allocateNextChunk(0, lock)) return; } @@ -317,7 +319,8 @@ js::Nursery::allocate(size_t size) if (chunkno == maxChunkCount()) return nullptr; if (MOZ_UNLIKELY(chunkno == allocatedChunkCount())) { - if (!allocateNextChunk(chunkno)) + AutoLockGCBgAlloc lock(runtime()); + if (!allocateNextChunk(chunkno, lock)) return nullptr; MOZ_ASSERT(chunkno < allocatedChunkCount()); } @@ -969,14 +972,15 @@ js::Nursery::setCurrentChunk(unsigned chunkno) } bool -js::Nursery::allocateNextChunk(const unsigned chunkno) +js::Nursery::allocateNextChunk(const unsigned chunkno, + AutoLockGCBgAlloc& lock) { if (chunkno == maxChunkCount()) return false; const unsigned priorCount = allocatedChunkCount(); const unsigned newCount = priorCount + 1; - MOZ_ASSERT(chunkno == currentChunk_ + 1); + MOZ_ASSERT((chunkno == currentChunk_ + 1) || (chunkno == 0 && allocatedChunkCount() == 0)); MOZ_ASSERT(chunkno == allocatedChunkCount()); if (!chunks_.resize(newCount)) @@ -986,10 +990,7 @@ js::Nursery::allocateNextChunk(const unsigned chunkno) MOZ_ASSERT(chunkno < maxChunkCount()); Chunk* newChunk; - { - AutoLockGCBgAlloc lock(runtime()); - newChunk = runtime()->gc.getOrAllocChunk(lock); - } + newChunk = runtime()->gc.getOrAllocChunk(lock); if (!newChunk) { chunks_.shrinkTo(priorCount); return false; @@ -1103,31 +1104,6 @@ js::Nursery::minimizeAllocableSpace() shrinkAllocableSpace(1); } -bool -js::Nursery::allocateFirstChunk(AutoLockGCBgAlloc& lock) -{ - // These assertions aren't required for correctness, but we do assume this - // is only called to initialize or re-enable the nursery. - MOZ_ASSERT(allocatedChunkCount() == 0); - MOZ_ASSERT(maxChunkCount() == 0); - - MOZ_ASSERT(chunkCountLimit() > 0); - - if (!chunks_.resize(1)) - return false; - - auto chunk = runtime()->gc.getOrAllocChunk(lock); - if (!chunk) { - chunks_.shrinkTo(0); - return false; - } - - chunks_[0] = NurseryChunk::fromChunk(chunk); - maxChunkCount_ = 1; - - return true; -} - bool js::Nursery::queueDictionaryModeObjectToSweep(NativeObject* obj) { diff --git a/js/src/gc/Nursery.h b/js/src/gc/Nursery.h index 7d07c6adcb74..3bc9ba85185c 100644 --- a/js/src/gc/Nursery.h +++ b/js/src/gc/Nursery.h @@ -436,12 +436,11 @@ class Nursery void setStartPosition(); /* - * Ensure that the first chunk has been allocated. Callers will probably - * want to call setCurrentChunk(0) next. + * Allocate the next chunk, or the first chunk for initialization. + * Callers will probably want to call setCurrentChunk(0) next. */ - MOZ_MUST_USE bool allocateFirstChunk(AutoLockGCBgAlloc& lock); - - MOZ_MUST_USE bool allocateNextChunk(unsigned chunkno); + MOZ_MUST_USE bool allocateNextChunk(unsigned chunkno, + AutoLockGCBgAlloc& lock); MOZ_ALWAYS_INLINE uintptr_t currentEnd() const; From 046d7cd77a886113c7ac810ca5ef922a8dc9ecc2 Mon Sep 17 00:00:00 2001 From: Phil Ringnalda Date: Mon, 30 Oct 2017 20:55:37 -0700 Subject: [PATCH 27/32] Backed out changeset 349b9517cb9b (bug 1403131) for wpt, talos, android, and marionette failures CLOSED TREE MozReview-Commit-ID: GEyvPM0mvkR --- .../mozharness/mozilla/blob_upload.py | 25 +++--- .../mozharness/mozharness/mozilla/buildbot.py | 33 +++---- .../mozharness/mozilla/building/buildbase.py | 63 +++++++------- .../mozharness/mozilla/building/hazards.py | 60 ++++++------- .../mozharness/mozilla/checksums.py | 6 +- .../mozharness/mozilla/l10n/locales.py | 1 + .../mozilla/l10n/multi_locale_build.py | 5 +- .../mozharness/mozharness/mozilla/mapper.py | 6 +- .../mozharness/mozharness/mozilla/merkle.py | 8 +- testing/mozharness/mozharness/mozilla/mock.py | 1 + .../mozharness/mozharness/mozilla/purge.py | 4 +- .../mozharness/mozharness/mozilla/release.py | 2 + .../mozharness/mozilla/repo_manifest.py | 3 +- .../mozharness/mozilla/repo_manipulation.py | 2 + .../mozharness/mozharness/mozilla/secrets.py | 3 + .../mozharness/mozilla/selfserve.py | 1 - .../mozilla/signed_certificate_timestamp.py | 5 +- .../mozharness/mozilla/taskcluster_helper.py | 7 +- .../mozilla/testing/codecoverage.py | 30 +++---- .../mozharness/mozilla/testing/device.py | 17 ++-- .../mozharness/mozilla/testing/errors.py | 40 +++------ .../mozilla/testing/firefox_ui_tests.py | 9 +- .../mozharness/mozilla/testing/mozpool.py | 33 +++---- .../mozharness/mozilla/testing/talos.py | 55 ++++-------- .../mozharness/mozilla/testing/testbase.py | 87 +++++++++---------- .../mozharness/mozilla/testing/try_tools.py | 14 ++- .../mozharness/mozilla/testing/unittest.py | 10 ++- .../mozilla/testing/verify_tools.py | 45 +++++----- .../mozharness/mozilla/updates/balrog.py | 13 ++- tools/lint/flake8.yml | 1 - 30 files changed, 255 insertions(+), 334 deletions(-) diff --git a/testing/mozharness/mozharness/mozilla/blob_upload.py b/testing/mozharness/mozharness/mozilla/blob_upload.py index 71e9a5472157..1607ddf994ad 100644 --- a/testing/mozharness/mozharness/mozilla/blob_upload.py +++ b/testing/mozharness/mozharness/mozilla/blob_upload.py @@ -12,12 +12,14 @@ from mozharness.base.script import PostScriptRun blobupload_config_options = [ [["--blob-upload-branch"], - {"dest": "blob_upload_branch", - "help": "Branch for blob server's metadata", }], + {"dest": "blob_upload_branch", + "help": "Branch for blob server's metadata", + }], [["--blob-upload-server"], - {"dest": "blob_upload_servers", - "action": "extend", - "help": "Blob servers's location", }] + {"dest": "blob_upload_servers", + "action": "extend", + "help": "Blob servers's location", + }] ] @@ -44,7 +46,7 @@ class BlobUploadMixin(VirtualenvMixin): if self.config.get('blob_upload_branch') and \ (self.config.get('blob_upload_servers') or self.config.get('default_blob_upload_servers')) and \ - self.config.get('blob_uploader_auth_file'): + self.config.get('blob_uploader_auth_file'): self.info("Blob upload gear active.") upload = [self.query_python_path(), self.query_python_path("blobberc.py")] @@ -73,7 +75,7 @@ class BlobUploadMixin(VirtualenvMixin): return blob_branch = self.config.get('blob_upload_branch') blob_servers_list = self.config.get('blob_upload_servers', - self.config.get('default_blob_upload_servers')) + self.config.get('default_blob_upload_servers')) servers = [] for server in blob_servers_list: @@ -85,12 +87,11 @@ class BlobUploadMixin(VirtualenvMixin): manifest_path = os.path.join(dirs['abs_work_dir'], 'uploaded_files.json') record_uploaded_files = ['--output-manifest', manifest_path] self.info("Files from %s are to be uploaded with <%s> branch at " - "the following location(s): %s" % - (blob_dir, blob_branch, ", ".join(["%s" % s for s in blob_servers_list]))) + "the following location(s): %s" % (blob_dir, blob_branch, + ", ".join(["%s" % s for s in blob_servers_list]))) # call blob client to upload files to server - self.run_command(upload + servers + auth + branch + - dir_to_upload + record_uploaded_files) + self.run_command(upload + servers + auth + branch + dir_to_upload + record_uploaded_files) uploaded_files = '{}' if os.path.isfile(manifest_path): @@ -99,7 +100,7 @@ class BlobUploadMixin(VirtualenvMixin): self.rmtree(manifest_path) self.set_buildbot_property(prop_name='blobber_files', - prop_value=uploaded_files, write_to_file=True) + prop_value=uploaded_files, write_to_file=True) else: self.warning("Blob upload gear skipped. Missing cmdline options.") diff --git a/testing/mozharness/mozharness/mozilla/buildbot.py b/testing/mozharness/mozharness/mozilla/buildbot.py index 79e8aacc4dcc..e17343633b27 100755 --- a/testing/mozharness/mozharness/mozilla/buildbot.py +++ b/testing/mozharness/mozharness/mozilla/buildbot.py @@ -88,17 +88,13 @@ class BuildbotMixin(object): if os.path.exists(log_file): file_size = os.path.getsize(log_file) if file_size > self.config['buildbot_max_log_size']: - self.error("Log file size %d is greater than max allowed %d! Setting " - "TBPL_FAILURE (was %s)..." % - (file_size, self.config['buildbot_max_log_size'], tbpl_status)) + self.error("Log file size %d is greater than max allowed %d! Setting TBPL_FAILURE (was %s)..." % (file_size, self.config['buildbot_max_log_size'], tbpl_status)) tbpl_status = TBPL_FAILURE if not level: level = TBPL_STATUS_DICT[tbpl_status] - self.worst_buildbot_status = self.worst_level(tbpl_status, self.worst_buildbot_status, - TBPL_WORST_LEVEL_TUPLE) + self.worst_buildbot_status = self.worst_level(tbpl_status, self.worst_buildbot_status, TBPL_WORST_LEVEL_TUPLE) if self.worst_buildbot_status != tbpl_status: - self.info("Current worst status %s is worse; keeping it." % - self.worst_buildbot_status) + self.info("Current worst status %s is worse; keeping it." % self.worst_buildbot_status) self.add_summary("# TBPL %s #" % self.worst_buildbot_status, level=level) if set_return_code: self.return_code = EXIT_STATUS_DICT[self.worst_buildbot_status] @@ -141,8 +137,7 @@ class BuildbotMixin(object): self.info("Writing buildbot properties to %s" % file_name) else: if not isinstance(prop_list, (list, tuple)): - self.log("dump_buildbot_properties: Can't dump non-list prop_list %s!" % - str(prop_list), level=error_level) + self.log("dump_buildbot_properties: Can't dump non-list prop_list %s!" % str(prop_list), level=error_level) return self.info("Writing buildbot properties %s to %s" % (str(prop_list), file_name)) contents = "" @@ -151,7 +146,7 @@ class BuildbotMixin(object): return self.write_to_file(file_name, contents) def invoke_sendchange(self, downloadables=None, branch=None, - username="sendchange-unittest", sendchange_props=None): + username="sendchange-unittest", sendchange_props=None): """ Generic sendchange, currently b2g- and unittest-specific. """ c = self.config @@ -159,11 +154,9 @@ class BuildbotMixin(object): if branch is None: if c.get("debug_build"): platform = re.sub('[_-]debug', '', self.buildbot_config["properties"]["platform"]) - branch = '%s-%s-debug-unittest' % (self.buildbot_config["properties"]["branch"], - platform) + branch = '%s-%s-debug-unittest' % (self.buildbot_config["properties"]["branch"], platform) else: - branch = '%s-%s-opt-unittest' % (self.buildbot_config["properties"]["branch"], - self.buildbot_config["properties"]["platform"]) + branch = '%s-%s-opt-unittest' % (self.buildbot_config["properties"]["branch"], self.buildbot_config["properties"]["platform"]) sendchange = [ 'sendchange', '--master', c.get("sendchange_masters")[0], @@ -174,18 +167,15 @@ class BuildbotMixin(object): sendchange += ['-r', self.buildbot_config['sourcestamp']["revision"]] if len(self.buildbot_config['sourcestamp']['changes']) > 0: if self.buildbot_config['sourcestamp']['changes'][0].get('who'): - sendchange += ['--username', - self.buildbot_config['sourcestamp']['changes'][0]['who']] + sendchange += ['--username', self.buildbot_config['sourcestamp']['changes'][0]['who']] if self.buildbot_config['sourcestamp']['changes'][0].get('comments'): - sendchange += ['--comments', self.buildbot_config['sourcestamp'] - ['changes'][0]['comments'].encode('ascii', 'ignore')] + sendchange += ['--comments', self.buildbot_config['sourcestamp']['changes'][0]['comments'].encode('ascii', 'ignore')] if sendchange_props: for key, value in sendchange_props.iteritems(): sendchange.extend(['--property', '%s:%s' % (key, value)]) else: if self.buildbot_config["properties"].get("builduid"): - sendchange += ['--property', "builduid:%s" % - self.buildbot_config["properties"]["builduid"]] + sendchange += ['--property', "builduid:%s" % self.buildbot_config["properties"]["builduid"]] sendchange += [ '--property', "buildid:%s" % self.query_buildid(), '--property', 'pgo_build:False', @@ -196,8 +186,7 @@ class BuildbotMixin(object): retcode = self.run_command(buildbot + sendchange) if retcode != 0: - self.info("The sendchange failed but we don't want to turn the build orange: %s" % - retcode) + self.info("The sendchange failed but we don't want to turn the build orange: %s" % retcode) def query_build_name(self): build_name = self.config.get('platform') diff --git a/testing/mozharness/mozharness/mozilla/building/buildbase.py b/testing/mozharness/mozharness/mozilla/building/buildbase.py index f55b310bff5a..abb8e0d41e14 100755 --- a/testing/mozharness/mozharness/mozilla/building/buildbase.py +++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py @@ -34,6 +34,7 @@ from mozharness.mozilla.buildbot import ( BuildbotMixin, EXIT_STATUS_DICT, TBPL_STATUS_DICT, + TBPL_EXCEPTION, TBPL_FAILURE, TBPL_RETRY, TBPL_WARNING, @@ -72,7 +73,7 @@ Skipping run_tooltool...', ERROR_MSGS.update(MOCK_ERROR_MSGS) -# Output Parsers +### Output Parsers TBPL_UPLOAD_ERRORS = [ { @@ -140,8 +141,7 @@ class MakeUploadOutputParser(OutputParser): self.info("Skipping wrong packageUrl: %s" % m) else: if 'completeMarUrl' in self.matches: - self.fatal("Found multiple package URLs. " - "Please update buildbase.py") + self.fatal("Found multiple package URLs. Please update buildbase.py") self.info("Using package as mar file: %s" % m) self.matches['completeMarUrl'] = m u, self.package_filename = os.path.split(m) @@ -155,8 +155,7 @@ class MakeUploadOutputParser(OutputParser): if m: self.matches['completeMarHash'] = m.group(1) self.matches['completeMarSize'] = m.group(2) - self.info("Using package as mar file and found package hash=%s size=%s" % - (m.group(1), m.group(2))) + self.info("Using package as mar file and found package hash=%s size=%s" % (m.group(1), m.group(2))) # now let's check for retry errors which will give log levels: # tbpl status as RETRY and mozharness status as WARNING @@ -260,20 +259,21 @@ class BuildingConfig(BaseConfig): # not matter. ie: you can supply --branch before --build-pool # or vice versa and the hierarchy will not be different - # The order from highest precedence to lowest is: - # There can only be one of these... + #### The order from highest precedence to lowest is: + ## There can only be one of these... # 1) build_pool: this can be either staging, pre-prod, and prod cfgs # 2) branch: eg: mozilla-central, cedar, cypress, etc # 3) build_variant: these could be known like asan and debug # or a custom config - # - # There can be many of these: + ## + ## There can be many of these # 4) all other configs: these are any configs that are passed with # --cfg and --opt-cfg. There order is kept in # which they were passed on the cmd line. This # behaviour is maintains what happens by default # in mozharness - # + ## + #### # so, let's first assign the configs that hold a known position of # importance (1 through 3) @@ -360,8 +360,7 @@ class BuildOptionParser(object): 'code-coverage': 'builds/releng_sub_%s_configs/%s_code_coverage.py', 'source': 'builds/releng_sub_%s_configs/%s_source.py', 'noopt-debug': 'builds/releng_sub_%s_configs/%s_noopt_debug.py', - 'api-16-gradle-dependencies': - 'builds/releng_sub_%s_configs/%s_api_16_gradle_dependencies.py', + 'api-16-gradle-dependencies': 'builds/releng_sub_%s_configs/%s_api_16_gradle_dependencies.py', 'api-16': 'builds/releng_sub_%s_configs/%s_api_16.py', 'api-16-old-id': 'builds/releng_sub_%s_configs/%s_api_16_old_id.py', 'api-16-artifact': 'builds/releng_sub_%s_configs/%s_api_16_artifact.py', @@ -380,7 +379,7 @@ class BuildOptionParser(object): 'android-checkstyle': 'builds/releng_sub_%s_configs/%s_checkstyle.py', 'android-lint': 'builds/releng_sub_%s_configs/%s_lint.py', 'android-findbugs': 'builds/releng_sub_%s_configs/%s_findbugs.py', - 'valgrind': 'builds/releng_sub_%s_configs/%s_valgrind.py', + 'valgrind' : 'builds/releng_sub_%s_configs/%s_valgrind.py', 'artifact': 'builds/releng_sub_%s_configs/%s_artifact.py', 'debug-artifact': 'builds/releng_sub_%s_configs/%s_debug_artifact.py', 'devedition': 'builds/releng_sub_%s_configs/%s_devedition.py', @@ -762,7 +761,8 @@ or run without that action (ie: --no-{action})" # dirs['abs_obj_dir'] can be different from env['MOZ_OBJDIR'] on # mac, and that confuses mach. del env['MOZ_OBJDIR'] - return self.get_output_from_command_m(cmd, cwd=dirs['abs_obj_dir'], env=env) + return self.get_output_from_command_m(cmd, + cwd=dirs['abs_obj_dir'], env=env) else: return None @@ -1165,7 +1165,7 @@ or run without that action (ie: --no-{action})" if 'revision' in self.buildbot_properties: revision = self.buildbot_properties['revision'] elif (self.buildbot_config and - self.buildbot_config.get('sourcestamp', {}).get('revision')): + self.buildbot_config.get('sourcestamp', {}).get('revision')): revision = self.buildbot_config['sourcestamp']['revision'] elif self.buildbot_config and self.buildbot_config.get('revision'): revision = self.buildbot_config['revision'] @@ -1331,7 +1331,7 @@ or run without that action (ie: --no-{action})" 'buildid from application.ini: "%s". buildid from buildbot ' 'properties: "%s"' % (app_ini_buildid, buildbot_buildid) ) - if (app_ini_buildid == buildbot_buildid) is not None: + if app_ini_buildid == buildbot_buildid != None: self.info('buildids match.') else: self.error( @@ -1409,7 +1409,7 @@ or run without that action (ie: --no-{action})" taskid = self.buildbot_config['properties'].get('upload_to_task_id') tc = Taskcluster( branch=self.branch, - rank=pushinfo.pushdate, # Use pushdate as the rank + rank=pushinfo.pushdate, # Use pushdate as the rank client_id=self.client_id, access_token=self.access_token, log_obj=self.log_obj, @@ -1480,8 +1480,7 @@ or run without that action (ie: --no-{action})" # which means we should have uploadFiles. files = self.query_buildbot_property('uploadFiles') or [] if not files: - self.warning('No files from the build system to upload to S3: uploadFiles' - 'property is missing or empty.') + self.warning('No files from the build system to upload to S3: uploadFiles property is missing or empty.') packageName = self.query_buildbot_property('packageFilename') self.info('packageFilename is: %s' % packageName) @@ -1507,7 +1506,7 @@ or run without that action (ie: --no-{action})" property_conditions = [ # key: property name, value: condition ('symbolsUrl', lambda m: m.endswith('crashreporter-symbols.zip') or - m.endswith('crashreporter-symbols-full.zip')), + m.endswith('crashreporter-symbols-full.zip')), ('testsUrl', lambda m: m.endswith(('tests.tar.bz2', 'tests.zip'))), ('robocopApkUrl', lambda m: m.endswith('apk') and 'robocop' in m), ('jsshellUrl', lambda m: 'jsshell-' in m and m.endswith('.zip')), @@ -1524,8 +1523,7 @@ or run without that action (ie: --no-{action})" ] # Also upload our mozharness log files - files.extend([os.path.join(self.log_obj.abs_log_dir, x) - for x in self.log_obj.log_files.values()]) + files.extend([os.path.join(self.log_obj.abs_log_dir, x) for x in self.log_obj.log_files.values()]) # Also upload our buildprops.json file. files.extend([os.path.join(dirs['base_work_dir'], 'buildprops.json')]) @@ -1631,8 +1629,8 @@ or run without that action (ie: --no-{action})" os.path.join(dirs['abs_work_dir'], 'buildprops.json')) if 'MOZILLABUILD' in os.environ: - # We found many issues with intermittent build failures when not invoking - # mach via bash. See bug 1364651 before considering changing. + # We found many issues with intermittent build failures when not invoking mach via bash. + # See bug 1364651 before considering changing. mach = [ os.path.join(os.environ['MOZILLABUILD'], 'msys', 'bin', 'bash.exe'), os.path.join(dirs['abs_src_dir'], 'mach') @@ -1717,8 +1715,7 @@ or run without that action (ie: --no-{action})" cwd=objdir, ) if not package_filename: - self.fatal("Unable to determine the package filename for the multi-l10n build." - "Was trying to run: %s" % package_cmd) + self.fatal("Unable to determine the package filename for the multi-l10n build. Was trying to run: %s" % package_cmd) self.info('Multi-l10n package filename is: %s' % package_filename) @@ -1921,6 +1918,7 @@ or run without that action (ie: --no-{action})" return data + def _load_sccache_stats(self): stats_file = os.path.join( self.query_abs_dirs()['abs_obj_dir'], 'sccache-stats.json' @@ -2036,7 +2034,7 @@ or run without that action (ie: --no-{action})" return alert - if installer.endswith('.apk'): # Android + if installer.endswith('.apk'): # Android yield filter_alert({ "name": "installer size", "value": installer_size, @@ -2187,9 +2185,9 @@ or run without that action (ie: --no-{action})" build_type, 'talos') self.invoke_sendchange(downloadables=[installer_url], - branch=talos_branch, - username='sendchange', - sendchange_props=sendchange_props) + branch=talos_branch, + username='sendchange', + sendchange_props=sendchange_props) elif test_type == 'unittest': # do unittest sendchange if c.get('debug_build'): @@ -2239,10 +2237,11 @@ or run without that action (ie: --no-{action})" if self.config.get('taskcluster_nightly'): env = self.query_mach_build_env(multiLocale=False) props_path = os.path.join(env["UPLOAD_PATH"], - 'balrog_props.json') + 'balrog_props.json') self.generate_balrog_props(props_path) return + def valgrind_test(self): '''Execute mach's valgrind-test for memory leaks''' env = self.query_build_env() @@ -2261,6 +2260,8 @@ or run without that action (ie: --no-{action})" self.fatal("'mach valgrind-test' did not run successfully. Please check " "log for errors.") + + def _post_fatal(self, message=None, exit_code=None): if not self.return_code: # only overwrite return_code if it's 0 self.error('setting return code to 2 because fatal was called') diff --git a/testing/mozharness/mozharness/mozilla/building/hazards.py b/testing/mozharness/mozharness/mozilla/building/hazards.py index ece46edd2a30..6de235f8929b 100644 --- a/testing/mozharness/mozharness/mozilla/building/hazards.py +++ b/testing/mozharness/mozharness/mozilla/building/hazards.py @@ -17,7 +17,6 @@ class HazardError(Exception): def splitlines(self): return str(self).splitlines() - class HazardAnalysis(object): def clobber_shell(self, builder): """Clobber the specially-built JS shell used to run the analysis""" @@ -33,9 +32,9 @@ class HazardAnalysis(object): js_src_dir = os.path.join(dirs['gecko_src'], 'js', 'src') rc = builder.run_command(['autoconf-2.13'], - cwd=js_src_dir, - env=builder.env, - error_list=MakefileErrorList) + cwd=js_src_dir, + env=builder.env, + error_list=MakefileErrorList) if rc != 0: rc = builder.run_command(['autoconf2.13'], cwd=js_src_dir, @@ -45,14 +44,14 @@ class HazardAnalysis(object): raise HazardError("autoconf failed, can't continue.") rc = builder.run_command([os.path.join(js_src_dir, 'configure'), - '--enable-optimize', - '--disable-debug', - '--enable-ctypes', - '--with-system-nspr', - '--without-intl-api'], - cwd=dirs['shell_objdir'], - env=builder.env, - error_list=MakefileErrorList) + '--enable-optimize', + '--disable-debug', + '--enable-ctypes', + '--with-system-nspr', + '--without-intl-api'], + cwd=dirs['shell_objdir'], + env=builder.env, + error_list=MakefileErrorList) if rc != 0: raise HazardError("Configure failed, can't continue.") @@ -61,9 +60,9 @@ class HazardAnalysis(object): dirs = builder.query_abs_dirs() rc = builder.run_command(['make', '-j', str(builder.config.get('concurrency', 4)), '-s'], - cwd=dirs['shell_objdir'], - env=builder.env, - error_list=MakefileErrorList) + cwd=dirs['shell_objdir'], + env=builder.env, + error_list=MakefileErrorList) if rc != 0: raise HazardError("Build failed, can't continue.") @@ -110,8 +109,9 @@ jobs = 4 build_script = builder.config['build_command'] builder.copyfile(os.path.join(dirs['mozharness_scriptdir'], - os.path.join('spidermonkey', build_script)), - os.path.join(analysis_dir, build_script), copystat=True) + os.path.join('spidermonkey', build_script)), + os.path.join(analysis_dir, build_script), + copystat=True) def run(self, builder, env, error_list): """Execute the analysis, which consists of building all analyzed @@ -132,9 +132,9 @@ jobs = 4 "--buildcommand", build_script, ] retval = builder.run_command(cmd, - cwd=analysis_dir, - env=env, - error_list=error_list) + cwd=analysis_dir, + env=env, + error_list=error_list) if retval != 0: raise HazardError("failed to build") @@ -171,8 +171,7 @@ jobs = 4 long_desc=long, compress=False, # blobber will compress upload_dir=upload_dir) - print("== Hazards (temporarily inline here, beware weirdly interleaved " - "output, see bug 1211402) ==") + print("== Hazards (temporarily inline here, beware weirdly interleaved output, see bug 1211402) ==") print(file(os.path.join(analysis_dir, "hazards.txt")).read()) def upload_results(self, builder): @@ -187,8 +186,7 @@ jobs = 4 dirs = builder.query_abs_dirs() analysis_dir = dirs['abs_analysis_dir'] - analysis_scriptdir = os.path.join(dirs['gecko_src'], 'js', 'src', - 'devtools', 'rootAnalysis') + analysis_scriptdir = os.path.join(dirs['gecko_src'], 'js', 'src', 'devtools', 'rootAnalysis') expect_file = os.path.join(analysis_scriptdir, builder.config['expect_file']) expect = builder.read_from_file(expect_file) if expect is None: @@ -218,13 +216,12 @@ jobs = 4 if expect_hazards is not None and expect_hazards != num_hazards: if expect_hazards < num_hazards: - builder.warning("TEST-UNEXPECTED-FAIL %d more hazards than expected " - "(expected %d, saw %d)" % - (num_hazards - expect_hazards, expect_hazards, num_hazards)) + builder.warning("TEST-UNEXPECTED-FAIL %d more hazards than expected (expected %d, saw %d)" % + (num_hazards - expect_hazards, expect_hazards, num_hazards)) builder.buildbot_status(TBPL_WARNING) else: builder.info("%d fewer hazards than expected! (expected %d, saw %d)" % - (expect_hazards - num_hazards, expect_hazards, num_hazards)) + (expect_hazards - num_hazards, expect_hazards, num_hazards)) expect_refs = data.get('expect-refs') if expect_refs is None: @@ -234,12 +231,11 @@ jobs = 4 if expect_refs is not None and expect_refs != num_refs: if expect_refs < num_refs: - builder.warning("TEST-UNEXPECTED-FAIL %d more unsafe refs than expected " - "(expected %d, saw %d)" % - (num_refs - expect_refs, expect_refs, num_refs)) + builder.warning("TEST-UNEXPECTED-FAIL %d more unsafe refs than expected (expected %d, saw %d)" % + (num_refs - expect_refs, expect_refs, num_refs)) builder.buildbot_status(TBPL_WARNING) else: builder.info("%d fewer unsafe refs than expected! (expected %d, saw %d)" % - (expect_refs - num_refs, expect_refs, num_refs)) + (expect_refs - num_refs, expect_refs, num_refs)) builder.info("TinderboxPrint: " + ", ".join(status)) diff --git a/testing/mozharness/mozharness/mozilla/checksums.py b/testing/mozharness/mozharness/mozilla/checksums.py index 718db9d19a5e..6b899737567c 100644 --- a/testing/mozharness/mozharness/mozilla/checksums.py +++ b/testing/mozharness/mozharness/mozilla/checksums.py @@ -12,12 +12,10 @@ def parse_checksums_file(checksums): # If the file already exists, make sure that the size matches the # previous entry. elif fileInfo[file_]['size'] != size: - raise ValueError("Found different sizes for same file %s (%s and %s)" % - (file_, fileInfo[file_]['size'], size)) + raise ValueError("Found different sizes for same file %s (%s and %s)" % (file_, fileInfo[file_]['size'], size)) # Same goes for the hash. elif type_ in fileInfo[file_]['hashes'] and fileInfo[file_]['hashes'][type_] != hash_: - raise ValueError("Found different %s hashes for same file %s (%s and %s)" % - (type_, file_, fileInfo[file_]['hashes'][type_], hash_)) + raise ValueError("Found different %s hashes for same file %s (%s and %s)" % (type_, file_, fileInfo[file_]['hashes'][type_], hash_)) fileInfo[file_]['size'] = size fileInfo[file_]['hashes'][type_] = hash_ return fileInfo diff --git a/testing/mozharness/mozharness/mozilla/l10n/locales.py b/testing/mozharness/mozharness/mozilla/l10n/locales.py index dc94374a6bc5..b06084309744 100755 --- a/testing/mozharness/mozharness/mozilla/l10n/locales.py +++ b/testing/mozharness/mozharness/mozilla/l10n/locales.py @@ -8,6 +8,7 @@ """ import os +from urlparse import urljoin import pprint import sys from copy import deepcopy diff --git a/testing/mozharness/mozharness/mozilla/l10n/multi_locale_build.py b/testing/mozharness/mozharness/mozilla/l10n/multi_locale_build.py index 34c974f6a615..aa476ce7941f 100755 --- a/testing/mozharness/mozharness/mozilla/l10n/multi_locale_build.py +++ b/testing/mozharness/mozharness/mozilla/l10n/multi_locale_build.py @@ -135,6 +135,7 @@ class MultiLocaleBuild(LocalesMixin, MercurialScript): self.fatal("Erroring out after the build failed.") def add_locales(self): + c = self.config dirs = self.query_abs_dirs() locales = self.query_locales() @@ -214,8 +215,7 @@ class MultiLocaleBuild(LocalesMixin, MercurialScript): rsync = self.query_exe('rsync') backup_dir = '%s-bak' % dirs['abs_objdir'] if not os.path.isdir(dirs['abs_objdir']) or not os.path.isdir(backup_dir): - self.warning("Both %s and %s need to exist to restore the objdir! Skipping..." % - (dirs['abs_objdir'], backup_dir)) + self.warning("Both %s and %s need to exist to restore the objdir! Skipping..." % (dirs['abs_objdir'], backup_dir)) return self.run_command([rsync, '-a', '--delete', '--partial', '%s/' % backup_dir, @@ -233,7 +233,6 @@ class MultiLocaleBuild(LocalesMixin, MercurialScript): """ return self.run_command(**kwargs) - # __main__ {{{1 if __name__ == '__main__': pass diff --git a/testing/mozharness/mozharness/mozilla/mapper.py b/testing/mozharness/mozharness/mozilla/mapper.py index 855e077a1164..c5a2d48959e1 100644 --- a/testing/mozharness/mozharness/mozilla/mapper.py +++ b/testing/mozharness/mozharness/mozilla/mapper.py @@ -51,11 +51,9 @@ class MapperMixin: j = json.loads(r.readline()) if j['%s_rev' % vcs] is None: if require_answer: - raise Exception("Mapper returned a revision of None; " - "maybe it needs more time.") + raise Exception("Mapper returned a revision of None; maybe it needs more time.") else: - self.warning("Mapper returned a revision of None. " - "Accepting because require_answer is False.") + self.warning("Mapper returned a revision of None. Accepting because require_answer is False.") return j['%s_rev' % vcs] except Exception, err: self.warning('Error: %s' % str(err)) diff --git a/testing/mozharness/mozharness/mozilla/merkle.py b/testing/mozharness/mozharness/mozilla/merkle.py index 0ff9d6cf5003..ed53f8cd798c 100644 --- a/testing/mozharness/mozharness/mozilla/merkle.py +++ b/testing/mozharness/mozharness/mozilla/merkle.py @@ -2,22 +2,18 @@ import struct - def _round2(n): k = 1 while k < n: k <<= 1 return k >> 1 - def _leaf_hash(hash_fn, leaf): return hash_fn(b'\x00' + leaf).digest() - def _pair_hash(hash_fn, left, right): return hash_fn(b'\x01' + left + right).digest() - class InclusionProof: """ Represents a Merkle inclusion proof for purposes of serialization, @@ -60,7 +56,7 @@ class InclusionProof: raise Exception('Inclusion proof too short for log ID header') log_id_len, = struct.unpack('B', serialized[start:start+read]) start += read - start += log_id_len # Ignore the log ID itself + start += log_id_len # Ignore the log ID itself read = 8 + 8 + 2 if len(serialized) < start + read: @@ -122,10 +118,10 @@ class InclusionProof: return node + def verify(self, hash_fn, leaf, leaf_index, tree_size, tree_head): return self._expected_head(hash_fn, leaf, leaf_index, tree_size) == tree_head - class MerkleTree: """ Implements a Merkle tree on a set of data items following the diff --git a/testing/mozharness/mozharness/mozilla/mock.py b/testing/mozharness/mozharness/mozilla/mock.py index 35f45ddb6202..f8587c0d66a9 100644 --- a/testing/mozharness/mozharness/mozilla/mock.py +++ b/testing/mozharness/mozharness/mozilla/mock.py @@ -18,6 +18,7 @@ Nothing to remove.' } + # MockMixin {{{1 class MockMixin(object): """Provides methods to setup and interact with mock environments. diff --git a/testing/mozharness/mozharness/mozilla/purge.py b/testing/mozharness/mozharness/mozilla/purge.py index 84db900fb3c6..ca80bd4ff81b 100644 --- a/testing/mozharness/mozharness/mozilla/purge.py +++ b/testing/mozharness/mozharness/mozilla/purge.py @@ -68,8 +68,8 @@ class PurgeMixin(object): }] retval = self.retry(self.run_command, attempts=3, good_statuses=(0,), args=[cmd], - kwargs={'cwd': os.path.dirname(dirs['base_work_dir']), - 'error_list': error_list}) + kwargs={'cwd':os.path.dirname(dirs['base_work_dir']), + 'error_list':error_list}) if retval != 0: self.fatal("failed to clobber build", exit_code=2) diff --git a/testing/mozharness/mozharness/mozilla/release.py b/testing/mozharness/mozharness/mozilla/release.py index 9463f7ad4458..52a84cdba31e 100755 --- a/testing/mozharness/mozharness/mozilla/release.py +++ b/testing/mozharness/mozharness/mozilla/release.py @@ -68,3 +68,5 @@ def get_previous_version(version, partial_versions): composed = sorted([(v, StrictVersion(v)) for v in partial_versions if v != version], key=lambda x: x[1], reverse=True) return composed[0][0] + + diff --git a/testing/mozharness/mozharness/mozilla/repo_manifest.py b/testing/mozharness/mozharness/mozilla/repo_manifest.py index a5b7915f83a0..2ffb34fe9707 100644 --- a/testing/mozharness/mozharness/mozilla/repo_manifest.py +++ b/testing/mozharness/mozharness/mozilla/repo_manifest.py @@ -24,8 +24,7 @@ def load_manifest(filename): if node.nodeType in (node.TEXT_NODE, node.COMMENT_NODE): continue - if node.tagName not in ('include', 'project', 'remote', 'default', 'manifest', 'copyfile', - 'remove-project'): + if node.tagName not in ('include', 'project', 'remote', 'default', 'manifest', 'copyfile', 'remove-project'): raise ValueError("Unsupported tag: %s" % node.tagName) to_visit.extend(node.childNodes) diff --git a/testing/mozharness/mozharness/mozilla/repo_manipulation.py b/testing/mozharness/mozharness/mozilla/repo_manipulation.py index 0408aeffda73..a2dfc46a2f43 100644 --- a/testing/mozharness/mozharness/mozilla/repo_manipulation.py +++ b/testing/mozharness/mozharness/mozilla/repo_manipulation.py @@ -160,3 +160,5 @@ the script (--clean-repos --pull --migrate). The second run will be faster.""" self.warning("No changes for %s!" % cwd) elif status: self.fatal(error_message) + + diff --git a/testing/mozharness/mozharness/mozilla/secrets.py b/testing/mozharness/mozharness/mozilla/secrets.py index ab045f7b0f57..215884354708 100644 --- a/testing/mozharness/mozharness/mozilla/secrets.py +++ b/testing/mozharness/mozharness/mozilla/secrets.py @@ -7,8 +7,11 @@ """Support for fetching secrets from the secrets API """ +import os +import mozharness import urllib2 import json +from mozharness.base.log import ERROR class SecretsMixin(object): diff --git a/testing/mozharness/mozharness/mozilla/selfserve.py b/testing/mozharness/mozharness/mozilla/selfserve.py index 266401f53251..69e243059b1d 100644 --- a/testing/mozharness/mozharness/mozilla/selfserve.py +++ b/testing/mozharness/mozharness/mozilla/selfserve.py @@ -1,7 +1,6 @@ import json import site - # SelfServeMixin {{{1 class SelfServeMixin(object): def _get_session(self): diff --git a/testing/mozharness/mozharness/mozilla/signed_certificate_timestamp.py b/testing/mozharness/mozharness/mozilla/signed_certificate_timestamp.py index 633191741daf..b3b309717404 100644 --- a/testing/mozharness/mozharness/mozilla/signed_certificate_timestamp.py +++ b/testing/mozharness/mozharness/mozilla/signed_certificate_timestamp.py @@ -3,7 +3,6 @@ import struct import base64 - class SignedCertificateTimestamp: """ Represents a Signed Certificate Timestamp from a Certificate Transparency @@ -76,6 +75,7 @@ class SignedCertificateTimestamp: if 'extensions' in response_json: self.extensions = base64.b64decode(response_json['extensions']) + @staticmethod def from_rfc6962(serialized): start = 0 @@ -120,10 +120,11 @@ class SignedCertificateTimestamp: sct.signature = struct.pack('!HH', alg, sig_len) + sig return sct + def to_rfc6962(self): version = struct.pack("B", self.version) timestamp = struct.pack("!Q", self.timestamp) ext_len = struct.pack("!H", len(self.extensions)) return version + self.id + timestamp + \ - ext_len + self.extensions + self.signature + ext_len + self.extensions + self.signature diff --git a/testing/mozharness/mozharness/mozilla/taskcluster_helper.py b/testing/mozharness/mozharness/mozilla/taskcluster_helper.py index 9ef1ee889db4..3a750d243ac8 100644 --- a/testing/mozharness/mozharness/mozilla/taskcluster_helper.py +++ b/testing/mozharness/mozharness/mozilla/taskcluster_helper.py @@ -4,6 +4,7 @@ import os from datetime import datetime, timedelta from urlparse import urljoin +from mozharness.base.log import INFO from mozharness.base.log import LogMixin @@ -250,8 +251,7 @@ class TaskClusterArtifactFinderMixin(object): self.set_artifacts( self.url_to_artifact(parent_id, installer_path), self.url_to_artifact(parent_id, 'public/build/target.test_packages.json'), - self.url_to_artifact(parent_id, - 'public/build/target.crashreporter-symbols.zip') + self.url_to_artifact(parent_id, 'public/build/target.crashreporter-symbols.zip') ) else: # Case 2: The parent task has an associated BBB task @@ -272,6 +272,5 @@ class TaskClusterArtifactFinderMixin(object): # Use the signed installer if it's set if 'signed_installer_url' in properties: signed_installer_url = properties['signed_installer_url'] - self.info('Overriding installer_url with signed_installer_url: %s' % - signed_installer_url) + self.info('Overriding installer_url with signed_installer_url: %s' % signed_installer_url) self.installer_url = signed_installer_url diff --git a/testing/mozharness/mozharness/mozilla/testing/codecoverage.py b/testing/mozharness/mozharness/mozilla/testing/codecoverage.py index 2a9afb136478..6beb2850c743 100644 --- a/testing/mozharness/mozharness/mozilla/testing/codecoverage.py +++ b/testing/mozharness/mozharness/mozilla/testing/codecoverage.py @@ -11,6 +11,7 @@ from mozharness.base.script import ( PreScriptAction, PostScriptAction, ) +from mozharness.mozilla.tooltool import TooltoolMixin code_coverage_config_options = [ [["--code-coverage"], @@ -76,17 +77,15 @@ class CodeCoverageMixin(object): # Create the grcov directory, get the tooltool manifest, and finally # download and unpack the grcov binary. self.grcov_dir = tempfile.mkdtemp() - manifest = os.path.join(dirs.get('abs_test_install_dir', - os.path.join(dirs['abs_work_dir'], 'tests')), - 'config/tooltool-manifests/linux64/ccov.manifest') + manifest = os.path.join(dirs.get('abs_test_install_dir', os.path.join(dirs['abs_work_dir'], 'tests')), \ + 'config/tooltool-manifests/linux64/ccov.manifest') tooltool_path = self._fetch_tooltool_py() - cmd = [tooltool_path, '--url', 'https://tooltool.mozilla-releng.net/', 'fetch', - '-m', manifest, '-o', '-c', '/builds/worker/tooltool-cache'] + cmd = [tooltool_path, '--url', 'https://tooltool.mozilla-releng.net/', 'fetch', \ + '-m', manifest, '-o', '-c', '/builds/worker/tooltool-cache'] self.run_command(cmd, cwd=self.grcov_dir) - self.run_command(['tar', '-jxvf', - os.path.join(self.grcov_dir, 'grcov-linux-standalone-x86_64.tar.bz2'), - '-C', self.grcov_dir], cwd=self.grcov_dir) + self.run_command(['tar', '-jxvf', os.path.join(self.grcov_dir, 'grcov-linux-standalone-x86_64.tar.bz2'), \ + '-C', self.grcov_dir], cwd=self.grcov_dir) @PostScriptAction('run-tests') def _package_coverage_data(self, action, success=None): @@ -99,9 +98,7 @@ class CodeCoverageMixin(object): # TODO This is fragile, find rel_topsrcdir properly somehow # We need to find the path relative to the gecko topsrcdir. Use # some known gecko directories as a test. - canary_dirs = [ - 'browser', 'docshell', 'dom', 'js', 'layout', 'toolkit', 'xpcom', 'xpfe' - ] + canary_dirs = ['browser', 'docshell', 'dom', 'js', 'layout', 'toolkit', 'xpcom', 'xpfe'] rel_topsrcdir = None for root, dirs, files in os.walk(self.gcov_dir): # need to use 'any' in case no gcda data was generated in that subdir. @@ -143,17 +140,14 @@ class CodeCoverageMixin(object): # 'grcov_output' will be a tuple, the first variable is the path to the lcov output, # the other is the path to the standard error output. - grcov_output = self.get_output_from_command( - grcov_command, cwd=self.grcov_dir, - silent=True, tmpfile_base_path=os.path.join(self.grcov_dir, 'grcov_lcov_output'), - save_tmpfiles=True, return_type='files' - ) + grcov_output = self.get_output_from_command(grcov_command, cwd=self.grcov_dir, \ + silent=True, tmpfile_base_path=os.path.join(self.grcov_dir, 'grcov_lcov_output'), \ + save_tmpfiles=True, return_type='files') new_output_name = grcov_output[0] + '.info' os.rename(grcov_output[0], new_output_name) # Zip the grcov output and upload it. - command = ['zip', os.path.join(dirs['abs_blob_upload_dir'], - 'code-coverage-grcov.zip'), new_output_name] + command = ['zip', os.path.join(dirs['abs_blob_upload_dir'], 'code-coverage-grcov.zip'), new_output_name] self.run_command(command, cwd=self.grcov_dir) shutil.rmtree(self.gcov_dir) shutil.rmtree(self.jsvm_dir) diff --git a/testing/mozharness/mozharness/mozilla/testing/device.py b/testing/mozharness/mozharness/mozilla/testing/device.py index b909b8df4a46..12c7a55ae18b 100644 --- a/testing/mozharness/mozharness/mozilla/testing/device.py +++ b/testing/mozharness/mozharness/mozilla/testing/device.py @@ -10,8 +10,11 @@ This code is largely from https://hg.mozilla.org/build/tools/file/default/sut_tools ''' +import datetime +import os import re import subprocess +import sys import time from mozharness.base.errors import ADBErrorList @@ -114,11 +117,9 @@ class ADBDeviceHandler(BaseDeviceHandler): devices = self._query_attached_devices() if not devices: self.add_device_flag(DEVICE_NOT_CONNECTED) - self.fatal("No device connected via adb!\nUse 'adb connect' or specify a " - "device_id or device_ip in config!") + self.fatal("No device connected via adb!\nUse 'adb connect' or specify a device_id or device_ip in config!") elif len(devices) > 1: - self.warning("""More than one device detected; specify 'device_id' " - "or\n'device_ip' to target a specific device!""") + self.warning("""More than one device detected; specify 'device_id' or\n'device_ip' to target a specific device!""") device_id = devices[0] self.info("Found %s." % device_id) self.device_id = device_id @@ -361,8 +362,9 @@ class ADBDeviceHandler(BaseDeviceHandler): retries = 0 while retries < 6: output = self.get_output_from_command([adb, "-s", device_id, - "install", '-r', file_path], - ignore_errors=True) + "install", '-r', + file_path], + ignore_errors=True) if output and output.lower().find("success") >= 0: install_complete = True break @@ -477,8 +479,7 @@ class DeviceMixin(object): device_protocol = c.get('device_protocol') device_class = DEVICE_PROTOCOL_DICT.get(device_protocol) if not device_class: - self.fatal("Unknown device_protocol %s; set via --device-protocol!" % - str(device_protocol)) + self.fatal("Unknown device_protocol %s; set via --device-protocol!" % str(device_protocol)) self.device_handler = device_class( log_obj=self.log_obj, config=self.config, diff --git a/testing/mozharness/mozharness/mozilla/testing/errors.py b/testing/mozharness/mozharness/mozilla/testing/errors.py index 0615efe65e9f..ab2a8a3d0751 100644 --- a/testing/mozharness/mozharness/mozilla/testing/errors.py +++ b/testing/mozharness/mozharness/mozilla/testing/errors.py @@ -19,8 +19,7 @@ from mozharness.base.log import INFO, WARNING, ERROR # ErrorLists {{{1 _mochitest_summary = { - 'regex': ( - re.compile(r'''(\d+ INFO (Passed|Failed|Todo):\ +(\d+)|\t(Passed|Failed|Todo): (\d+))''')), + 'regex': re.compile(r'''(\d+ INFO (Passed|Failed|Todo):\ +(\d+)|\t(Passed|Failed|Todo): (\d+))'''), 'pass_group': "Passed", 'fail_group': "Failed", 'known_fail_group': "Todo", @@ -40,15 +39,13 @@ TinderBoxPrintRe = { 'known_fail_group': "todo", }, "reftest_summary": { - 'regex': ( - re.compile(r'''REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \(''')), + 'regex': re.compile(r'''REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \('''), 'pass_group': "Successful", 'fail_group': "Unexpected", 'known_fail_group': "Known problems", }, "crashtest_summary": { - 'regex': ( - re.compile(r'''REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \(''')), + 'regex': re.compile(r'''REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \('''), 'pass_group': "Successful", 'fail_group': "Unexpected", 'known_fail_group': "Known problems", @@ -60,8 +57,7 @@ TinderBoxPrintRe = { 'known_fail_group': None, }, "jsreftest_summary": { - 'regex': ( - re.compile(r'''REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \(''')), + 'regex': re.compile(r'''REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \('''), 'pass_group': "Successful", 'fail_group': "Unexpected", 'known_fail_group': "Known problems", @@ -106,16 +102,9 @@ TinderBoxPrintRe = { }, "harness_error": { - 'full_regex': - re.compile(r"(?:TEST-UNEXPECTED-FAIL|PROCESS-CRASH) \| .* \| " - "(application crashed|missing output line for total leaks!|negative leaks " - "caught!|\d+ bytes leaked)"), + 'full_regex': re.compile(r"(?:TEST-UNEXPECTED-FAIL|PROCESS-CRASH) \| .* \| (application crashed|missing output line for total leaks!|negative leaks caught!|\d+ bytes leaked)"), 'minimum_regex': re.compile(r'''(TEST-UNEXPECTED|PROCESS-CRASH)'''), - 'retry_regex': - re.compile(r'''(FAIL-SHOULD-RETRY|No space left on device|DMError|Connection to the " - "other side was lost in a non-clean fashion|program finished with exit " - "code 80|INFRA-ERROR|twisted.spread.pb.PBConnectionLost|_dl_open: " - "Assertion|Timeout exceeded for _runCmd call)''') + 'retry_regex': re.compile(r'''(FAIL-SHOULD-RETRY|No space left on device|DMError|Connection to the other side was lost in a non-clean fashion|program finished with exit code 80|INFRA-ERROR|twisted.spread.pb.PBConnectionLost|_dl_open: Assertion|Timeout exceeded for _runCmd call)''') }, } @@ -131,17 +120,8 @@ HarnessErrorList = [ ] LogcatErrorList = [ - {'substr': - 'Fatal signal 11 (SIGSEGV)', - 'level': ERROR, 'explanation': 'This usually indicates the B2G process has crashed'}, - {'substr': - 'Fatal signal 7 (SIGBUS)', - 'level': ERROR, 'explanation': 'This usually indicates the B2G process has crashed'}, - {'substr': - '[JavaScript Error:', - 'level': WARNING}, - {'substr': - 'seccomp sandbox violation', - 'level': ERROR, - 'explanation': 'A content process has violated the system call sandbox (bug 790923)'}, + {'substr': 'Fatal signal 11 (SIGSEGV)', 'level': ERROR, 'explanation': 'This usually indicates the B2G process has crashed'}, + {'substr': 'Fatal signal 7 (SIGBUS)', 'level': ERROR, 'explanation': 'This usually indicates the B2G process has crashed'}, + {'substr': '[JavaScript Error:', 'level': WARNING}, + {'substr': 'seccomp sandbox violation', 'level': ERROR, 'explanation': 'A content process has violated the system call sandbox (bug 790923)'}, ] diff --git a/testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py b/testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py index 96c33d7cc03a..f4df25b6837d 100644 --- a/testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py +++ b/testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py @@ -10,7 +10,8 @@ import copy import os import sys -from mozharness.base.python import PreScriptAction +from mozharness.base.log import FATAL, WARNING +from mozharness.base.python import PostScriptRun, PreScriptAction from mozharness.mozilla.structuredlog import StructuredOutputParser from mozharness.mozilla.testing.testbase import ( TestingMixin, @@ -29,8 +30,7 @@ firefox_ui_tests_config_options = [ "action": "store_true", "dest": "allow_software_gl_layers", "default": False, - "help": "Permits a software GL implementation (such as LLVMPipe) to use " - "the GL compositor.", + "help": "Permits a software GL implementation (such as LLVMPipe) to use the GL compositor.", }], [["--enable-webrender"], { "action": "store_true", @@ -254,8 +254,7 @@ class FirefoxUITests(TestingMixin, VCSToolsScript, CodeCoverageMixin): env.update({'MINIDUMP_STACKWALK': self.minidump_stackwalk_path}) env['RUST_BACKTRACE'] = 'full' - # If code coverage is enabled, set GCOV_PREFIX and JS_CODE_COVERAGE_OUTPUT_DIR - # env variables + # If code coverage is enabled, set GCOV_PREFIX and JS_CODE_COVERAGE_OUTPUT_DIR env variables if self.config.get('code_coverage'): env['GCOV_PREFIX'] = self.gcov_dir env['JS_CODE_COVERAGE_OUTPUT_DIR'] = self.jsvm_dir diff --git a/testing/mozharness/mozharness/mozilla/testing/mozpool.py b/testing/mozharness/mozharness/mozilla/testing/mozpool.py index 6ecc710258ac..7e49f33ba913 100644 --- a/testing/mozharness/mozharness/mozilla/testing/mozpool.py +++ b/testing/mozharness/mozharness/mozilla/testing/mozpool.py @@ -14,15 +14,14 @@ import sys from time import sleep from mozharness.mozilla.buildbot import TBPL_RETRY, TBPL_EXCEPTION -# TODO - adjust these values +#TODO - adjust these values MAX_RETRIES = 20 RETRY_INTERVAL = 60 - # MozpoolMixin {{{1 class MozpoolMixin(object): mozpool_handler = None - mobile_imaging_format = "http://mobile-imaging" + mobile_imaging_format= "http://mobile-imaging" def determine_mozpool_host(self, device): if "mobile_imaging_format" in self.config: @@ -33,13 +32,11 @@ class MozpoolMixin(object): return imaging_server_fqdn def query_mozpool_handler(self, device=None, mozpool_api_url=None): - if self.mozpool_handler is not None: + if self.mozpool_handler != None: return self.mozpool_handler else: - self.mozpool_api_url = ( - self.determine_mozpool_host(device) if device else mozpool_api_url - ) - assert self.mozpool_api_url is not None, \ + self.mozpool_api_url = self.determine_mozpool_host(device) if device else mozpool_api_url + assert self.mozpool_api_url != None, \ "query_mozpool_handler() requires either a device or mozpool_api_url!" site_packages_path = self.query_python_site_packages_path() @@ -47,8 +44,7 @@ class MozpoolMixin(object): sys.path.append(mph_path) sys.path.append(site_packages_path) try: - from mozpoolclient import (MozpoolHandler, MozpoolException, - MozpoolConflictException) + from mozpoolclient import MozpoolHandler, MozpoolException, MozpoolConflictException self.MozpoolException = MozpoolException self.MozpoolConflictException = MozpoolConflictException self.mozpool_handler = MozpoolHandler(self.mozpool_api_url, log_obj=self) @@ -64,10 +60,9 @@ class MozpoolMixin(object): tbpl_status=TBPL_RETRY): try: image = 'panda-android-4.0.4_v3.3' - duration = 4 * 60 * 60 # request valid for 14400 seconds == 4 hours - response = mph.request_device(self.mozpool_device, image, - assignee=self.mozpool_assignee, b2gbase=b2gbase, - pxe_config=None, duration=duration) + duration = 4 * 60 * 60 # request valid for 14400 seconds == 4 hours + response = mph.request_device(self.mozpool_device, image, assignee=self.mozpool_assignee, \ + b2gbase=b2gbase, pxe_config=None, duration=duration) break except self.MozpoolConflictException: self.warning("Device unavailable. Retry#%i.." % retry) @@ -81,9 +76,7 @@ class MozpoolMixin(object): def _retry_job_and_close_request(self, message, exception=None): mph = self.query_mozpool_handler(self.mozpool_device) - exception_message = ( - str(exception) if exception is not None and str(exception) is not None else "" - ) + exception_message = str(exception) if exception!=None and str(exception) != None else "" self.error("%s -> %s" % (message, exception_message)) if self.request_url: mph.close_request(self.request_url) @@ -106,15 +99,13 @@ class MozpoolMixin(object): def _wait_for_request_ready(self): mph = self.query_mozpool_handler(self.mozpool_device) - def on_fail(): # Device is not ready after retries... self.info("Aborting mozpool request.") self.close_request() for retry in self._retry_sleep(sleep_time=RETRY_INTERVAL, max_retries=MAX_RETRIES, - error_message="INFRA-ERROR: Request did not become " - "ready in time", - tbpl_status=TBPL_EXCEPTION, fail_cb=on_fail): + error_message="INFRA-ERROR: Request did not become ready in time", + tbpl_status=TBPL_EXCEPTION, fail_cb=on_fail): response = mph.query_request_status(self.request_url) state = response['state'] if state == 'ready': diff --git a/testing/mozharness/mozharness/mozilla/testing/talos.py b/testing/mozharness/mozharness/mozilla/testing/talos.py index 41ff61b2fff5..be0c51e6bcd3 100755 --- a/testing/mozharness/mozharness/mozilla/testing/talos.py +++ b/testing/mozharness/mozharness/mozilla/testing/talos.py @@ -48,13 +48,11 @@ TalosErrorList = PythonErrorList + [ {'regex': re.compile(r'''No machine_name called '.*' can be found'''), 'level': CRITICAL}, {'substr': r"""No such file or directory: 'browser_output.txt'""", 'level': CRITICAL, - 'explanation': r"""Most likely the browser failed to launch, or the test was otherwise " - "unsuccessful in even starting."""}, + 'explanation': r"""Most likely the browser failed to launch, or the test was otherwise unsuccessful in even starting."""}, ] # TODO: check for running processes on script invocation - class TalosOutputParser(OutputParser): minidump_regex = re.compile(r'''talosError: "error executing: '(\S+) (\S+) (\S+)'"''') RE_PERF_DATA = re.compile(r'.*PERFHERDER_DATA:\s+(\{.*\})') @@ -200,17 +198,12 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin, self.gecko_profile = self.config.get('gecko_profile') self.gecko_profile_interval = self.config.get('gecko_profile_interval') self.pagesets_name = None - # some platforms download a mitmproxy release binary - self.mitmproxy_rel_bin = None - # zip file found on tooltool that contains all of the mitmproxy recordings - self.mitmproxy_recording_set = None - # files inside the recording set - self.mitmproxy_recordings_file_list = self.config.get('mitmproxy', None) - # path to mitdump tool itself, in py3 venv - self.mitmdump = None + self.mitmproxy_rel_bin = None # some platforms download a mitmproxy release binary + self.mitmproxy_recording_set = None # zip file found on tooltool that contains all of the mitmproxy recordings + self.mitmproxy_recordings_file_list = self.config.get('mitmproxy', None) # files inside the recording set + self.mitmdump = None # path to mitdump tool itself, in py3 venv - # We accept some configuration options from the try commit message - # in the format mozharness: + # We accept some configuration options from the try commit message in the format mozharness: # Example try commit message: # mozharness: --geckoProfile try: def query_gecko_profile_options(self): @@ -219,9 +212,7 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin, # this is inside automation # now let's see if we added GeckoProfile specs in the commit message try: - junk, junk, opts = self.buildbot_config( - ['sourcestamp']['changes'][-1]['comments'].partition('mozharness:') - ) + junk, junk, opts = self.buildbot_config['sourcestamp']['changes'][-1]['comments'].partition('mozharness:') except IndexError: # when we don't have comments on changes (bug 1255187) opts = None @@ -258,8 +249,7 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin, if self.abs_dirs: return self.abs_dirs abs_dirs = super(Talos, self).query_abs_dirs() - abs_dirs['abs_blob_upload_dir'] = os.path.join(abs_dirs['abs_work_dir'], - 'blobber_upload_dir') + abs_dirs['abs_blob_upload_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'blobber_upload_dir') self.abs_dirs = abs_dirs return self.abs_dirs @@ -298,8 +288,7 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin, def get_suite_from_test(self): """ Retrieve the talos suite name from a given talos test name.""" - # running locally, single test name provided instead of suite; - # go through tests and find suite name + # running locally, single test name provided instead of suite; go through tests and find suite name suite_name = None if self.query_talos_json_config(): if '-a' in self.config['talos_extra_options']: @@ -321,7 +310,7 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin, def validate_suite(self): """ Ensure suite name is a valid talos suite. """ if self.query_talos_json_config() and self.suite is not None: - if self.suite not in self.talos_json_config.get('suites'): + if not self.suite in self.talos_json_config.get('suites'): self.fatal("Suite '%s' is not valid (not found in talos json config)" % self.suite) def talos_options(self, args=None, **kw): @@ -329,8 +318,7 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin, # binary path binary_path = self.binary_path or self.config.get('binary_path') if not binary_path: - self.fatal("Talos requires a path to the binary. You can specify binary_path or " - "add download-and-extract to your action list.") + self.fatal("Talos requires a path to the binary. You can specify binary_path or add download-and-extract to your action list.") # talos options options = [] @@ -386,8 +374,7 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin, if self.config.get('run_local'): # talos initiated locally, get and verify test/suite from cmd line self.talos_path = os.path.dirname(self.talos_json) - if '-a' in (self.config['talos_extra_options'] or - '--activeTests' in self.config['talos_extra_options']): + if '-a' in self.config['talos_extra_options'] or '--activeTests' in self.config['talos_extra_options']: # test name (-a or --activeTests) specified, find out what suite it is a part of self.suite = self.get_suite_from_test() elif '--suite' in self.config['talos_extra_options']: @@ -455,8 +442,7 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin, # now create the py3 venv self.py3_venv_configuration(python_path=self.py3_path, venv_path='py3venv') self.py3_create_venv() - requirements = [os.path.join(self.talos_path, 'talos', 'mitmproxy', - 'mitmproxy_requirements.txt')] + requirements = [os.path.join(self.talos_path, 'talos', 'mitmproxy', 'mitmproxy_requirements.txt')] self.py3_install_requirement_files(requirements) # add py3 executables path to system path sys.path.insert(1, self.py3_path_to_executables()) @@ -483,8 +469,7 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin, self.fatal("Aborting: mitmproxy_release_bin_osx not found in talos.json") self.download_mitmproxy_binary(_platform) else: - self.info("Not downloading mitmproxy rel binary because " - "no-download was specified") + self.info("Not downloading mitmproxy rel binary because no-download was specified") self.info('The mitmdump macosx binary is found at: %s' % self.mitmdump) self.run_command([self.mitmdump, '--version'], env=self.query_env()) @@ -494,9 +479,7 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin, return self.mitmproxy_rel_bin if self.query_talos_json_config() and self.suite is not None: config_key = "mitmproxy_release_bin_" + platform - self.mitmproxy_rel_bin = ( - self.talos_json_config['suites'][self.suite].get(config_key, False) - ) + self.mitmproxy_rel_bin = self.talos_json_config['suites'][self.suite].get(config_key, False) return self.mitmproxy_rel_bin def download_mitmproxy_binary(self, platform): @@ -523,17 +506,14 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin, if self.mitmproxy_recording_set: return self.mitmproxy_recording_set if self.query_talos_json_config() and self.suite is not None: - self.mitmproxy_recording_set = ( - self.talos_json_config['suites'][self.suite].get('mitmproxy_recording_set', False) - ) + self.mitmproxy_recording_set = self.talos_json_config['suites'][self.suite].get('mitmproxy_recording_set', False) return self.mitmproxy_recording_set def download_mitmproxy_recording_set(self): """Download the set of mitmproxy recording files that will be played back""" self.info("Downloading the mitmproxy recording set using tooltool") dest = os.path.join(self.talos_path, 'talos', 'mitmproxy') - manifest_file = os.path.join(self.talos_path, 'talos', 'mitmproxy', - 'mitmproxy-playback-set.manifest') + manifest_file = os.path.join(self.talos_path, 'talos', 'mitmproxy', 'mitmproxy-playback-set.manifest') self.tooltool_fetch( manifest_file, output_dir=dest, @@ -617,6 +597,7 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin, shutil.copyfile(src, dest) except: self.critical("Error copying results %s to upload dir %s" % (src, dest)) + parser.update_worst_log_and_tbpl_levels(CRITICAL, TBPL_FAILURE) def run_tests(self, args=None, **kw): """run Talos tests""" diff --git a/testing/mozharness/mozharness/mozilla/testing/testbase.py b/testing/mozharness/mozharness/mozilla/testing/testbase.py index 65cfae5e7654..be5e9256858e 100755 --- a/testing/mozharness/mozharness/mozilla/testing/testbase.py +++ b/testing/mozharness/mozharness/mozilla/testing/testbase.py @@ -9,8 +9,10 @@ import copy import os import platform import pprint +import re import urllib2 import json +import socket from urlparse import urlparse, ParseResult from mozharness.base.errors import BaseErrorList @@ -50,53 +52,52 @@ TOOLTOOL_PLATFORM_DIR = { testing_config_options = [ [["--installer-url"], {"action": "store", - "dest": "installer_url", - "default": None, - "help": "URL to the installer to install", + "dest": "installer_url", + "default": None, + "help": "URL to the installer to install", }], [["--installer-path"], {"action": "store", - "dest": "installer_path", - "default": None, - "help": "Path to the installer to install. This is set automatically if run with " - "--download-and-extract.", + "dest": "installer_path", + "default": None, + "help": "Path to the installer to install. This is set automatically if run with --download-and-extract.", }], [["--binary-path"], {"action": "store", - "dest": "binary_path", - "default": None, - "help": "Path to installed binary. This is set automatically if run with --install.", + "dest": "binary_path", + "default": None, + "help": "Path to installed binary. This is set automatically if run with --install.", }], [["--exe-suffix"], {"action": "store", - "dest": "exe_suffix", - "default": None, - "help": "Executable suffix for binaries on this platform", + "dest": "exe_suffix", + "default": None, + "help": "Executable suffix for binaries on this platform", }], [["--test-url"], {"action": "store", - "dest": "test_url", - "default": None, - "help": "URL to the zip file containing the actual tests", + "dest": "test_url", + "default": None, + "help": "URL to the zip file containing the actual tests", }], [["--test-packages-url"], {"action": "store", - "dest": "test_packages_url", - "default": None, - "help": "URL to a json file describing which tests archives to download", + "dest": "test_packages_url", + "default": None, + "help": "URL to a json file describing which tests archives to download", }], [["--jsshell-url"], {"action": "store", - "dest": "jsshell_url", - "default": None, - "help": "URL to the jsshell to install", + "dest": "jsshell_url", + "default": None, + "help": "URL to the jsshell to install", }], [["--download-symbols"], {"action": "store", - "dest": "download_symbols", - "type": "choice", - "choices": ['ondemand', 'true'], - "help": "Download and extract crash reporter symbols.", + "dest": "download_symbols", + "type": "choice", + "choices": ['ondemand', 'true'], + "help": "Download and extract crash reporter symbols.", }], ] + copy.deepcopy(virtualenv_config_options) \ + copy.deepcopy(try_config_options) \ @@ -211,7 +212,7 @@ class TestingMixin(VirtualenvMixin, BuildbotMixin, ResourceMonitoringMixin, self.symbols_url = symbols_url except Exception as ex: self.warning("Cannot open symbols url %s (installer url: %s): %s" % - (symbols_url, self.installer_url, ex)) + (symbols_url, self.installer_url, ex)) if raise_on_failure: raise @@ -242,7 +243,7 @@ class TestingMixin(VirtualenvMixin, BuildbotMixin, ResourceMonitoringMixin, c = self.config orig_config = copy.deepcopy(c) self.warning("When you use developer_config.py, we drop " - "'read-buildbot-config' from the list of actions.") + "'read-buildbot-config' from the list of actions.") if "read-buildbot-config" in rw_config.actions: rw_config.actions.remove("read-buildbot-config") self.actions = tuple(rw_config.actions) @@ -259,8 +260,7 @@ class TestingMixin(VirtualenvMixin, BuildbotMixin, ResourceMonitoringMixin, self.exception("You must use --installer-url with developer_config.py") if c.get("require_test_zip"): if not c.get('test_url') and not c.get('test_packages_url'): - self.exception("You must use --test-url or " - "--test-packages-url with developer_config.py") + self.exception("You must use --test-url or --test-packages-url with developer_config.py") c["installer_url"] = _replace_url(c["installer_url"], c["replace_urls"]) if c.get("test_url"): @@ -292,8 +292,7 @@ class TestingMixin(VirtualenvMixin, BuildbotMixin, ResourceMonitoringMixin, self.https_username, self.https_password = get_credentials() # This creates a password manager passman = urllib2.HTTPPasswordMgrWithDefaultRealm() - # Because we have put None at the start it will use this username/password combination - # from here on + # Because we have put None at the start it will use this username/password combination from here on passman.add_password(None, url, self.https_username, self.https_password) authhandler = urllib2.HTTPBasicAuthHandler(passman) @@ -312,6 +311,7 @@ class TestingMixin(VirtualenvMixin, BuildbotMixin, ResourceMonitoringMixin, c = self.config try: files = self.buildbot_config['sourcestamp']['changes'][-1]['files'] + buildbot_prop_branch = self.buildbot_config['properties']['branch'] # Bug 868490 - Only require exactly two files if require_test_zip; # otherwise accept either 1 or 2, since we'll be getting a @@ -321,8 +321,7 @@ class TestingMixin(VirtualenvMixin, BuildbotMixin, ResourceMonitoringMixin, expected_length = [2, 3] actual_length = len(files) if actual_length not in expected_length: - self.fatal("Unexpected number of files in buildbot config %s.\n" - "Expected these number(s) of files: %s, but got: %d" % + self.fatal("Unexpected number of files in buildbot config %s.\nExpected these number(s) of files: %s, but got: %d" % (c['buildbot_json_path'], str(expected_length), actual_length)) for f in files: if f['name'].endswith('tests.zip'): # yuk @@ -336,8 +335,7 @@ class TestingMixin(VirtualenvMixin, BuildbotMixin, ResourceMonitoringMixin, elif f['name'].endswith('test_packages.json'): self.test_packages_url = str(f['name']) self.info("Found a test packages url %s." % self.test_packages_url) - elif not any(f['name'].endswith(s) for s in ('code-coverage-gcno.zip', - 'stylo-bindings.zip')): + elif not any(f['name'].endswith(s) for s in ('code-coverage-gcno.zip', 'stylo-bindings.zip')): if not self.installer_url: self.installer_url = str(f['name']) self.info("Found installer url %s." % self.installer_url) @@ -403,8 +401,7 @@ You can set this by: 2. running via buildbot and running the read-buildbot-config action """ - if (self.config.get("require_test_zip") - and not self.test_url and not self.test_packages_url): + if self.config.get("require_test_zip") and not self.test_url and not self.test_packages_url: message += """test_url isn't set! You can set this by: @@ -529,8 +526,10 @@ You can set this by: if self.installer_path: file_name = self.installer_path dirs = self.query_abs_dirs() - source = self.download_file(self.installer_url, file_name=file_name, - parent_dir=dirs['abs_work_dir'], error_level=FATAL) + source = self.download_file(self.installer_url, + file_name=file_name, + parent_dir=dirs['abs_work_dir'], + error_level=FATAL) self.installer_path = os.path.realpath(source) self.set_buildbot_property("build_url", self.installer_url, write_to_file=True) @@ -620,8 +619,7 @@ Did you run with --create-virtualenv? Is mozinstall in virtualenv_modules?""") def install_app(self, app=None, target_dir=None, installer_path=None): """ Dependent on mozinstall """ # install the application - cmd = self.query_exe("mozinstall", default=self.query_python_path("mozinstall"), - return_type="list") + cmd = self.query_exe("mozinstall", default=self.query_python_path("mozinstall"), return_type="list") if app: cmd.extend(['--app', app]) # Remove the below when we no longer need to support mozinstall 0.3 @@ -722,7 +720,7 @@ Did you run with --create-virtualenv? Is mozinstall in virtualenv_modules?""") return self.nodejs_path c = self.config - dirs = self.query_abs_dirs() + dirs = self.query_abs_dirs(); nodejs_path = self.query_nodejs_filename() if not self.config.get('download_nodejs'): @@ -752,8 +750,7 @@ Did you run with --create-virtualenv? Is mozinstall in virtualenv_modules?""") self.chmod(abs_nodejs_path, 0755) self.nodejs_path = abs_nodejs_path else: - self.warning("nodejs path was given but couldn't be found. " - "Tried looking in '%s'" % abs_nodejs_path) + self.warning("nodejs path was given but couldn't be found. Tried looking in '%s'" % abs_nodejs_path) self.buildbot_status(TBPL_WARNING, WARNING) return self.nodejs_path diff --git a/testing/mozharness/mozharness/mozilla/testing/try_tools.py b/testing/mozharness/mozharness/mozilla/testing/try_tools.py index 296c98441a46..583ba5b217ed 100644 --- a/testing/mozharness/mozharness/mozilla/testing/try_tools.py +++ b/testing/mozharness/mozharness/mozilla/testing/try_tools.py @@ -16,9 +16,9 @@ from mozharness.base.transfer import TransferMixin try_config_options = [ [["--try-message"], {"action": "store", - "dest": "try_message", - "default": None, - "help": "try syntax string to select tests to run", + "dest": "try_message", + "default": None, + "help": "try syntax string to select tests to run", }], ] @@ -27,7 +27,7 @@ test_flavors = { 'chrome': {}, 'devtools-chrome': {}, 'mochitest': {}, - 'xpcshell': {}, + 'xpcshell' :{}, 'reftest': { "path": lambda x: os.path.join("tests", "reftest", "tests", x) }, @@ -45,7 +45,6 @@ test_flavors = { }, } - class TryToolsMixin(TransferMixin): """Utility functions for an interface between try syntax and out test harnesses. Requires log and script mixins.""" @@ -144,7 +143,7 @@ class TryToolsMixin(TransferMixin): all_try_args = re.findall(r'(?:\[.*?\]|\S)+', try_message[1]) break if not all_try_args: - self.warning('Try syntax not found in: %s.' % msg) + self.warning('Try syntax not found in: %s.' % msg ) return all_try_args def try_message_has_flag(self, flag, message=None): @@ -196,7 +195,6 @@ class TryToolsMixin(TransferMixin): ' and forward them to the underlying test harness command.')) label_dict = {} - def label_from_val(val): if val in label_dict: return label_dict[val] @@ -259,7 +257,7 @@ class TryToolsMixin(TransferMixin): 'files: %s.' % ','.join(self.try_test_paths[flavor])) args.extend(['--this-chunk=1', '--total-chunks=1']) - path_func = test_flavors[flavor].get("path", lambda x: x) + path_func = test_flavors[flavor].get("path", lambda x:x) tests = [path_func(os.path.normpath(item)) for item in self.try_test_paths[flavor]] else: tests = [] diff --git a/testing/mozharness/mozharness/mozilla/testing/unittest.py b/testing/mozharness/mozharness/mozilla/testing/unittest.py index 50c450acd7f0..d935ff699a8e 100755 --- a/testing/mozharness/mozharness/mozilla/testing/unittest.py +++ b/testing/mozharness/mozharness/mozilla/testing/unittest.py @@ -186,11 +186,13 @@ class DesktopUnittestOutputParser(OutputParser): levels=TBPL_WORST_LEVEL_TUPLE) # Account for the possibility that no test summary was output. - if self.pass_count <= 0 and self.fail_count <= 0 and (self.known_fail_count is None - or self.known_fail_count <= 0): + if self.pass_count <= 0 and self.fail_count <= 0 and \ + (self.known_fail_count is None or self.known_fail_count <= 0): self.error('No tests run or test summary not found') - self.worst_log_level = self.worst_level(WARNING, self.worst_log_level) - self.tbpl_status = self.worst_level(TBPL_WARNING, self.tbpl_status, + self.worst_log_level = self.worst_level(WARNING, + self.worst_log_level) + self.tbpl_status = self.worst_level(TBPL_WARNING, + self.tbpl_status, levels=TBPL_WORST_LEVEL_TUPLE) if return_code not in success_codes: diff --git a/testing/mozharness/mozharness/mozilla/testing/verify_tools.py b/testing/mozharness/mozharness/mozilla/testing/verify_tools.py index 5775fae8e15c..27521c125e00 100644 --- a/testing/mozharness/mozharness/mozilla/testing/verify_tools.py +++ b/testing/mozharness/mozharness/mozilla/testing/verify_tools.py @@ -5,6 +5,7 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. # ***** END LICENSE BLOCK ***** +import argparse import os import posixpath import re @@ -42,7 +43,7 @@ class VerifyToolsMixin(object): been downloaded and extracted. """ - if not self.config.get('verify'): + if self.config.get('verify') != True: return repository = os.environ.get("GECKO_HEAD_REPOSITORY") @@ -61,8 +62,7 @@ class VerifyToolsMixin(object): manifests = [ (os.path.join(dirs['abs_mochitest_dir'], 'tests', 'mochitest.ini'), 'plain'), (os.path.join(dirs['abs_mochitest_dir'], 'chrome', 'chrome.ini'), 'chrome'), - (os.path.join(dirs['abs_mochitest_dir'], 'browser', - 'browser-chrome.ini'), 'browser-chrome'), + (os.path.join(dirs['abs_mochitest_dir'], 'browser', 'browser-chrome.ini'), 'browser-chrome'), (os.path.join(dirs['abs_mochitest_dir'], 'a11y', 'a11y.ini'), 'a11y'), (os.path.join(dirs['abs_xpcshell_dir'], 'tests', 'xpcshell.ini'), 'xpcshell'), ] @@ -71,16 +71,13 @@ class VerifyToolsMixin(object): if os.path.exists(path): man = TestManifest([path], strict=False) active = man.active_tests(exists=False, disabled=False, filters=[], **mozinfo.info) - tests_by_path.update({t['relpath']: (suite, t.get('subsuite')) for t in active}) + tests_by_path.update({t['relpath']:(suite,t.get('subsuite')) for t in active}) self.info("Verification updated with manifest %s" % path) ref_manifests = [ - (os.path.join(dirs['abs_reftest_dir'], 'tests', 'layout', 'reftests', - 'reftest.list'), 'reftest'), - (os.path.join(dirs['abs_reftest_dir'], 'tests', 'testing', 'crashtest', - 'crashtests.list'), 'crashtest'), - # TODO (os.path.join(dirs['abs_test_install_dir'], 'jsreftest', 'tests', - # 'jstests.list'), 'jstestbrowser'), + (os.path.join(dirs['abs_reftest_dir'], 'tests', 'layout', 'reftests', 'reftest.list'), 'reftest'), + (os.path.join(dirs['abs_reftest_dir'], 'tests', 'testing', 'crashtest', 'crashtests.list'), 'crashtest'), + # TODO (os.path.join(dirs['abs_test_install_dir'], 'jsreftest', 'tests', 'jstests.list'), 'jstestbrowser'), ] sys.path.append(dirs['abs_reftest_dir']) import manifest @@ -89,8 +86,7 @@ class VerifyToolsMixin(object): if os.path.exists(path): man = manifest.ReftestManifest() man.load(path) - tests_by_path.update({os.path.relpath(t, self.reftest_test_dir): (suite, None) - for t in man.files}) + tests_by_path.update({os.path.relpath(t,self.reftest_test_dir):(suite,None) for t in man.files}) self.info("Verification updated with manifest %s" % path) # determine which files were changed on this push @@ -112,15 +108,15 @@ class VerifyToolsMixin(object): if entry: self.info("Verification found test %s" % file) subsuite_mapping = { - ('browser-chrome', 'clipboard'): 'browser-chrome-clipboard', - ('chrome', 'clipboard'): 'chrome-clipboard', - ('plain', 'clipboard'): 'plain-clipboard', - ('browser-chrome', 'devtools'): 'mochitest-devtools-chrome', - ('browser-chrome', 'gpu'): 'browser-chrome-gpu', - ('chrome', 'gpu'): 'chrome-gpu', - ('plain', 'gpu'): 'plain-gpu', - ('plain', 'media'): 'mochitest-media', - ('plain', 'webgl'): 'mochitest-gl', + ('browser-chrome', 'clipboard') : 'browser-chrome-clipboard', + ('chrome', 'clipboard') : 'chrome-clipboard', + ('plain', 'clipboard') : 'plain-clipboard', + ('browser-chrome', 'devtools') : 'mochitest-devtools-chrome', + ('browser-chrome', 'gpu') : 'browser-chrome-gpu', + ('chrome', 'gpu') : 'chrome-gpu', + ('plain', 'gpu') : 'plain-gpu', + ('plain', 'media') : 'mochitest-media', + ('plain', 'webgl') : 'mochitest-gl', } if entry in subsuite_mapping: suite = subsuite_mapping[entry] @@ -146,7 +142,7 @@ class VerifyToolsMixin(object): # when verifying long-running tests. MAX_TIME_PER_TEST = 900 - if self.config.get('verify'): + if self.config.get('verify') != True: # not in verify mode: run once, with no additional args args = [[]] else: @@ -178,10 +174,10 @@ class VerifyToolsMixin(object): suite category. """ suites = None - if self.config.get('verify'): + if self.config.get('verify') == True: if all_suites and self.verify_downloaded: suites = dict((key, all_suites.get(key)) for key in - self.verify_suites if key in all_suites.keys()) + self.verify_suites if key in all_suites.keys()) else: # Until test zips are downloaded, manifests are not available, # so it is not possible to determine which suites are active/ @@ -212,3 +208,4 @@ class VerifyToolsMixin(object): test_name = test_name.rstrip(os.path.sep) self.log("TinderboxPrint: Verification of %s
: %s" % (test_name, tbpl_status), level=log_level) + diff --git a/testing/mozharness/mozharness/mozilla/updates/balrog.py b/testing/mozharness/mozharness/mozilla/updates/balrog.py index 7082eae385e5..ec8a1d2c0792 100644 --- a/testing/mozharness/mozharness/mozilla/updates/balrog.py +++ b/testing/mozharness/mozharness/mozilla/updates/balrog.py @@ -24,6 +24,7 @@ class BalrogMixin(object): python = 'python2.7' return python + def generate_balrog_props(self, props_path): self.set_buildbot_property( "hashType", self.config.get("hash_type", "sha512"), write_to_file=True @@ -106,16 +107,12 @@ class BalrogMixin(object): product = self.buildbot_config["properties"]["product"] cmd = [ self.query_python(), - os.path.join(os.path.join(dirs['abs_tools_dir'], - "scripts/updates/balrog-release-pusher.py")) + os.path.join(os.path.join(dirs['abs_tools_dir'], "scripts/updates/balrog-release-pusher.py")) ] - cmd.extend(["--build-properties", - os.path.join(dirs["base_work_dir"], "balrog_props.json")]) + cmd.extend(["--build-properties", os.path.join(dirs["base_work_dir"], "balrog_props.json")]) cmd.extend(["--buildbot-configs", "https://hg.mozilla.org/build/buildbot-configs"]) - cmd.extend(["--release-config", os.path.join(dirs['build_dir'], - self.config.get("release_config_file"))]) - cmd.extend(["--credentials-file", os.path.join(dirs['base_work_dir'], - self.config.get("balrog_credentials_file"))]) + cmd.extend(["--release-config", os.path.join(dirs['build_dir'], self.config.get("release_config_file"))]) + cmd.extend(["--credentials-file", os.path.join(dirs['base_work_dir'], self.config.get("balrog_credentials_file"))]) cmd.extend(["--release-channel", self.query_release_config()['release_channel']]) return_codes = [] diff --git a/tools/lint/flake8.yml b/tools/lint/flake8.yml index f4bd052cc860..46fd1dbaf1c6 100644 --- a/tools/lint/flake8.yml +++ b/tools/lint/flake8.yml @@ -23,7 +23,6 @@ flake8: - testing/mochitest - testing/mozbase - testing/mozharness/mozfile - - testing/mozharness/mozharness/mozilla - testing/mozharness/mozinfo - testing/mozharness/scripts - testing/remotecppunittests.py From 84fe6b366f9ee4751786b651ad2bed3bf484fc1d Mon Sep 17 00:00:00 2001 From: Phil Ringnalda Date: Mon, 30 Oct 2017 21:29:46 -0700 Subject: [PATCH 28/32] Backed out 2 changesets (bug 1298018) on suspicion of being the parts that cause assertion failures in jsapi-tests CLOSED TREE Backed out changeset f029195245c0 (bug 1298018) Backed out changeset 668f1422c955 (bug 1298018) MozReview-Commit-ID: JvokXPrAvdN --- js/src/gc/Nursery.cpp | 155 +++++++++++++++++++++--------------------- js/src/gc/Nursery.h | 46 ++++--------- 2 files changed, 92 insertions(+), 109 deletions(-) diff --git a/js/src/gc/Nursery.cpp b/js/src/gc/Nursery.cpp index 45f9ffc81bdf..a0ab5471466d 100644 --- a/js/src/gc/Nursery.cpp +++ b/js/src/gc/Nursery.cpp @@ -117,8 +117,7 @@ js::Nursery::Nursery(JSRuntime* rt) , currentStartPosition_(0) , currentEnd_(0) , currentChunk_(0) - , maxChunkCount_(0) - , chunkCountLimit_(0) + , maxNurseryChunks_(0) , previousPromotionRate_(0) , profileThreshold_(0) , enableProfiling_(false) @@ -142,14 +141,13 @@ js::Nursery::init(uint32_t maxNurseryBytes, AutoLockGCBgAlloc& lock) return false; /* maxNurseryBytes parameter is rounded down to a multiple of chunk size. */ - chunkCountLimit_ = maxNurseryBytes >> ChunkShift; + maxNurseryChunks_ = maxNurseryBytes >> ChunkShift; /* If no chunks are specified then the nursery is permanently disabled. */ - if (chunkCountLimit_ == 0) + if (maxNurseryChunks_ == 0) return true; - maxChunkCount_ = 1; - if (!allocateNextChunk(0, lock)) + if (!allocateFirstChunk(lock)) return false; setCurrentChunk(0); @@ -194,13 +192,12 @@ js::Nursery::enable() { MOZ_ASSERT(isEmpty()); MOZ_ASSERT(!runtime()->gc.isVerifyPreBarriersEnabled()); - if (isEnabled() || !chunkCountLimit()) + if (isEnabled() || !maxChunks()) return; { AutoLockGCBgAlloc lock(runtime()); - maxChunkCount_ = 1; - if (!allocateNextChunk(0, lock)) + if (!allocateFirstChunk(lock)) return; } @@ -222,10 +219,7 @@ js::Nursery::disable() return; freeChunksFrom(0); - maxChunkCount_ = 0; - currentEnd_ = 0; - runtime()->gc.storeBuffer().disable(); } @@ -246,7 +240,7 @@ js::Nursery::isEmpty() const void js::Nursery::enterZealMode() { if (isEnabled()) - maxChunkCount_ = chunkCountLimit(); + growAllocableSpace(maxChunks()); } void @@ -312,19 +306,9 @@ js::Nursery::allocate(size_t size) #endif if (currentEnd() < position() + size) { - unsigned chunkno = currentChunk_ + 1; - MOZ_ASSERT(chunkno <= chunkCountLimit()); - MOZ_ASSERT(chunkno <= maxChunkCount()); - MOZ_ASSERT(chunkno <= allocatedChunkCount()); - if (chunkno == maxChunkCount()) + if (currentChunk_ + 1 == numChunks()) return nullptr; - if (MOZ_UNLIKELY(chunkno == allocatedChunkCount())) { - AutoLockGCBgAlloc lock(runtime()); - if (!allocateNextChunk(chunkno, lock)) - return nullptr; - MOZ_ASSERT(chunkno < allocatedChunkCount()); - } - setCurrentChunk(chunkno); + setCurrentChunk(currentChunk_ + 1); } void* thing = (void*)position(); @@ -531,8 +515,7 @@ js::Nursery::renderProfileJSON(JSONPrinter& json) const json.property("bytes_tenured", previousGC.tenuredBytes); json.floatProperty("promotion_rate", calcPromotionRate(nullptr), 0); json.property("nursery_bytes", previousGC.nurseryUsedBytes); - json.property("new_nursery_bytes", maxChunkCount() * ChunkSize); - json.property("new_nursery_bytes_alloc", allocatedChunkCount() * ChunkSize); + json.property("new_nursery_bytes", numChunks() * ChunkSize); json.beginObjectProperty("timings"); @@ -695,7 +678,7 @@ js::Nursery::collect(JS::gcreason::Reason reason) // Disable the nursery if the user changed the configuration setting. The // nursery can only be re-enabled by resetting the configurationa and // restarting firefox. - if (chunkCountLimit_ == 0) + if (maxNurseryChunks_ == 0) disable(); endProfile(ProfileKey::Total); @@ -718,7 +701,7 @@ js::Nursery::collect(JS::gcreason::Reason reason) fprintf(stderr, "MinorGC: %20s %5.1f%% %4u ", JS::gcreason::ExplainReason(reason), promotionRate * 100, - maxChunkCount()); + numChunks()); printProfileDurations(profileDurations_); if (reportTenurings_) { @@ -923,18 +906,18 @@ js::Nursery::clear() { #ifdef JS_GC_ZEAL /* Poison the nursery contents so touching a freed object will crash. */ - for (unsigned i = 0; i < allocatedChunkCount(); i++) + for (unsigned i = 0; i < numChunks(); i++) chunk(i).poisonAndInit(runtime(), JS_SWEPT_NURSERY_PATTERN); if (runtime()->hasZealMode(ZealMode::GenerationalGC)) { /* Only reset the alloc point when we are close to the end. */ - if (currentChunk_ + 1 == maxChunkCount()) + if (currentChunk_ + 1 == numChunks()) setCurrentChunk(0); } else #endif { #ifdef JS_CRASH_DIAGNOSTICS - for (unsigned i = 0; i < allocatedChunkCount(); ++i) + for (unsigned i = 0; i < numChunks(); ++i) chunk(i).poisonAndInit(runtime(), JS_SWEPT_NURSERY_PATTERN); #endif setCurrentChunk(0); @@ -947,7 +930,7 @@ js::Nursery::clear() size_t js::Nursery::spaceToEnd() const { - unsigned lastChunk = maxChunkCount() - 1; + unsigned lastChunk = numChunks() - 1; MOZ_ASSERT(lastChunk >= currentStartChunk_); MOZ_ASSERT(currentStartPosition_ - chunk(currentStartChunk_).start() <= NurseryChunkUsableSize); @@ -955,7 +938,7 @@ js::Nursery::spaceToEnd() const size_t bytes = (chunk(currentStartChunk_).end() - currentStartPosition_) + ((lastChunk - currentStartChunk_) * NurseryChunkUsableSize); - MOZ_ASSERT(bytes <= maxChunkCount() * NurseryChunkUsableSize); + MOZ_ASSERT(bytes <= numChunks() * NurseryChunkUsableSize); return bytes; } @@ -963,43 +946,14 @@ js::Nursery::spaceToEnd() const MOZ_ALWAYS_INLINE void js::Nursery::setCurrentChunk(unsigned chunkno) { - MOZ_ASSERT(chunkno < chunkCountLimit()); - MOZ_ASSERT(chunkno < allocatedChunkCount()); + MOZ_ASSERT(chunkno < maxChunks()); + MOZ_ASSERT(chunkno < numChunks()); currentChunk_ = chunkno; position_ = chunk(chunkno).start(); currentEnd_ = chunk(chunkno).end(); chunk(chunkno).poisonAndInit(runtime(), JS_FRESH_NURSERY_PATTERN); } -bool -js::Nursery::allocateNextChunk(const unsigned chunkno, - AutoLockGCBgAlloc& lock) -{ - if (chunkno == maxChunkCount()) - return false; - - const unsigned priorCount = allocatedChunkCount(); - const unsigned newCount = priorCount + 1; - MOZ_ASSERT((chunkno == currentChunk_ + 1) || (chunkno == 0 && allocatedChunkCount() == 0)); - MOZ_ASSERT(chunkno == allocatedChunkCount()); - - if (!chunks_.resize(newCount)) - return false; - - MOZ_ASSERT(chunkno < chunkCountLimit()); - MOZ_ASSERT(chunkno < maxChunkCount()); - - Chunk* newChunk; - newChunk = runtime()->gc.getOrAllocChunk(lock); - if (!newChunk) { - chunks_.shrinkTo(priorCount); - return false; - } - - chunks_[chunkno] = NurseryChunk::fromChunk(newChunk); - return true; -} - MOZ_ALWAYS_INLINE void js::Nursery::setStartPosition() { @@ -1036,10 +990,10 @@ js::Nursery::maybeResizeNursery(JS::gcreason::Reason reason) float(previousGC.tenuredBytes) / float(previousGC.nurseryCapacity); newMaxNurseryChunks = runtime()->gc.tunables.gcMaxNurseryBytes() >> ChunkShift; - if (newMaxNurseryChunks != chunkCountLimit_) { - chunkCountLimit_ = newMaxNurseryChunks; + if (newMaxNurseryChunks != maxNurseryChunks_) { + maxNurseryChunks_ = newMaxNurseryChunks; /* The configured maximum nursery size is changing */ - if (maxChunkCount() > newMaxNurseryChunks) { + if (numChunks() > newMaxNurseryChunks) { /* We need to shrink the nursery */ shrinkAllocableSpace(newMaxNurseryChunks); @@ -1053,16 +1007,43 @@ js::Nursery::maybeResizeNursery(JS::gcreason::Reason reason) // nursery chunks we do not report an error. growAllocableSpace(); } else if (promotionRate < ShrinkThreshold && previousPromotionRate_ < ShrinkThreshold) { - shrinkAllocableSpace(maxChunkCount() - 1); + shrinkAllocableSpace(numChunks() - 1); } previousPromotionRate_ = promotionRate; } -void +bool js::Nursery::growAllocableSpace() { - maxChunkCount_ = Min(maxChunkCount() * 2, chunkCountLimit()); + return growAllocableSpace(Min(numChunks() * 2, maxChunks())); +} + +bool +js::Nursery::growAllocableSpace(unsigned newCount) +{ + unsigned priorCount = numChunks(); + + MOZ_ASSERT(newCount > priorCount); + MOZ_ASSERT(newCount <= maxChunks()); + MOZ_ASSERT(priorCount >= 1); + + if (!chunks_.resize(newCount)) + return false; + + AutoLockGCBgAlloc lock(runtime()); + for (unsigned i = priorCount; i < newCount; i++) { + auto newChunk = runtime()->gc.getOrAllocChunk(lock); + if (!newChunk) { + chunks_.shrinkTo(i); + return false; + } + + chunks_[i] = NurseryChunk::fromChunk(newChunk); + chunk(i).poisonAndInit(runtime(), JS_FRESH_NURSERY_PATTERN); + } + + return true; } void @@ -1087,15 +1068,12 @@ js::Nursery::shrinkAllocableSpace(unsigned newCount) // Don't shrink the nursery to zero (use Nursery::disable() instead) and // don't attempt to shrink it to the same size. - if ((newCount == 0) || (newCount == maxChunkCount())) + if ((newCount == 0) || (newCount == numChunks())) return; - MOZ_ASSERT(newCount < maxChunkCount()); + MOZ_ASSERT(newCount < numChunks()); - if (newCount < allocatedChunkCount()) - freeChunksFrom(newCount); - - maxChunkCount_ = newCount; + freeChunksFrom(newCount); } void @@ -1104,6 +1082,29 @@ js::Nursery::minimizeAllocableSpace() shrinkAllocableSpace(1); } +bool +js::Nursery::allocateFirstChunk(AutoLockGCBgAlloc& lock) +{ + // This assertion isn't required for correctness, but we do assume this + // is only called to initialize or re-enable the nursery. + MOZ_ASSERT(numChunks() == 0); + + MOZ_ASSERT(maxChunks() > 0); + + if (!chunks_.resize(1)) + return false; + + auto chunk = runtime()->gc.getOrAllocChunk(lock); + if (!chunk) { + chunks_.shrinkTo(0); + return false; + } + + chunks_[0] = NurseryChunk::fromChunk(chunk); + + return true; +} + bool js::Nursery::queueDictionaryModeObjectToSweep(NativeObject* obj) { diff --git a/js/src/gc/Nursery.h b/js/src/gc/Nursery.h index 3bc9ba85185c..96ef0ffa9074 100644 --- a/js/src/gc/Nursery.h +++ b/js/src/gc/Nursery.h @@ -141,22 +141,14 @@ class Nursery MOZ_MUST_USE bool init(uint32_t maxNurseryBytes, AutoLockGCBgAlloc& lock); - unsigned chunkCountLimit() const { return chunkCountLimit_; } + unsigned maxChunks() const { return maxNurseryChunks_; } + unsigned numChunks() const { return chunks_.length(); } - // Number of allocated (ready to use) chunks. - unsigned allocatedChunkCount() const { return chunks_.length(); } - - // Total number of chunks and the capacity of the nursery. Chunks will be - // lazilly allocated and added to the chunks array up to this limit, after - // that the nursery must be collected, this limit may be raised during - // collection. - unsigned maxChunkCount() const { return maxChunkCount_; } - - bool exists() const { return chunkCountLimit() != 0; } + bool exists() const { return maxChunks() != 0; } void enable(); void disable(); - bool isEnabled() const { return maxChunkCount() != 0; } + bool isEnabled() const { return numChunks() != 0; } /* Return true if no allocations have been made since the last collection. */ bool isEmpty() const; @@ -241,7 +233,7 @@ class Nursery MOZ_MUST_USE bool queueDictionaryModeObjectToSweep(NativeObject* obj); size_t sizeOfHeapCommitted() const { - return allocatedChunkCount() * gc::ChunkSize; + return numChunks() * gc::ChunkSize; } size_t sizeOfMallocedBuffers(mozilla::MallocSizeOf mallocSizeOf) const { if (!mallocedBuffers.initialized()) @@ -260,7 +252,7 @@ class Nursery MOZ_ALWAYS_INLINE size_t freeSpace() const { MOZ_ASSERT(currentEnd_ - position_ <= NurseryChunkUsableSize); return (currentEnd_ - position_) + - (maxChunkCount() - currentChunk_ - 1) * NurseryChunkUsableSize; + (numChunks() - currentChunk_ - 1) * NurseryChunkUsableSize; } #ifdef JS_GC_ZEAL @@ -310,18 +302,8 @@ class Nursery /* The index of the chunk that is currently being allocated from. */ unsigned currentChunk_; - /* - * The nursery may grow the chunks_ vector up to this size without a - * collection. This allows the nursery to grow lazilly. This limit may - * change during maybeResizeNursery() each collection. - */ - unsigned maxChunkCount_; - - /* - * This limit is fixed by configuration. It represents the maximum size - * the nursery is permitted to tune itself to in maybeResizeNursery(); - */ - unsigned chunkCountLimit_; + /* Maximum number of chunks to allocate for the nursery. */ + unsigned maxNurseryChunks_; /* Promotion rate for the previous minor collection. */ float previousPromotionRate_; @@ -436,11 +418,10 @@ class Nursery void setStartPosition(); /* - * Allocate the next chunk, or the first chunk for initialization. - * Callers will probably want to call setCurrentChunk(0) next. + * Ensure that the first chunk has been allocated. Callers will probably + * want to call setCurrentChunk(0) next. */ - MOZ_MUST_USE bool allocateNextChunk(unsigned chunkno, - AutoLockGCBgAlloc& lock); + MOZ_MUST_USE bool allocateFirstChunk(AutoLockGCBgAlloc& lock); MOZ_ALWAYS_INLINE uintptr_t currentEnd() const; @@ -492,12 +473,13 @@ class Nursery /* Change the allocable space provided by the nursery. */ void maybeResizeNursery(JS::gcreason::Reason reason); - void growAllocableSpace(); + bool growAllocableSpace(); + bool growAllocableSpace(unsigned newSize); void shrinkAllocableSpace(unsigned newCount); void minimizeAllocableSpace(); // Free the chunks starting at firstFreeChunk until the end of the chunks - // vector. Shrinks the vector but does not update maxChunkCount(). + // vector. Shrinks the vector but does not update maxChunksAlloc(). void freeChunksFrom(unsigned firstFreeChunk); /* Profile recording and printing. */ From 4046844eac48fd51a99f8ea6f2a85c857430aaaf Mon Sep 17 00:00:00 2001 From: Phil Ringnalda Date: Mon, 30 Oct 2017 21:51:12 -0700 Subject: [PATCH 29/32] Backed out 3 changesets (bug 1298018) for assertion failures in jittests Backed out changeset b6dafc346ef5 (bug 1298018) Backed out changeset 679e75a67b8e (bug 1298018) Backed out changeset 9c9299bd7441 (bug 1298018) MozReview-Commit-ID: 2qJiowUW4Qj --- js/src/gc/GCRuntime.h | 9 ++- js/src/gc/Nursery.cpp | 141 +++++++++++++++++------------------------- js/src/gc/Nursery.h | 20 +++--- 3 files changed, 70 insertions(+), 100 deletions(-) diff --git a/js/src/gc/GCRuntime.h b/js/src/gc/GCRuntime.h index 2e94138d9513..33e46f8d324f 100644 --- a/js/src/gc/GCRuntime.h +++ b/js/src/gc/GCRuntime.h @@ -1210,11 +1210,10 @@ class GCRuntime AtomMarkingRuntime atomMarking; private: - // When chunks are empty, they reside in the emptyChunks pool and are - // re-used as needed or eventually expired if not re-used. The emptyChunks - // pool gets refilled from the background allocation task heuristically so - // that empty chunks should always be available for immediate allocation - // without syscalls. + // When empty, chunks reside in the emptyChunks pool and are re-used as + // needed or eventually expired if not re-used. The emptyChunks pool gets + // refilled from the background allocation task heuristically so that empty + // chunks should always available for immediate allocation without syscalls. GCLockData emptyChunks_; // Chunks which have had some, but not all, of their arenas allocated live diff --git a/js/src/gc/Nursery.cpp b/js/src/gc/Nursery.cpp index a0ab5471466d..1718e3c6b747 100644 --- a/js/src/gc/Nursery.cpp +++ b/js/src/gc/Nursery.cpp @@ -147,7 +147,8 @@ js::Nursery::init(uint32_t maxNurseryBytes, AutoLockGCBgAlloc& lock) if (maxNurseryChunks_ == 0) return true; - if (!allocateFirstChunk(lock)) + updateNumChunksLocked(1, lock); + if (numChunks() == 0) return false; setCurrentChunk(0); @@ -195,11 +196,9 @@ js::Nursery::enable() if (isEnabled() || !maxChunks()) return; - { - AutoLockGCBgAlloc lock(runtime()); - if (!allocateFirstChunk(lock)) - return; - } + updateNumChunks(1); + if (numChunks() == 0) + return; setCurrentChunk(0); setStartPosition(); @@ -217,8 +216,7 @@ js::Nursery::disable() MOZ_ASSERT(isEmpty()); if (!isEnabled()) return; - - freeChunksFrom(0); + updateNumChunks(0); currentEnd_ = 0; runtime()->gc.storeBuffer().disable(); } @@ -240,7 +238,7 @@ js::Nursery::isEmpty() const void js::Nursery::enterZealMode() { if (isEnabled()) - growAllocableSpace(maxChunks()); + updateNumChunks(maxNurseryChunks_); } void @@ -993,116 +991,93 @@ js::Nursery::maybeResizeNursery(JS::gcreason::Reason reason) if (newMaxNurseryChunks != maxNurseryChunks_) { maxNurseryChunks_ = newMaxNurseryChunks; /* The configured maximum nursery size is changing */ - if (numChunks() > newMaxNurseryChunks) { + const int extraChunks = numChunks() - newMaxNurseryChunks; + if (extraChunks > 0) { /* We need to shrink the nursery */ - shrinkAllocableSpace(newMaxNurseryChunks); + shrinkAllocableSpace(extraChunks); previousPromotionRate_ = promotionRate; return; } } - if (promotionRate > GrowThreshold) { - // The GC nursery is an optimization and so if we fail to allocate - // nursery chunks we do not report an error. + if (promotionRate > GrowThreshold) growAllocableSpace(); - } else if (promotionRate < ShrinkThreshold && previousPromotionRate_ < ShrinkThreshold) { - shrinkAllocableSpace(numChunks() - 1); - } + else if (promotionRate < ShrinkThreshold && previousPromotionRate_ < ShrinkThreshold) + shrinkAllocableSpace(1); previousPromotionRate_ = promotionRate; } -bool +void js::Nursery::growAllocableSpace() { - return growAllocableSpace(Min(numChunks() * 2, maxChunks())); -} - -bool -js::Nursery::growAllocableSpace(unsigned newCount) -{ - unsigned priorCount = numChunks(); - - MOZ_ASSERT(newCount > priorCount); - MOZ_ASSERT(newCount <= maxChunks()); - MOZ_ASSERT(priorCount >= 1); - - if (!chunks_.resize(newCount)) - return false; - - AutoLockGCBgAlloc lock(runtime()); - for (unsigned i = priorCount; i < newCount; i++) { - auto newChunk = runtime()->gc.getOrAllocChunk(lock); - if (!newChunk) { - chunks_.shrinkTo(i); - return false; - } - - chunks_[i] = NurseryChunk::fromChunk(newChunk); - chunk(i).poisonAndInit(runtime(), JS_FRESH_NURSERY_PATTERN); - } - - return true; + updateNumChunks(Min(numChunks() * 2, maxNurseryChunks_)); } void -js::Nursery::freeChunksFrom(unsigned firstFreeChunk) -{ - MOZ_ASSERT(firstFreeChunk < chunks_.length()); - { - AutoLockGC lock(runtime()); - for (unsigned i = firstFreeChunk; i < chunks_.length(); i++) - runtime()->gc.recycleChunk(chunk(i).toChunk(runtime()), lock); - } - chunks_.shrinkTo(firstFreeChunk); -} - -void -js::Nursery::shrinkAllocableSpace(unsigned newCount) +js::Nursery::shrinkAllocableSpace(unsigned removeNumChunks) { #ifdef JS_GC_ZEAL if (runtime()->hasZealMode(ZealMode::GenerationalGC)) return; #endif - - // Don't shrink the nursery to zero (use Nursery::disable() instead) and - // don't attempt to shrink it to the same size. - if ((newCount == 0) || (newCount == numChunks())) - return; - - MOZ_ASSERT(newCount < numChunks()); - - freeChunksFrom(newCount); + updateNumChunks(Max(numChunks() - removeNumChunks, 1u)); } void js::Nursery::minimizeAllocableSpace() { - shrinkAllocableSpace(1); +#ifdef JS_GC_ZEAL + if (runtime()->hasZealMode(ZealMode::GenerationalGC)) + return; +#endif + updateNumChunks(1); } -bool -js::Nursery::allocateFirstChunk(AutoLockGCBgAlloc& lock) +void +js::Nursery::updateNumChunks(unsigned newCount) { - // This assertion isn't required for correctness, but we do assume this - // is only called to initialize or re-enable the nursery. - MOZ_ASSERT(numChunks() == 0); + if (numChunks() != newCount) { + AutoLockGCBgAlloc lock(runtime()); + updateNumChunksLocked(newCount, lock); + } +} - MOZ_ASSERT(maxChunks() > 0); +void +js::Nursery::updateNumChunksLocked(unsigned newCount, + AutoLockGCBgAlloc& lock) +{ + // The GC nursery is an optimization and so if we fail to allocate nursery + // chunks we do not report an error. - if (!chunks_.resize(1)) - return false; + MOZ_ASSERT(newCount <= maxChunks()); - auto chunk = runtime()->gc.getOrAllocChunk(lock); - if (!chunk) { - chunks_.shrinkTo(0); - return false; + unsigned priorCount = numChunks(); + MOZ_ASSERT(priorCount != newCount); + + if (newCount < priorCount) { + // Shrink the nursery and free unused chunks. + for (unsigned i = newCount; i < priorCount; i++) + runtime()->gc.recycleChunk(chunk(i).toChunk(runtime()), lock); + chunks_.shrinkTo(newCount); + return; } - chunks_[0] = NurseryChunk::fromChunk(chunk); + // Grow the nursery and allocate new chunks. + if (!chunks_.resize(newCount)) + return; - return true; + for (unsigned i = priorCount; i < newCount; i++) { + auto newChunk = runtime()->gc.getOrAllocChunk(lock); + if (!newChunk) { + chunks_.shrinkTo(i); + return; + } + + chunks_[i] = NurseryChunk::fromChunk(newChunk); + chunk(i).poisonAndInit(runtime(), JS_FRESH_NURSERY_PATTERN); + } } bool diff --git a/js/src/gc/Nursery.h b/js/src/gc/Nursery.h index 96ef0ffa9074..08ecbe099880 100644 --- a/js/src/gc/Nursery.h +++ b/js/src/gc/Nursery.h @@ -145,6 +145,7 @@ class Nursery unsigned numChunks() const { return chunks_.length(); } bool exists() const { return maxChunks() != 0; } + size_t nurserySize() const { return maxChunks() << ChunkShift; } void enable(); void disable(); @@ -410,6 +411,8 @@ class Nursery Canary* lastCanary_; #endif + NurseryChunk* allocChunk(); + NurseryChunk& chunk(unsigned index) const { return *chunks_[index]; } @@ -417,11 +420,9 @@ class Nursery void setCurrentChunk(unsigned chunkno); void setStartPosition(); - /* - * Ensure that the first chunk has been allocated. Callers will probably - * want to call setCurrentChunk(0) next. - */ - MOZ_MUST_USE bool allocateFirstChunk(AutoLockGCBgAlloc& lock); + void updateNumChunks(unsigned newCount); + void updateNumChunksLocked(unsigned newCount, + AutoLockGCBgAlloc& lock); MOZ_ALWAYS_INLINE uintptr_t currentEnd() const; @@ -473,15 +474,10 @@ class Nursery /* Change the allocable space provided by the nursery. */ void maybeResizeNursery(JS::gcreason::Reason reason); - bool growAllocableSpace(); - bool growAllocableSpace(unsigned newSize); - void shrinkAllocableSpace(unsigned newCount); + void growAllocableSpace(); + void shrinkAllocableSpace(unsigned removeNumChunks); void minimizeAllocableSpace(); - // Free the chunks starting at firstFreeChunk until the end of the chunks - // vector. Shrinks the vector but does not update maxChunksAlloc(). - void freeChunksFrom(unsigned firstFreeChunk); - /* Profile recording and printing. */ void maybeClearProfileDurations(); void startProfile(ProfileKey key); From 1df202b1778b1d821d2d4fe8c090b269e1305e80 Mon Sep 17 00:00:00 2001 From: Chris Peterson Date: Tue, 24 Oct 2017 23:30:31 -0700 Subject: [PATCH 30/32] Bug 1412048 - Replace NS_RUNTIMEABORT("...") with MOZ_CRASH("..."). r=froydnj And remove unreachable code after MOZ_CRASH(). MozReview-Commit-ID: 6ShBtPRKYlF --HG-- extra : rebase_source : 0fe45a59411bda663828336e2686707b550144ae extra : source : 8473fd7333d2abe1ea1cc176510c292a5b34df45 --- dom/ipc/ContentChild.cpp | 12 ++++-------- dom/ipc/ContentParent.cpp | 3 +-- dom/plugins/ipc/PluginInstanceChild.cpp | 12 ++++-------- dom/plugins/ipc/PluginModuleChild.cpp | 11 ++++------- dom/plugins/ipc/PluginModuleParent.cpp | 13 ++++--------- dom/presentation/PresentationConnection.cpp | 3 +-- editor/libeditor/HTMLEditRules.cpp | 2 +- gfx/layers/d3d11/ReadbackManagerD3D11.cpp | 4 ++-- gfx/layers/ipc/ShadowLayers.cpp | 2 +- gfx/tests/gtest/TestLayers.cpp | 13 ++++--------- gfx/vr/ipc/VRManagerChild.cpp | 6 ++---- hal/sandbox/SandboxHal.cpp | 17 +++++++---------- ipc/ipdl/test/cxx/TestCrashCleanup.cpp | 4 +--- ipc/ipdl/test/cxx/TestInterruptErrorCleanup.cpp | 4 +--- ipc/ipdl/test/cxx/TestInterruptShutdownRace.cpp | 4 +--- js/xpconnect/src/XPCJSContext.cpp | 14 ++++++-------- js/xpconnect/src/nsXPConnect.cpp | 2 +- layout/style/StyleSheet.cpp | 2 +- layout/style/nsComputedDOMStyle.cpp | 13 ++++--------- toolkit/xre/AutoSQLiteLifetime.cpp | 2 +- .../gtest/TestDeadlockDetectorScalability.cpp | 4 ++-- 21 files changed, 53 insertions(+), 94 deletions(-) diff --git a/dom/ipc/ContentChild.cpp b/dom/ipc/ContentChild.cpp index 348ea09e861d..5ed07350b25a 100644 --- a/dom/ipc/ContentChild.cpp +++ b/dom/ipc/ContentChild.cpp @@ -3121,8 +3121,7 @@ ContentChild::RecvSetAudioSessionData(const nsID& aId, mozilla::widget::StartAudioSession(); return IPC_OK(); #else - NS_RUNTIMEABORT("Not Reached!"); - return IPC_FAIL_NO_REASON(this); + MOZ_CRASH("Not Reached!"); #endif } @@ -3580,8 +3579,7 @@ ContentChild::RecvShareCodeCoverageMutex(const CrossProcessMutexHandle& aHandle) CodeCoverageHandler::Init(aHandle); return IPC_OK(); #else - NS_RUNTIMEABORT("Shouldn't receive this message in non-code coverage builds!"); - return IPC_FAIL_NO_REASON(this); + MOZ_CRASH("Shouldn't receive this message in non-code coverage builds!"); #endif } @@ -3592,8 +3590,7 @@ ContentChild::RecvDumpCodeCoverageCounters() CodeCoverageHandler::DumpCounters(0); return IPC_OK(); #else - NS_RUNTIMEABORT("Shouldn't receive this message in non-code coverage builds!"); - return IPC_FAIL_NO_REASON(this); + MOZ_CRASH("Shouldn't receive this message in non-code coverage builds!"); #endif } @@ -3604,8 +3601,7 @@ ContentChild::RecvResetCodeCoverageCounters() CodeCoverageHandler::ResetCounters(0); return IPC_OK(); #else - NS_RUNTIMEABORT("Shouldn't receive this message in non-code coverage builds!"); - return IPC_FAIL_NO_REASON(this); + MOZ_CRASH("Shouldn't receive this message in non-code coverage builds!"); #endif } diff --git a/dom/ipc/ContentParent.cpp b/dom/ipc/ContentParent.cpp index d76c67dd8923..17b58afe9b81 100644 --- a/dom/ipc/ContentParent.cpp +++ b/dom/ipc/ContentParent.cpp @@ -1063,8 +1063,7 @@ mozilla::ipc::IPCResult ContentParent::RecvUngrabPointer(const uint32_t& aTime) { #if !defined(MOZ_WIDGET_GTK) - NS_RUNTIMEABORT("This message only makes sense on GTK platforms"); - return IPC_OK(); + MOZ_CRASH("This message only makes sense on GTK platforms"); #else gdk_pointer_ungrab(aTime); return IPC_OK(); diff --git a/dom/plugins/ipc/PluginInstanceChild.cpp b/dom/plugins/ipc/PluginInstanceChild.cpp index ae300d9c8bc1..e5b2382ca8aa 100644 --- a/dom/plugins/ipc/PluginInstanceChild.cpp +++ b/dom/plugins/ipc/PluginInstanceChild.cpp @@ -755,8 +755,7 @@ PluginInstanceChild::AnswerNPP_GetValue_NPPVpluginNativeAccessibleAtkPlugId( #else - NS_RUNTIMEABORT("shouldn't be called on non-ATK platforms"); - return IPC_FAIL_NO_REASON(this); + MOZ_CRASH("shouldn't be called on non-ATK platforms"); #endif } @@ -1075,8 +1074,7 @@ PluginInstanceChild::AnswerNPP_HandleEvent_IOSurface(const NPRemoteEvent& event, const uint32_t &surfaceid, int16_t* handled) { - NS_RUNTIMEABORT("NPP_HandleEvent_IOSurface is a OSX-only message"); - return IPC_FAIL_NO_REASON(this); + MOZ_CRASH("NPP_HandleEvent_IOSurface is a OSX-only message"); } #endif @@ -1090,8 +1088,7 @@ PluginInstanceChild::RecvWindowPosChanged(const NPRemoteEvent& event) int16_t dontcare; return AnswerNPP_HandleEvent(event, &dontcare); #else - NS_RUNTIMEABORT("WindowPosChanged is a windows-only message"); - return IPC_FAIL_NO_REASON(this); + MOZ_CRASH("WindowPosChanged is a windows-only message"); #endif } @@ -1110,8 +1107,7 @@ PluginInstanceChild::RecvContentsScaleFactorChanged(const double& aContentsScale #endif return IPC_OK(); #else - NS_RUNTIMEABORT("ContentsScaleFactorChanged is an Windows or OSX only message"); - return IPC_FAIL_NO_REASON(this); + MOZ_CRASH("ContentsScaleFactorChanged is an Windows or OSX only message"); #endif } diff --git a/dom/plugins/ipc/PluginModuleChild.cpp b/dom/plugins/ipc/PluginModuleChild.cpp index f650bda5413a..98b1fa34d27e 100644 --- a/dom/plugins/ipc/PluginModuleChild.cpp +++ b/dom/plugins/ipc/PluginModuleChild.cpp @@ -728,8 +728,7 @@ PluginModuleChild::RecvSetAudioSessionData(const nsID& aId, const nsString& aIconPath) { #if !defined XP_WIN - NS_RUNTIMEABORT("Not Reached!"); - return IPC_FAIL_NO_REASON(this); + MOZ_CRASH("Not Reached!"); #else nsresult rv = mozilla::widget::RecvAudioSessionData(aId, aDisplayName, aIconPath); NS_ENSURE_SUCCESS(rv, IPC_OK()); // Bail early if this fails @@ -2616,7 +2615,7 @@ PluginModuleChild::RecvProcessNativeEventsInInterruptCall() ProcessNativeEventsInInterruptCall(); return IPC_OK(); #else - NS_RUNTIMEABORT( + MOZ_CRASH( "PluginModuleChild::RecvProcessNativeEventsInInterruptCall not implemented!"); return IPC_FAIL_NO_REASON(this); #endif @@ -2663,8 +2662,7 @@ PluginModuleChild::PluginRequiresAudioDeviceChanges( } return rv; #else - NS_RUNTIMEABORT("PluginRequiresAudioDeviceChanges is not available on this platform."); - return NPERR_GENERIC_ERROR; + MOZ_CRASH("PluginRequiresAudioDeviceChanges is not available on this platform."); #endif // XP_WIN } @@ -2683,7 +2681,6 @@ PluginModuleChild::RecvNPP_SetValue_NPNVaudioDeviceChangeDetails( } return IPC_OK(); #else - NS_RUNTIMEABORT("NPP_SetValue_NPNVaudioDeviceChangeDetails is a Windows-only message"); - return IPC_FAIL_NO_REASON(this); + MOZ_CRASH("NPP_SetValue_NPNVaudioDeviceChangeDetails is a Windows-only message"); #endif } diff --git a/dom/plugins/ipc/PluginModuleParent.cpp b/dom/plugins/ipc/PluginModuleParent.cpp index e5f72b05367d..d53f426f5173 100644 --- a/dom/plugins/ipc/PluginModuleParent.cpp +++ b/dom/plugins/ipc/PluginModuleParent.cpp @@ -1845,9 +1845,7 @@ PluginModuleChromeParent::AnswerNPN_SetValue_NPPVpluginRequiresAudioDeviceChange } return IPC_OK(); #else - NS_RUNTIMEABORT("NPPVpluginRequiresAudioDeviceChanges is not valid on this platform."); - *result = NPERR_GENERIC_ERROR; - return IPC_OK(); + MOZ_CRASH("NPPVpluginRequiresAudioDeviceChanges is not valid on this platform."); #endif } @@ -2582,8 +2580,7 @@ PluginModuleParent::AnswerProcessSomeEvents() mozilla::ipc::IPCResult PluginModuleParent::AnswerProcessSomeEvents() { - NS_RUNTIMEABORT("unreached"); - return IPC_FAIL_NO_REASON(this); + MOZ_CRASH("unreached"); } #else @@ -2812,10 +2809,8 @@ mozilla::ipc::IPCResult PluginModuleParent::AnswerNPN_SetValue_NPPVpluginRequiresAudioDeviceChanges( const bool& shouldRegister, NPError* result) { - NS_RUNTIMEABORT("SetValue_NPPVpluginRequiresAudioDeviceChanges is only valid " - "with PluginModuleChromeParent"); - *result = NPERR_GENERIC_ERROR; - return IPC_OK(); + MOZ_CRASH("SetValue_NPPVpluginRequiresAudioDeviceChanges is only valid " + "with PluginModuleChromeParent"); } #ifdef MOZ_CRASHREPORTER_INJECTOR diff --git a/dom/presentation/PresentationConnection.cpp b/dom/presentation/PresentationConnection.cpp index 9afb9cf237a5..9e83f846d9dd 100644 --- a/dom/presentation/PresentationConnection.cpp +++ b/dom/presentation/PresentationConnection.cpp @@ -577,8 +577,7 @@ PresentationConnection::DoReceiveMessage(const nsACString& aData, bool aIsBinary } jsData.setObject(*arrayBuf); } else { - NS_RUNTIMEABORT("Unknown binary type!"); - return NS_ERROR_UNEXPECTED; + MOZ_CRASH("Unknown binary type!"); } } else { NS_ConvertUTF8toUTF16 utf16Data(aData); diff --git a/editor/libeditor/HTMLEditRules.cpp b/editor/libeditor/HTMLEditRules.cpp index be188e9802d9..6a73715743ab 100644 --- a/editor/libeditor/HTMLEditRules.cpp +++ b/editor/libeditor/HTMLEditRules.cpp @@ -5188,7 +5188,7 @@ HTMLEditRules::CheckForEmptyBlock(nsINode* aStartNode, NS_ENSURE_SUCCESS(rv, rv); } } else if (aAction != nsIEditor::eNone) { - NS_RUNTIMEABORT("CheckForEmptyBlock doesn't support this action yet"); + MOZ_CRASH("CheckForEmptyBlock doesn't support this action yet"); } } NS_ENSURE_STATE(htmlEditor); diff --git a/gfx/layers/d3d11/ReadbackManagerD3D11.cpp b/gfx/layers/d3d11/ReadbackManagerD3D11.cpp index 64d8cbe32fe5..62ed4fd0e62a 100644 --- a/gfx/layers/d3d11/ReadbackManagerD3D11.cpp +++ b/gfx/layers/d3d11/ReadbackManagerD3D11.cpp @@ -105,7 +105,7 @@ ReadbackManagerD3D11::~ReadbackManagerD3D11() ::CloseHandle(mTaskSemaphore); ::CloseHandle(mTaskThread); } else { - NS_RUNTIMEABORT("ReadbackManager: Task thread did not shutdown in 5 seconds."); + MOZ_CRASH("ReadbackManager: Task thread did not shutdown in 5 seconds."); } } @@ -136,7 +136,7 @@ ReadbackManagerD3D11::ProcessTasks() ::EnterCriticalSection(&mTaskMutex); if (mPendingReadbackTasks.Length() == 0) { - NS_RUNTIMEABORT("Trying to read from an empty array, bad bad bad"); + MOZ_CRASH("Trying to read from an empty array, bad bad bad"); } ReadbackTask *nextReadbackTask = mPendingReadbackTasks[0].forget(); mPendingReadbackTasks.RemoveElementAt(0); diff --git a/gfx/layers/ipc/ShadowLayers.cpp b/gfx/layers/ipc/ShadowLayers.cpp index 77b223147729..4f257eb6d675 100644 --- a/gfx/layers/ipc/ShadowLayers.cpp +++ b/gfx/layers/ipc/ShadowLayers.cpp @@ -975,7 +975,7 @@ DestroySurfaceDescriptor(IShmemAllocator* aAllocator, SurfaceDescriptor* aSurfac break; } default: - NS_RUNTIMEABORT("surface type not implemented!"); + MOZ_CRASH("surface type not implemented!"); } *aSurface = SurfaceDescriptor(); } diff --git a/gfx/tests/gtest/TestLayers.cpp b/gfx/tests/gtest/TestLayers.cpp index 3bee497537dd..e56be233586b 100644 --- a/gfx/tests/gtest/TestLayers.cpp +++ b/gfx/tests/gtest/TestLayers.cpp @@ -68,29 +68,24 @@ public: virtual bool BeginTransaction() { return true; } virtual already_AddRefed CreateImageLayer() { MOZ_CRASH("Not implemented."); - return nullptr; } virtual already_AddRefed CreatePaintedLayer() { RefPtr layer = new TestPaintedLayer(this); return layer.forget(); } virtual already_AddRefed CreateColorLayer() { - NS_RUNTIMEABORT("Not implemented."); - return nullptr; + MOZ_CRASH("Not implemented."); } virtual already_AddRefed CreateTextLayer() { - NS_RUNTIMEABORT("Not implemented."); - return nullptr; + MOZ_CRASH("Not implemented."); } virtual already_AddRefed CreateBorderLayer() { - NS_RUNTIMEABORT("Not implemented."); - return nullptr; + MOZ_CRASH("Not implemented."); } virtual void SetRoot(Layer* aLayer) {} virtual bool BeginTransactionWithTarget(gfxContext* aTarget) { return true; } virtual already_AddRefed CreateCanvasLayer() { - NS_RUNTIMEABORT("Not implemented."); - return nullptr; + MOZ_CRASH("Not implemented."); } virtual void EndTransaction(DrawPaintedLayerCallback aCallback, void* aCallbackData, diff --git a/gfx/vr/ipc/VRManagerChild.cpp b/gfx/vr/ipc/VRManagerChild.cpp index 48a63d1d81b5..03dc89771c0e 100644 --- a/gfx/vr/ipc/VRManagerChild.cpp +++ b/gfx/vr/ipc/VRManagerChild.cpp @@ -93,8 +93,7 @@ VRManagerChild::InitForContent(Endpoint&& aEndpoint) RefPtr child(new VRManagerChild()); if (!aEndpoint.Bind(child)) { - NS_RUNTIMEABORT("Couldn't Open() Compositor channel."); - return false; + MOZ_CRASH("Couldn't Open() Compositor channel."); } sVRManagerChildSingleton = child; return true; @@ -131,8 +130,7 @@ VRManagerChild::InitWithGPUProcess(Endpoint&& aEndpoint) sVRManagerChildSingleton = new VRManagerChild(); if (!aEndpoint.Bind(sVRManagerChildSingleton)) { - NS_RUNTIMEABORT("Couldn't Open() Compositor channel."); - return; + MOZ_CRASH("Couldn't Open() Compositor channel."); } } diff --git a/hal/sandbox/SandboxHal.cpp b/hal/sandbox/SandboxHal.cpp index 24ef7b163b27..73b46b9127bc 100644 --- a/hal/sandbox/SandboxHal.cpp +++ b/hal/sandbox/SandboxHal.cpp @@ -219,47 +219,44 @@ DisableSwitchNotifications(SwitchDevice aDevice) bool EnableAlarm() { - NS_RUNTIMEABORT("Alarms can't be programmed from sandboxed contexts. Yet."); - return false; + MOZ_CRASH("Alarms can't be programmed from sandboxed contexts. Yet."); } void DisableAlarm() { - NS_RUNTIMEABORT("Alarms can't be programmed from sandboxed contexts. Yet."); + MOZ_CRASH("Alarms can't be programmed from sandboxed contexts. Yet."); } bool SetAlarm(int32_t aSeconds, int32_t aNanoseconds) { - NS_RUNTIMEABORT("Alarms can't be programmed from sandboxed contexts. Yet."); - return false; + MOZ_CRASH("Alarms can't be programmed from sandboxed contexts. Yet."); } void SetProcessPriority(int aPid, ProcessPriority aPriority) { - NS_RUNTIMEABORT("Only the main process may set processes' priorities."); + MOZ_CRASH("Only the main process may set processes' priorities."); } bool SetProcessPrioritySupported() { - NS_RUNTIMEABORT("Only the main process may call SetProcessPrioritySupported()."); - return false; + MOZ_CRASH("Only the main process may call SetProcessPrioritySupported()."); } void SetCurrentThreadPriority(ThreadPriority aThreadPriority) { - NS_RUNTIMEABORT("Setting current thread priority cannot be called from sandboxed contexts."); + MOZ_CRASH("Setting current thread priority cannot be called from sandboxed contexts."); } void SetThreadPriority(PlatformThreadId aThreadId, ThreadPriority aThreadPriority) { - NS_RUNTIMEABORT("Setting thread priority cannot be called from sandboxed contexts."); + MOZ_CRASH("Setting thread priority cannot be called from sandboxed contexts."); } void diff --git a/ipc/ipdl/test/cxx/TestCrashCleanup.cpp b/ipc/ipdl/test/cxx/TestCrashCleanup.cpp index e122bccdb2ea..6c57441db678 100644 --- a/ipc/ipdl/test/cxx/TestCrashCleanup.cpp +++ b/ipc/ipdl/test/cxx/TestCrashCleanup.cpp @@ -107,10 +107,8 @@ mozilla::ipc::IPCResult TestCrashCleanupChild::AnswerDIEDIEDIE() { _exit(0); - NS_RUNTIMEABORT("unreached"); - return IPC_FAIL_NO_REASON(this); + MOZ_CRASH("unreached"); } - } // namespace _ipdltest } // namespace mozilla diff --git a/ipc/ipdl/test/cxx/TestInterruptErrorCleanup.cpp b/ipc/ipdl/test/cxx/TestInterruptErrorCleanup.cpp index 5e6baaebe1bf..f6b16b098264 100644 --- a/ipc/ipdl/test/cxx/TestInterruptErrorCleanup.cpp +++ b/ipc/ipdl/test/cxx/TestInterruptErrorCleanup.cpp @@ -146,10 +146,8 @@ mozilla::ipc::IPCResult TestInterruptErrorCleanupChild::AnswerError() { _exit(0); - NS_RUNTIMEABORT("unreached"); - return IPC_FAIL_NO_REASON(this); + MOZ_CRASH("unreached"); } - } // namespace _ipdltest } // namespace mozilla diff --git a/ipc/ipdl/test/cxx/TestInterruptShutdownRace.cpp b/ipc/ipdl/test/cxx/TestInterruptShutdownRace.cpp index 5252f538617c..120a4b60b6a7 100644 --- a/ipc/ipdl/test/cxx/TestInterruptShutdownRace.cpp +++ b/ipc/ipdl/test/cxx/TestInterruptShutdownRace.cpp @@ -126,10 +126,8 @@ mozilla::ipc::IPCResult TestInterruptShutdownRaceChild::AnswerExit() { _exit(0); - NS_RUNTIMEABORT("unreached"); - return IPC_FAIL_NO_REASON(this); + MOZ_CRASH("unreached"); } - } // namespace _ipdltest } // namespace mozilla diff --git a/js/xpconnect/src/XPCJSContext.cpp b/js/xpconnect/src/XPCJSContext.cpp index 291e13b1f995..a930c5636750 100644 --- a/js/xpconnect/src/XPCJSContext.cpp +++ b/js/xpconnect/src/XPCJSContext.cpp @@ -127,10 +127,11 @@ class Watchdog MOZ_ASSERT(NS_IsMainThread()); mLock = PR_NewLock(); if (!mLock) - NS_RUNTIMEABORT("PR_NewLock failed."); + MOZ_CRASH("PR_NewLock failed."); + mWakeup = PR_NewCondVar(mLock); if (!mWakeup) - NS_RUNTIMEABORT("PR_NewCondVar failed."); + MOZ_CRASH("PR_NewCondVar failed."); { AutoLockWatchdog lock(this); @@ -142,7 +143,7 @@ class Watchdog PR_PRIORITY_NORMAL, PR_GLOBAL_THREAD, PR_JOINABLE_THREAD, 0); if (!mThread) - NS_RUNTIMEABORT("PR_CreateThread failed!"); + MOZ_CRASH("PR_CreateThread failed!"); // WatchdogMain acquires the lock and then asserts mInitialized. So // make sure to set mInitialized before releasing the lock here so @@ -1148,16 +1149,13 @@ XPCJSContext::NewXPCJSContext(XPCJSContext* aPrimaryContext) XPCJSContext* self = new XPCJSContext(); nsresult rv = self->Initialize(aPrimaryContext); if (NS_FAILED(rv)) { - NS_RUNTIMEABORT("new XPCJSContext failed to initialize."); - delete self; - return nullptr; + MOZ_CRASH("new XPCJSContext failed to initialize."); } if (self->Context()) return self; - NS_RUNTIMEABORT("new XPCJSContext failed to initialize."); - return nullptr; + MOZ_CRASH("new XPCJSContext failed to initialize."); } void diff --git a/js/xpconnect/src/nsXPConnect.cpp b/js/xpconnect/src/nsXPConnect.cpp index 6f19e85ce975..17d82a1a98fa 100644 --- a/js/xpconnect/src/nsXPConnect.cpp +++ b/js/xpconnect/src/nsXPConnect.cpp @@ -73,7 +73,7 @@ nsXPConnect::nsXPConnect() XPCJSContext* xpccx = XPCJSContext::NewXPCJSContext(nullptr); if (!xpccx) { - NS_RUNTIMEABORT("Couldn't create XPCJSContext."); + MOZ_CRASH("Couldn't create XPCJSContext."); } gPrimaryContext = xpccx; mRuntime = xpccx->Runtime(); diff --git a/layout/style/StyleSheet.cpp b/layout/style/StyleSheet.cpp index 9ea477fd77fd..f2375bc72cb4 100644 --- a/layout/style/StyleSheet.cpp +++ b/layout/style/StyleSheet.cpp @@ -243,7 +243,7 @@ StyleSheetInfo::StyleSheetInfo(CORSMode aCORSMode, #endif { if (!mPrincipal) { - NS_RUNTIMEABORT("NullPrincipal::Init failed"); + MOZ_CRASH("NullPrincipal::Init failed"); } } diff --git a/layout/style/nsComputedDOMStyle.cpp b/layout/style/nsComputedDOMStyle.cpp index c2e92cbab643..7d48a297e15e 100644 --- a/layout/style/nsComputedDOMStyle.cpp +++ b/layout/style/nsComputedDOMStyle.cpp @@ -840,30 +840,25 @@ nsComputedDOMStyle::GetPresShellForContent(const nsIContent* aContent) DeclarationBlock* nsComputedDOMStyle::GetCSSDeclaration(Operation) { - NS_RUNTIMEABORT("called nsComputedDOMStyle::GetCSSDeclaration"); - return nullptr; + MOZ_CRASH("called nsComputedDOMStyle::GetCSSDeclaration"); } nsresult nsComputedDOMStyle::SetCSSDeclaration(DeclarationBlock*) { - NS_RUNTIMEABORT("called nsComputedDOMStyle::SetCSSDeclaration"); - return NS_ERROR_FAILURE; + MOZ_CRASH("called nsComputedDOMStyle::SetCSSDeclaration"); } nsIDocument* nsComputedDOMStyle::DocToUpdate() { - NS_RUNTIMEABORT("called nsComputedDOMStyle::DocToUpdate"); - return nullptr; + MOZ_CRASH("called nsComputedDOMStyle::DocToUpdate"); } void nsComputedDOMStyle::GetCSSParsingEnvironment(CSSParsingEnvironment& aCSSParseEnv) { - NS_RUNTIMEABORT("called nsComputedDOMStyle::GetCSSParsingEnvironment"); - // Just in case NS_RUNTIMEABORT ever stops killing us for some reason - aCSSParseEnv.mPrincipal = nullptr; + MOZ_CRASH("called nsComputedDOMStyle::GetCSSParsingEnvironment"); } nsDOMCSSDeclaration::ServoCSSParsingEnvironment diff --git a/toolkit/xre/AutoSQLiteLifetime.cpp b/toolkit/xre/AutoSQLiteLifetime.cpp index a122c4900530..041f470c2821 100644 --- a/toolkit/xre/AutoSQLiteLifetime.cpp +++ b/toolkit/xre/AutoSQLiteLifetime.cpp @@ -131,7 +131,7 @@ namespace mozilla { AutoSQLiteLifetime::AutoSQLiteLifetime() { if (++AutoSQLiteLifetime::sSingletonEnforcer != 1) { - NS_RUNTIMEABORT("multiple instances of AutoSQLiteLifetime constructed!"); + MOZ_CRASH("multiple instances of AutoSQLiteLifetime constructed!"); } #ifdef MOZ_STORAGE_MEMORY diff --git a/xpcom/tests/gtest/TestDeadlockDetectorScalability.cpp b/xpcom/tests/gtest/TestDeadlockDetectorScalability.cpp index e25217223825..1acfb1c340ac 100644 --- a/xpcom/tests/gtest/TestDeadlockDetectorScalability.cpp +++ b/xpcom/tests/gtest/TestDeadlockDetectorScalability.cpp @@ -52,7 +52,7 @@ TEST(DeadlockDetectorScalability, DISABLED_OneLockNDeps) mozilla::Mutex* lock = new mozilla::Mutex("deadlockDetector.scalability.t2.master"); mozilla::Mutex** locks = new mozilla::Mutex*[N]; if (!locks) - NS_RUNTIMEABORT("couldn't allocate lock array"); + MOZ_CRASH("couldn't allocate lock array"); for (int i = 0; i < N; ++i) locks[i] = @@ -94,7 +94,7 @@ TEST(DeadlockDetectorScalability, MaxDepsNsq) mozilla::Mutex** locks = new mozilla::Mutex*[N]; if (!locks) - NS_RUNTIMEABORT("couldn't allocate lock array"); + MOZ_CRASH("couldn't allocate lock array"); for (int i = 0; i < N; ++i) locks[i] = new mozilla::Mutex("deadlockDetector.scalability.t3"); From 73037a0f6ab5ba2f4a9c41b4117a5f57e4751ef2 Mon Sep 17 00:00:00 2001 From: Chris Peterson Date: Tue, 24 Oct 2017 23:38:38 -0700 Subject: [PATCH 31/32] Bug 1412048 - Replace NS_RUNTIMEABORT(var) with MOZ_CRASH_UNSAFE_OOL(var). r=froydnj data-review=francois And remove unreachable code after MOZ_CRASH_UNSAFE_OOL(). MOZ_CRASH_UNSAFE_OOL causes data collection because crash strings are annotated to crash-stats and are publicly visible. Firefox data stewards must do data review on usages of this macro. However, all the crash strings this patch collects with MOZ_CRASH_UNSAFE_OOL are already collected with NS_RUNTIMEABORT. MozReview-Commit-ID: IHmJfuxXSqw --HG-- extra : rebase_source : 031f30934b58a7b87f960e57179641d44aefe5c5 extra : source : fe9f638a56a53c8721eecc4273dcc074c988546e --- gfx/layers/opengl/GLBlitTextureImageHelper.cpp | 2 +- gfx/thebes/gfxTextRun.cpp | 2 +- ipc/glue/MessageChannel.cpp | 2 +- ipc/glue/ProtocolUtils.cpp | 4 ++-- layout/style/nsLayoutStylesheetCache.cpp | 3 ++- netwerk/ipc/NeckoCommon.h | 2 +- toolkit/xre/nsGDKErrorHandler.cpp | 18 +++++++++--------- toolkit/xre/nsX11ErrorHandler.cpp | 3 +-- xpcom/base/nsTraceRefcnt.cpp | 2 +- xpcom/build/XPCOMInit.cpp | 2 +- 10 files changed, 20 insertions(+), 20 deletions(-) diff --git a/gfx/layers/opengl/GLBlitTextureImageHelper.cpp b/gfx/layers/opengl/GLBlitTextureImageHelper.cpp index c13d379921e9..858a56d344f7 100644 --- a/gfx/layers/opengl/GLBlitTextureImageHelper.cpp +++ b/gfx/layers/opengl/GLBlitTextureImageHelper.cpp @@ -189,7 +189,7 @@ GLBlitTextureImageHelper::SetBlitFramebufferForDestTexture(GLuint aTexture) // your texture is not texture complete -- that is, you // allocated a texture name, but didn't actually define its // size via a call to TexImage2D. - NS_RUNTIMEABORT(msg.get()); + MOZ_CRASH_UNSAFE_OOL(msg.get()); } } diff --git a/gfx/thebes/gfxTextRun.cpp b/gfx/thebes/gfxTextRun.cpp index 507bb739ad77..2c16f328d7a8 100644 --- a/gfx/thebes/gfxTextRun.cpp +++ b/gfx/thebes/gfxTextRun.cpp @@ -2095,7 +2095,7 @@ gfxFontGroup::GetDefaultFont() mFamilyList.ToString(familiesString); SprintfLiteral(msg, "unable to find a usable font (%.220s)", NS_ConvertUTF16toUTF8(familiesString).get()); - NS_RUNTIMEABORT(msg); + MOZ_CRASH_UNSAFE_OOL(msg); } return mDefaultFont.get(); diff --git a/ipc/glue/MessageChannel.cpp b/ipc/glue/MessageChannel.cpp index 66fded532917..58a56bcf70aa 100644 --- a/ipc/glue/MessageChannel.cpp +++ b/ipc/glue/MessageChannel.cpp @@ -2784,7 +2784,7 @@ MessageChannel::DebugAbort(const char* file, int line, const char* cond, pending.popFirst(); } - NS_RUNTIMEABORT(why); + MOZ_CRASH_UNSAFE_OOL(why); } void diff --git a/ipc/glue/ProtocolUtils.cpp b/ipc/glue/ProtocolUtils.cpp index df15cf901527..ae3216828e04 100644 --- a/ipc/glue/ProtocolUtils.cpp +++ b/ipc/glue/ProtocolUtils.cpp @@ -303,14 +303,14 @@ FatalError(const char* aProtocolName, const char* aMsg, bool aIsParent) MOZ_CRASH("IPC FatalError in the parent process!"); } else { formattedMessage.AppendLiteral("\". abort()ing as a result."); - NS_RUNTIMEABORT(formattedMessage.get()); + MOZ_CRASH_UNSAFE_OOL(formattedMessage.get()); } } void LogicError(const char* aMsg) { - NS_RUNTIMEABORT(aMsg); + MOZ_CRASH_UNSAFE_OOL(aMsg); } void diff --git a/layout/style/nsLayoutStylesheetCache.cpp b/layout/style/nsLayoutStylesheetCache.cpp index 28ae5f2d490b..531cbd58a60a 100644 --- a/layout/style/nsLayoutStylesheetCache.cpp +++ b/layout/style/nsLayoutStylesheetCache.cpp @@ -773,7 +773,8 @@ ErrorLoadingSheet(nsIURI* aURI, const char* aMsg, FailureAction aFailureAction) #ifdef MOZ_CRASHREPORTER AnnotateCrashReport(aURI); #endif - NS_RUNTIMEABORT(errorMessage.get()); + + MOZ_CRASH_UNSAFE_OOL(errorMessage.get()); } void diff --git a/netwerk/ipc/NeckoCommon.h b/netwerk/ipc/NeckoCommon.h index f9309af32e05..f8980eb68440 100644 --- a/netwerk/ipc/NeckoCommon.h +++ b/netwerk/ipc/NeckoCommon.h @@ -37,7 +37,7 @@ class TabChild; if (abort) { \ msg.AppendLiteral(" (set NECKO_ERRORS_ARE_FATAL=0 in your environment " \ "to convert this error into a warning.)"); \ - NS_RUNTIMEABORT(msg.get()); \ + MOZ_CRASH_UNSAFE_OOL(msg.get()); \ } else { \ msg.AppendLiteral(" (set NECKO_ERRORS_ARE_FATAL=1 in your environment " \ "to convert this warning into a fatal error.)"); \ diff --git a/toolkit/xre/nsGDKErrorHandler.cpp b/toolkit/xre/nsGDKErrorHandler.cpp index e0891f479260..2ae1e1756b64 100644 --- a/toolkit/xre/nsGDKErrorHandler.cpp +++ b/toolkit/xre/nsGDKErrorHandler.cpp @@ -40,42 +40,42 @@ GdkErrorHandler(const gchar *log_domain, GLogLevelFlags log_level, NS_NAMED_LITERAL_CSTRING(serialString, "(Details: serial "); int32_t start = buffer.Find(serialString); if (start == kNotFound) - NS_RUNTIMEABORT(message); + MOZ_CRASH_UNSAFE_OOL(message); start += serialString.Length(); errno = 0; event.serial = strtol(buffer.BeginReading() + start, &endptr, 10); if (errno) - NS_RUNTIMEABORT(message); + MOZ_CRASH_UNSAFE_OOL(message); NS_NAMED_LITERAL_CSTRING(errorCodeString, " error_code "); if (!StringBeginsWith(Substring(endptr, buffer.EndReading()), errorCodeString)) - NS_RUNTIMEABORT(message); + MOZ_CRASH_UNSAFE_OOL(message); errno = 0; event.error_code = strtol(endptr + errorCodeString.Length(), &endptr, 10); if (errno) - NS_RUNTIMEABORT(message); + MOZ_CRASH_UNSAFE_OOL(message); NS_NAMED_LITERAL_CSTRING(requestCodeString, " request_code "); if (!StringBeginsWith(Substring(endptr, buffer.EndReading()), requestCodeString)) - NS_RUNTIMEABORT(message); + MOZ_CRASH_UNSAFE_OOL(message); errno = 0; event.request_code = strtol(endptr + requestCodeString.Length(), &endptr, 10); if (errno) - NS_RUNTIMEABORT(message); + MOZ_CRASH_UNSAFE_OOL(message); NS_NAMED_LITERAL_CSTRING(minorCodeString, " minor_code "); start = buffer.Find(minorCodeString, /* aIgnoreCase = */ false, endptr - buffer.BeginReading()); if (!start) - NS_RUNTIMEABORT(message); + MOZ_CRASH_UNSAFE_OOL(message); errno = 0; event.minor_code = strtol(buffer.BeginReading() + start + minorCodeString.Length(), nullptr, 10); if (errno) - NS_RUNTIMEABORT(message); + MOZ_CRASH_UNSAFE_OOL(message); event.display = GDK_DISPLAY_XDISPLAY(gdk_display_get_default()); // Gdk does not provide resource ID @@ -84,7 +84,7 @@ GdkErrorHandler(const gchar *log_domain, GLogLevelFlags log_level, X11Error(event.display, &event); } else { g_log_default_handler(log_domain, log_level, message, user_data); - NS_RUNTIMEABORT(message); + MOZ_CRASH_UNSAFE_OOL(message); } } diff --git a/toolkit/xre/nsX11ErrorHandler.cpp b/toolkit/xre/nsX11ErrorHandler.cpp index d0fe61ae1154..15bddd7290b7 100644 --- a/toolkit/xre/nsX11ErrorHandler.cpp +++ b/toolkit/xre/nsX11ErrorHandler.cpp @@ -144,8 +144,7 @@ X11Error(Display *display, XErrorEvent *event) { #endif #endif - NS_RUNTIMEABORT(notes.get()); - return 0; // not reached + MOZ_CRASH_UNSAFE_OOL(notes.get()); } } diff --git a/xpcom/base/nsTraceRefcnt.cpp b/xpcom/base/nsTraceRefcnt.cpp index fba97a53e97a..b007b4fa1384 100644 --- a/xpcom/base/nsTraceRefcnt.cpp +++ b/xpcom/base/nsTraceRefcnt.cpp @@ -166,7 +166,7 @@ AssertActivityIsLegal() { if (gActivityTLS == BAD_TLS_INDEX || PR_GetThreadPrivate(gActivityTLS)) { if (PR_GetEnv("MOZ_FATAL_STATIC_XPCOM_CTORS_DTORS")) { - NS_RUNTIMEABORT(kStaticCtorDtorWarning); + MOZ_CRASH_UNSAFE_OOL(kStaticCtorDtorWarning); } else { NS_WARNING(kStaticCtorDtorWarning); } diff --git a/xpcom/build/XPCOMInit.cpp b/xpcom/build/XPCOMInit.cpp index 501a6b8bc15a..db66ad797878 100644 --- a/xpcom/build/XPCOMInit.cpp +++ b/xpcom/build/XPCOMInit.cpp @@ -671,7 +671,7 @@ NS_InitXPCOM2(nsIServiceManager** aResult, // Initialize the JS engine. const char* jsInitFailureReason = JS_InitWithFailureDiagnostic(); if (jsInitFailureReason) { - NS_RUNTIMEABORT(jsInitFailureReason); + MOZ_CRASH_UNSAFE_OOL(jsInitFailureReason); } sInitializedJS = true; From 12d00a2d709b86a7ca33792b47e5fae6648a1694 Mon Sep 17 00:00:00 2001 From: JerryShih Date: Tue, 31 Oct 2017 13:52:41 +0800 Subject: [PATCH 32/32] Bug 1413043 - check WR enabling status before the WR calls. r=pchang MozReview-Commit-ID: 29mkllmDQ0L --- gfx/thebes/gfxPlatform.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gfx/thebes/gfxPlatform.cpp b/gfx/thebes/gfxPlatform.cpp index a1d52a4f7137..8c4b4de6cb91 100644 --- a/gfx/thebes/gfxPlatform.cpp +++ b/gfx/thebes/gfxPlatform.cpp @@ -1008,6 +1008,10 @@ gfxPlatform::Shutdown() gfx::Factory::ShutDown(); + if (gfxConfig::IsEnabled(Feature::WEBRENDER)) { + mozilla::wr::wr_shutdown_external_log_handler(); + } + delete gGfxPlatformPrefsLock; gfxVars::Shutdown(); @@ -1016,8 +1020,6 @@ gfxPlatform::Shutdown() gfxConfig::Shutdown(); - mozilla::wr::wr_shutdown_external_log_handler(); - gPlatform->WillShutdown(); delete gPlatform;