Merge mozilla-central to autoland a=merge on a CLOSED TREE

browser/base/content/browser-feeds.js

@@ -593,6 +593,9 @@ var FeedHandler = {
           LOG(`Invalid reader ${settings.reader}`);
           return;
         }
+
+        Services.telemetry.scalarAdd("browser.feeds.feed_subscribed", 1);
+
         const actionPref = getPrefActionForType(settings.feedType);
         this._setPref(actionPref, settings.action);
         const readerPref = getPrefReaderForType(settings.feedType);

browser/components/feeds/FeedWriter.js

@@ -70,6 +70,7 @@ function FeedWriter() {
   this._subscribeCallback = null;
   this._defaultHandlerMenuItem = null;
 
+  Services.telemetry.scalarAdd("browser.feeds.preview_loaded", 1);
 
   XPCOMUtils.defineLazyGetter(this, "_mm", () =>
     this._window.QueryInterface(Ci.nsIInterfaceRequestor).

browser/components/feeds/moz.build

@@ -6,6 +6,7 @@
 
 MOCHITEST_CHROME_MANIFESTS += ['test/chrome/chrome.ini']
 MOCHITEST_MANIFESTS += ['test/mochitest.ini']
+BROWSER_CHROME_MANIFESTS += ['test/browser/browser.ini']
 
 JAR_MANIFESTS += ['jar.mn']
 

browser/components/feeds/test/browser/.eslintrc.js

@@ -0,0 +1,7 @@
+"use strict";
+
+module.exports = {
+  "extends": [
+    "plugin:mozilla/browser-test"
+  ]
+};

browser/components/feeds/test/browser/browser.ini

@@ -0,0 +1,3 @@
+[browser_telemetry_checks.js]
+support-files =
+  valid-feed.xml

browser/components/feeds/test/browser/browser_telemetry_checks.js

@@ -0,0 +1,97 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+add_task(async function() {
+  function getSnapShot() {
+    return Services.telemetry.snapshotScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTOUT, false);
+  }
+  const TEST_PATH = getRootDirectory(gTestPath).replace("chrome://mochitests/content", "http://example.com");
+  const FEED_URI = TEST_PATH + "valid-feed.xml";
+
+  // Ensure we don't have any pre-existing telemetry that'd mess up counts in here:
+  Services.telemetry.clearScalars();
+  const kScalarPrefix = "browser.feeds.";
+  const kPreviewLoaded = kScalarPrefix + "preview_loaded";
+  const kSubscribed = kScalarPrefix + "feed_subscribed";
+  const kLivemarkCount = kScalarPrefix + "livebookmark_count";
+  const kLivemarkOpened = kScalarPrefix + "livebookmark_opened";
+  const kLivemarkItemOpened = kScalarPrefix + "livebookmark_item_opened";
+
+  let scalarForContent = gMultiProcessBrowser ? "content" : "parent";
+  let tab = await BrowserTestUtils.openNewForegroundTab(gBrowser, FEED_URI);
+  // Ensure we get telemetry from the content process:
+  let previewCount = await TestUtils.waitForCondition(() => {
+    let snapshot = getSnapShot()[scalarForContent];
+    return snapshot && snapshot[kPreviewLoaded];
+  });
+  Assert.equal(previewCount, 1, "Should register the preview in telemetry.");
+
+  // Now check subscription. We stub out the actual code for adding the live bookmark,
+  // because the dialog creates an initial copy of the bookmark when it's opened and
+  // that's hard to deal with deterministically in tests.
+  let old = PlacesCommandHook.addLiveBookmark;
+  let createBMPromise = new Promise(resolve => {
+    PlacesCommandHook.addLiveBookmark = function(...args) {
+      resolve(args);
+      // Return the promise because Feeds.jsm expects a promise:
+      return createBMPromise;
+    };
+  });
+  registerCleanupFunction(() => PlacesCommandHook.addLiveBookmark = old);
+  await BrowserTestUtils.synthesizeMouseAtCenter("#subscribeButton", {}, tab.linkedBrowser);
+  let bmArgs = await createBMPromise;
+  Assert.deepEqual(bmArgs, [FEED_URI, "Example Feed", ""], "Should have been trying to subscribe");
+  let snapshot = getSnapShot();
+  Assert.equal(snapshot.parent[kSubscribed], 1, "Should have subscribed once.");
+
+  // Now manually add a livemark in the menu and one in the bookmarks toolbar:
+  let livemarks = await Promise.all([
+    PlacesUtils.livemarks.addLivemark({
+      parentGuid: PlacesUtils.bookmarks.menuGuid,
+      feedURI: Services.io.newURI(FEED_URI),
+    }),
+    PlacesUtils.livemarks.addLivemark({
+      parentGuid: PlacesUtils.bookmarks.toolbarGuid,
+      feedURI: Services.io.newURI(FEED_URI),
+    }),
+  ]);
+  registerCleanupFunction(async () => {
+    for (let mark of livemarks) {
+      await PlacesUtils.livemarks.removeLivemark(mark);
+    }
+  });
+
+  if (document.getElementById("PersonalToolbar").getAttribute("collapsed") == "true") {
+    CustomizableUI.setToolbarVisibility("PersonalToolbar", true);
+    registerCleanupFunction(() => CustomizableUI.setToolbarVisibility("PersonalToolbar", false));
+  }
+
+  // Force updating telemetry:
+  let {PlacesDBUtils} = ChromeUtils.import("resource://gre/modules/PlacesDBUtils.jsm", {});
+  await PlacesDBUtils._telemetryForFeeds();
+  Assert.equal(getSnapShot().parent[kLivemarkCount], 2,
+    "Should have created two livemarks and counted them.");
+
+  info("Waiting for livemark");
+  // Check we count opening the livemark popup:
+  let livemarkOnToolbar = await TestUtils.waitForCondition(
+    () => document.querySelector("#PersonalToolbar .bookmark-item[livemark]"));
+  let popup = livemarkOnToolbar.querySelector("menupopup");
+  let popupShownPromise = BrowserTestUtils.waitForEvent(popup, "popupshown");
+  info("Clicking on livemark");
+  // Using .click() or .doCommand() doesn't seem to work.
+  EventUtils.synthesizeMouseAtCenter(livemarkOnToolbar, {});
+  await popupShownPromise;
+  Assert.equal(getSnapShot().parent[kLivemarkOpened], 1, "Should count livemark opening");
+
+  // And opening an item in the popup:
+  let item = await TestUtils.waitForCondition(
+    () => popup.querySelector("menuitem.bookmark-item"));
+  item.doCommand();
+  Assert.equal(getSnapShot().parent[kLivemarkItemOpened], 1, "Should count livemark item opening");
+  popup.hidePopup();
+  await BrowserTestUtils.removeTab(tab);
+});
+

browser/components/feeds/test/browser/valid-feed.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+
+  <title>Example Feed</title>
+  <link href="http://example.org/"/>
+  <updated>2010-08-22T18:30:02Z</updated>
+
+  <author>
+    <name>John Doe</name>
+  </author>
+  <id>urn:uuid:e2df8375-99be-4848-b05e-b9d407555267</id>
+
+  <entry>
+
+    <title>Item</title>
+    <link href="http://example.org/first"/>
+    <id>urn:uuid:9e0f4bed-33d3-4a9d-97ab-ecaa31b3f14a</id>
+    <updated>2010-08-22T18:30:02Z</updated>
+
+    <summary>Some text.</summary>
+  </entry>
+
+</feed>

browser/components/places/PlacesUIUtils.jsm

@@ -748,6 +748,10 @@ this.PlacesUIUtils = {
     }
 
     this._openNodeIn(aNode, where, window);
+    let view = this.getViewForNode(aEvent.target);
+    if (view && view.controller.hasCachedLivemarkInfo(aNode.parent)) {
+      Services.telemetry.scalarAdd("browser.feeds.livebookmark_item_opened", 1);
+    }
   },
 
   /**

browser/components/places/content/browserPlacesViews.js

@@ -689,6 +689,10 @@ PlacesViewBase.prototype = {
 
         PlacesUtils.livemarks.getLivemark({ id: aPlacesNode.itemId })
           .then(aLivemark => {
+            if (!this.controller) {
+              // We might have been destroyed in the interim...
+              return;
+            }
             let shouldInvalidate =
               !this.controller.hasCachedLivemarkInfo(aPlacesNode);
             this.controller.cacheLivemarkInfo(aPlacesNode, aLivemark);
@@ -932,6 +936,9 @@ PlacesViewBase.prototype = {
     }
 
     if (popup._placesNode && PlacesUIUtils.getViewForNode(popup) == this) {
+      if (this.controller.hasCachedLivemarkInfo(popup._placesNode)) {
+        Services.telemetry.scalarAdd("browser.feeds.livebookmark_opened", 1);
+      }
       if (!popup._placesNode.containerOpen)
         popup._placesNode.containerOpen = true;
       if (!popup._built)

dom/media/webrtc/MediaEngineWebRTCAudio.cpp

@@ -586,7 +586,12 @@ MediaEngineWebRTCMicrophoneSource::Allocate(const dom::MediaTrackConstraints &aC
   if (NS_FAILED(rv)) {
     return rv;
   }
-  mAllocations.AppendElement(Allocation(handle));
+
+  {
+    MutexAutoLock lock(mMutex);
+    mAllocations.AppendElement(Allocation(handle));
+  }
+
   handle.forget(aOutHandle);
   return NS_OK;
 }

gfx/thebes/gfxPlatformFontList.cpp

@@ -367,7 +367,14 @@ gfxPlatformFontList::InitOtherFamilyNames(bool aDeferOtherFamilyNamesLoading)
         return;
     }
 
-    if (aDeferOtherFamilyNamesLoading) {
+    // If the font loader delay has been set to zero, we don't defer loading
+    // additional family names (regardless of the aDefer... parameter), as we
+    // take this to mean availability of font info is to be prioritized over
+    // potential startup perf or main-thread jank.
+    // (This is used so we can reliably run reftests that depend on localized
+    // font-family names being available.)
+    if (aDeferOtherFamilyNamesLoading &&
+        Preferences::GetUint(FONT_LOADER_DELAY_PREF) > 0) {
         if (!mPendingOtherFamilyNameTask) {
             RefPtr<mozilla::CancelableRunnable> task = new InitOtherFamilyNamesRunnable();
             mPendingOtherFamilyNameTask = task;

js/moz.configure

@@ -35,6 +35,7 @@ def js_disable_shell(value):
 
 set_config('JS_DISABLE_SHELL', js_disable_shell)
 
+set_define('JS_64BIT', depends(target)(lambda t: t.bitness == 64 or None))
 
 set_define('JS_PUNBOX64', depends(target)(lambda t: t.bitness == 64 or None))
 set_define('JS_NUNBOX32', depends(target)(lambda t: t.bitness == 32 or None))

js/src/jit-test/tests/SIMD/bug1435317.js

@@ -1,3 +1,5 @@
+load(libdir + 'simd.js');
+
 var ab = new ArrayBuffer(64 * 1024);
 var arr = new Uint8Array(ab);
 
@@ -21,4 +23,3 @@ function testSimdX4() {
 
 setJitCompilerOption('ion.warmup.trigger', 0);
 testSimdX4();
-

js/src/jit-test/tests/wasm/spec/harness/index.js

@@ -229,7 +229,7 @@ function get(instance, name) {
     // code.
 
     if (typeof WebAssembly.Global === "function")
-	return ValueResult(Number(instance.value.exports[name]));
+        return ValueResult(Number(instance.value.exports[name]));
 
     return ValueResult(instance.value.exports[name]);
 }

js/src/jit/MacroAssembler.h

@@ -1306,7 +1306,7 @@ class MacroAssembler : public MacroAssemblerSpecific
         DEFINED_ON(x86);
     template <typename T>
     void branchValueIsNurseryCellImpl(Condition cond, const T& value, Register temp, Label* label)
-        DEFINED_ON(arm64, mips64, x64);
+        DEFINED_ON(arm64, x64);
 
     template <typename T>
     inline void branchTestUndefinedImpl(Condition cond, const T& t, Label* label)

js/src/jit/mips-shared/MacroAssembler-mips-shared.cpp

@@ -1620,6 +1620,15 @@ MacroAssembler::pushFakeReturnAddress(Register scratch)
     return retAddr;
 }
 
+void
+MacroAssembler::loadStoreBuffer(Register ptr, Register buffer)
+{
+    if (ptr != buffer)
+        movePtr(ptr, buffer);
+    orPtr(Imm32(gc::ChunkMask), buffer);
+    loadPtr(Address(buffer, gc::ChunkStoreBufferOffsetFromLastByte), buffer);
+}
+
 void
 MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr, Register temp,
                                         Label* label)

js/src/jit/mips32/MacroAssembler-mips32.cpp

@@ -2471,6 +2471,19 @@ MacroAssembler::branchValueIsNurseryCell(Condition cond, ValueOperand value,
     bind(&done);
 }
 
+void
+MacroAssembler::branchValueIsNurseryObject(Condition cond, ValueOperand value,
+                                           Register temp, Label* label)
+{
+    MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+    Label done;
+
+    branchTestObject(Assembler::NotEqual, value, cond == Assembler::Equal ? &done : label);
+    branchPtrInNurseryChunk(cond, value.payloadReg(), temp, label);
+
+    bind(&done);
+}
+
 void
 MacroAssembler::branchTestValue(Condition cond, const ValueOperand& lhs,
                                 const Value& rhs, Label* label)

js/src/jit/mips64/MacroAssembler-mips64.cpp

@@ -2300,32 +2300,34 @@ void
 MacroAssembler::branchValueIsNurseryCell(Condition cond, const Address& address, Register temp,
                                          Label* label)
 {
-    branchValueIsNurseryCellImpl(cond, address, temp, label);
+    MOZ_ASSERT(temp != InvalidReg);
+    loadValue(address, ValueOperand(temp));
+    branchValueIsNurseryCell(cond, ValueOperand(temp), InvalidReg, label);
 }
 
 void
-MacroAssembler::branchValueIsNurseryCell(Condition cond, ValueOperand value,
-                                         Register temp, Label* label)
-{
-    branchValueIsNurseryCellImpl(cond, value, temp, label);
-}
-
-template <typename T>
-void
-MacroAssembler::branchValueIsNurseryCellImpl(Condition cond, const T& value, Register temp,
-                                             Label* label)
+MacroAssembler::branchValueIsNurseryCell(Condition cond, ValueOperand value, Register temp,
+                                         Label* label)
 {
     MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
 
-    Label done, checkAddress;
-    branchTestObject(Assembler::Equal, value, &checkAddress);
-    branchTestString(Assembler::NotEqual, value, cond == Assembler::Equal ? &done : label);
+    Label done, checkAddress, checkObjectAddress;
+    SecondScratchRegisterScope scratch2(*this);
+
+    splitTag(value, scratch2);
+    branchTestObject(Assembler::Equal, scratch2, &checkObjectAddress);
+    branchTestString(Assembler::NotEqual, scratch2, cond == Assembler::Equal ? &done : label);
+
+    unboxString(value, scratch2);
+    jump(&checkAddress);
+
+    bind(&checkObjectAddress);
+    unboxObject(value, scratch2);
 
     bind(&checkAddress);
-    extractCell(value, SecondScratchReg);
-    orPtr(Imm32(gc::ChunkMask), SecondScratchReg);
-    branch32(cond, Address(SecondScratchReg, gc::ChunkLocationOffsetFromLastByte),
-             Imm32(int32_t(gc::ChunkLocation::Nursery)), label);
+    orPtr(Imm32(gc::ChunkMask), scratch2);
+    load32(Address(scratch2, gc::ChunkLocationOffsetFromLastByte), scratch2);
+    branch32(cond, scratch2, Imm32(int32_t(gc::ChunkLocation::Nursery)), label);
 
     bind(&done);
 }

js/src/jit/mips64/MacroAssembler-mips64.h

@@ -447,13 +447,6 @@ class MacroAssemblerMIPS64Compat : public MacroAssemblerMIPS64
         unboxSymbol(value, scratch);
         return scratch;
     }
-    Register extractCell(const Address& address, Register scratch) {
-        return extractObject(address, scratch);
-    }
-    Register extractCell(const ValueOperand& value, Register scratch) {
-        unboxNonDouble(value, scratch);
-        return scratch;
-    }
     Register extractInt32(const ValueOperand& value, Register scratch) {
         unboxInt32(value, scratch);
         return scratch;

js/src/jstypes.h

@@ -138,36 +138,6 @@
 #define JS_HOWMANY(x,y) (((x)+(y)-1)/(y))
 #define JS_ROUNDUP(x,y) (JS_HOWMANY(x,y)*(y))
 
-/*
- * Define JS_64BIT iff we are building in an environment with 64-bit
- * addresses.
- */
-#ifdef _MSC_VER
-# if defined(_M_X64) || defined(_M_AMD64)
-#  define JS_64BIT
-# endif
-#elif defined(__GNUC__)
-/* Additional GCC defines are when running on Solaris, AIX, and HPUX */
-# if defined(__x86_64__) || defined(__sparcv9) || \
-        defined(__64BIT__) || defined(__LP64__)
-#  define JS_64BIT
-# endif
-#elif defined(__SUNPRO_C) || defined(__SUNPRO_CC) /* Sun Studio C/C++ */
-# if defined(__x86_64) || defined(__sparcv9)
-#  define JS_64BIT
-# endif
-#elif defined(__xlc__) || defined(__xlC__)        /* IBM XL C/C++ */
-# if defined(__64BIT__)
-#  define JS_64BIT
-# endif
-#elif defined(__HP_cc) || defined(__HP_aCC)       /* HP-UX cc/aCC */
-# if defined(__LP64__)
-#  define JS_64BIT
-# endif
-#else
-# error "Implement me"
-#endif
-
 #define JS_BITS_PER_BYTE 8
 #define JS_BITS_PER_BYTE_LOG2 3
 

js/src/wasm/WasmBinaryToAST.cpp

@@ -392,12 +392,17 @@ AstDecodeGetBlockRef(AstDecodeContext& c, uint32_t depth, AstRef* ref)
 static bool
 AstDecodeBrTable(AstDecodeContext& c)
 {
+    bool unreachable = c.iter().currentBlockHasPolymorphicBase();
+
     Uint32Vector depths;
     uint32_t defaultDepth;
     ExprType type;
     if (!c.iter().readBrTable(&depths, &defaultDepth, &type, nullptr, nullptr))
         return false;
 
+    if (unreachable)
+        return true;
+
     AstRefVector table(c.lifo);
     if (!table.resize(depths.length()))
         return false;
@@ -652,15 +657,19 @@ AstDecodeSelect(AstDecodeContext& c)
     if (!c.iter().readSelect(&type, nullptr, nullptr, nullptr))
         return false;
 
+    if (c.iter().currentBlockHasPolymorphicBase())
+        return true;
+
     AstDecodeStackItem selectFalse = c.popCopy();
     AstDecodeStackItem selectTrue = c.popCopy();
     AstDecodeStackItem cond = c.popCopy();
 
-    AstTernaryOperator* ternary = new(c.lifo) AstTernaryOperator(Op::Select, cond.expr, selectTrue.expr, selectFalse.expr);
-    if (!ternary)
+    auto* select = new(c.lifo) AstTernaryOperator(Op::Select, cond.expr, selectTrue.expr,
+                                                  selectFalse.expr);
+    if (!select)
         return false;
 
-    if (!c.push(AstDecodeStackItem(ternary)))
+    if (!c.push(AstDecodeStackItem(select)))
         return false;
 
     return true;

js/src/wasm/WasmBinaryToText.cpp

@@ -1663,8 +1663,13 @@ RenderResizableMemory(WasmRenderContext& c, const Limits& memory)
     resizedMemory.initial /= PageSize;
 
     if (resizedMemory.maximum) {
-        MOZ_ASSERT(*resizedMemory.maximum % PageSize == 0);
-        *resizedMemory.maximum /= PageSize;
+        if (*resizedMemory.maximum == UINT32_MAX) {
+            // See special casing in DecodeMemoryLimits.
+            *resizedMemory.maximum = MaxMemoryMaximumPages;
+        } else {
+            MOZ_ASSERT(*resizedMemory.maximum % PageSize == 0);
+            *resizedMemory.maximum /= PageSize;
+        }
     }
 
     if (!RenderLimits(c, resizedMemory))

layout/reftests/font-matching/reftest.list

@@ -1,3 +1,9 @@
+# Run the font loader eagerly: this also ensures we will eagerly load localized
+# family names when looking up a requested font-family. Otherwise, tests here
+# may fail if they're running too soon after browser startup, and localized
+# names have not yet been read. (See bug 1439937.)
+default-preferences pref(gfx.font_loader.delay,0)
+
 == CSS21-t1502-no-inherited-font-family.xhtml CSS21-t1502-no-inherited-font-family-ref.xhtml
 
 # tests for bug 1394311 (case-insensitive lang tag processing)
@@ -145,3 +151,6 @@ skip-if(gtkWidget||/^Windows\x20NT\x206\.1/.test(http.oscpu)) == emoji-fallback-
 # and are not expected to pass on platforms that don't have those same fonts.
 skip-if(!cocoaWidget) == legacy-family-names-1.html legacy-family-names-1-ref.html
 skip-if(!winWidget) == legacy-family-names-2.html legacy-family-names-2-ref.html
+
+# Reset default prefs.
+default-preferences

python/moz.build

@@ -29,6 +29,9 @@ with Files('mozversioncontrol/**'):
 with Files('l10n/**'):
     BUG_COMPONENT = ('Core', 'Localization')
 
+with Files('mozrelease/**'):
+    BUG_COMPONENT = ('Release Engineering', 'Release Automation')
+
 with Files('mach_commands.py'):
     BUG_COMPONENT = ('Testing', 'Python Test')
 

python/mozrelease/mozrelease/chunking.py

@@ -0,0 +1,24 @@
+from __future__ import absolute_import
+
+from copy import copy
+
+
+class ChunkingError(Exception):
+    pass
+
+
+def getChunk(things, chunks, thisChunk):
+    if thisChunk > chunks:
+        raise ChunkingError("thisChunk (%d) is greater than total chunks (%d)" %
+                           (thisChunk, chunks))
+    possibleThings = copy(things)
+    nThings = len(possibleThings)
+    for c in range(1, chunks + 1):
+        n = nThings / chunks
+        # If our things aren't evenly divisible by the number of chunks
+        # we need to append one more onto some of them
+        if c <= (nThings % chunks):
+            n += 1
+        if c == thisChunk:
+            return possibleThings[0:n]
+        del possibleThings[0:n]

python/mozrelease/mozrelease/l10n.py

@@ -0,0 +1,17 @@
+from __future__ import absolute_import
+
+from .platforms import shippedLocales2ftp
+
+
+def getPlatformLocales(shipped_locales, platform):
+    platform_locales = []
+    for line in shipped_locales.splitlines():
+        entry = line.strip().split()
+        locale = entry[0]
+        if len(entry) > 1:
+            for sl_platform in entry[1:]:
+                if platform in shippedLocales2ftp(sl_platform):
+                    platform_locales.append(locale)
+        else:
+            platform_locales.append(locale)
+    return platform_locales

python/mozrelease/mozrelease/paths.py

@@ -0,0 +1,63 @@
+from __future__ import absolute_import
+
+from urlparse import urlunsplit
+
+product_ftp_map = {
+    "fennec": "mobile",
+}
+
+def product2ftp(product):
+    return product_ftp_map.get(product, product)
+
+
+def getCandidatesDir(product, version, buildNumber, protocol=None, server=None):
+    if protocol:
+        assert server is not None, "server is required with protocol"
+
+    product = product2ftp(product)
+    directory = "/{}/candidates/{}-candidates/build{}".format(
+        product, str(version), str(buildNumber)
+    )
+
+    if protocol:
+        return urlunsplit((protocol, server, directory, None, None))
+    else:
+        return directory
+
+
+def getReleasesDir(product, version=None, protocol=None, server=None):
+    if protocol:
+        assert server is not None, "server is required with protocol"
+
+    directory = "/{}/releases".format(product)
+    if version:
+        directory = "{}/{}".format(directory, version)
+
+    if protocol:
+        return urlunsplit((protocol, server, directory, None, None))
+    else:
+        return directory
+
+
+def getReleaseInstallerPath(productName, brandName, version, platform,
+                            locale='en-US'):
+    if productName not in ('fennec',):
+        if platform.startswith('linux'):
+            return '/'.join([p.strip('/') for p in [
+                platform, locale, '%s-%s.tar.bz2' % (productName, version)]])
+        elif 'mac' in platform:
+            return '/'.join([p.strip('/') for p in [
+                platform, locale, '%s %s.dmg' % (brandName, version)]])
+        elif platform.startswith('win'):
+            return '/'.join([p.strip('/') for p in [
+                platform, locale, '%s Setup %s.exe' % (brandName, version)]])
+        else:
+            raise "Unsupported platform"
+    else:
+        if platform.startswith('android'):
+            filename = '%s-%s.%s.android-arm.apk' % (
+                productName, version, locale)
+            return '/'.join([p.strip('/') for p in [
+                platform, locale, filename]])
+        else:
+            raise "Unsupported platform"

python/mozrelease/mozrelease/platforms.py

@@ -0,0 +1,60 @@
+from __future__ import absolute_import
+
+# ftp -> update platform map
+update_platform_map = {
+    "android": ["Android_arm-eabi-gcc3"],
+    "android-api-11": ["Android_arm-eabi-gcc3"],
+    "android-api-15": ["Android_arm-eabi-gcc3"],
+    "android-api-15-old-id": ["Android_arm-eabi-gcc3"],
+    "android-api-16": ["Android_arm-eabi-gcc3"],
+    "android-api-16-old-id": ["Android_arm-eabi-gcc3"],
+    "android-x86": ["Android_x86-gcc3"],
+    "android-x86-old-id": ["Android_x86-gcc3"],
+    "android-aarch64": ["Android_aarch64-gcc3"],
+    "linux-i686": ["Linux_x86-gcc3"],
+    "linux-x86_64": ["Linux_x86_64-gcc3"],
+    "mac": ["Darwin_x86_64-gcc3-u-i386-x86_64",  # The main platofrm
+            "Darwin_x86-gcc3-u-i386-x86_64",
+            # We don"t ship builds with these build targets, but some users
+            # modify their builds in a way that has them report like these.
+            # See bug 1071576 for details.
+            "Darwin_x86-gcc3", "Darwin_x86_64-gcc3"],
+    "win32": ["WINNT_x86-msvc", "WINNT_x86-msvc-x86", "WINNT_x86-msvc-x64"],
+    "win64": ["WINNT_x86_64-msvc", "WINNT_x86_64-msvc-x64"],
+}
+
+# ftp -> shipped locales map
+sl_platform_map = {
+    "linux-i686": "linux",
+    "linux-x86_64": "linux",
+    "mac": "osx",
+    "win32": "win32",
+    "win64": "win64",
+}
+
+# ftp -> info file platform map
+info_file_platform_map = {
+    "linux-i686": "linux",
+    "linux-x86_64": "linux64",
+    "mac": "macosx64",
+    "win32": "win32",
+    "win64": "win64",
+}
+
+def ftp2updatePlatforms(platform):
+    return update_platform_map.get(platform, platform)
+
+def ftp2shippedLocales(platform):
+    return sl_platform_map.get(platform, platform)
+
+def shippedLocales2ftp(platform):
+    matches = []
+    try:
+        [matches.append(
+            k) for k, v in sl_platform_map.iteritems() if v == platform][0]
+        return matches
+    except IndexError:
+        return [platform]
+
+def ftp2infoFile(platform):
+    return info_file_platform_map.get(platform, platform)

python/mozrelease/mozrelease/update_verify.py

@@ -0,0 +1,190 @@
+from __future__ import absolute_import
+
+import os
+import re
+
+from .chunking import getChunk
+
+
+class UpdateVerifyError(Exception):
+    pass
+
+
+class UpdateVerifyConfig(object):
+    comment_regex = re.compile("^#")
+    key_write_order = ("release", "product", "platform", "build_id", "locales",
+                       "channel", "patch_types", "from", "aus_server",
+                       "ftp_server_from", "ftp_server_to", "to",
+                       "mar_channel_IDs", "to_build_id", "to_display_version",
+                       "to_app_version", "updater_package")
+    global_keys = ("product", "channel", "aus_server", "to", "to_build_id",
+                   "to_display_version", "to_app_version")
+    release_keys = ("release", "build_id", "locales", "patch_types", "from",
+                    "ftp_server_from", "ftp_server_to", "mar_channel_IDs",
+                    "platform", "updater_package")
+    first_only_keys = ("from", "aus_server", "to", "to_build_id",
+                       "to_display_version", "to_app_version")
+    compare_attrs = global_keys + ("releases",)
+
+    def __init__(self, product=None, channel=None,
+                 aus_server=None, to=None, to_build_id=None,
+                 to_display_version=None, to_app_version=None):
+        self.product = product
+        self.channel = channel
+        self.aus_server = aus_server
+        self.to = to
+        self.to_build_id = to_build_id
+        self.to_display_version = to_display_version
+        self.to_app_version = to_app_version
+        self.releases = []
+
+    def __eq__(self, other):
+        self_list = [getattr(self, attr) for attr in self.compare_attrs]
+        other_list = [getattr(other, attr) for attr in self.compare_attrs]
+        return self_list == other_list
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def _parseLine(self, line):
+        entry = {}
+        items = re.findall("\w+=[\"'][^\"']*[\"']", line)
+        for i in items:
+            m = re.search(
+                "(?P<key>\w+)=[\"'](?P<value>.+)[\"']", i).groupdict()
+            if m["key"] not in self.global_keys and m["key"] not in self.release_keys:
+                raise UpdateVerifyError(
+                    "Unknown key '%s' found on line:\n%s" % (m["key"], line))
+            if m["key"] in entry:
+                raise UpdateVerifyError("Multiple values found for key '%s' on line:\n%s" % (m["key"], line))
+            entry[m["key"]] = m["value"]
+        if not entry:
+            raise UpdateVerifyError("No parseable data in line '%s'" % line)
+        return entry
+
+    def _addEntry(self, entry, first):
+        releaseKeys = {}
+        for k, v in entry.items():
+            if k in self.global_keys:
+                setattr(self, k, entry[k])
+            elif k in self.release_keys:
+                # "from" is reserved in Python
+                if k == "from":
+                    releaseKeys["from_path"] = v
+                else:
+                    releaseKeys[k] = v
+        self.addRelease(**releaseKeys)
+
+    def read(self, config):
+        f = open(config)
+        # Only the first non-comment line of an update verify config should
+        # have a "from" and"ausServer". Ignore any subsequent lines with them.
+        first = True
+        for line in f.readlines():
+            # Skip comment lines
+            if self.comment_regex.search(line):
+                continue
+            self._addEntry(self._parseLine(line), first)
+            first = False
+
+    def write(self, fh):
+        first = True
+        for releaseInfo in self.releases:
+            for key in self.key_write_order:
+                if key in self.global_keys and (first or key not in self.first_only_keys):
+                    value = getattr(self, key)
+                elif key in self.release_keys:
+                    value = releaseInfo[key]
+                else:
+                    value = None
+                if value is not None:
+                    fh.write(key)
+                    fh.write("=")
+                    if isinstance(value, (list, tuple)):
+                        fh.write('"%s" ' % " ".join(value))
+                    else:
+                        fh.write('"%s" ' % str(value))
+            # Rewind one character to avoid having a trailing space
+            fh.seek(-1, os.SEEK_CUR)
+            fh.write("\n")
+            first = False
+
+    def addRelease(self, release=None, build_id=None, locales=[],
+                   patch_types=['complete'], from_path=None,
+                   ftp_server_from=None, ftp_server_to=None,
+                   mar_channel_IDs=None, platform=None, updater_package=None):
+        """Locales and patch_types can be passed as either a string or a list.
+           If a string is passed, they will be converted to a list for internal
+           storage"""
+        if self.getRelease(build_id, from_path):
+            raise UpdateVerifyError("Couldn't add release identified by build_id '%s' and from_path '%s': already exists in config" % (build_id, from_path))
+        if isinstance(locales, basestring):
+            locales = sorted(list(locales.split()))
+        if isinstance(patch_types, basestring):
+            patch_types = list(patch_types.split())
+        self.releases.append({
+            "release": release,
+            "build_id": build_id,
+            "locales": locales,
+            "patch_types": patch_types,
+            "from": from_path,
+            "ftp_server_from": ftp_server_from,
+            "ftp_server_to": ftp_server_to,
+            "mar_channel_IDs": mar_channel_IDs,
+            "platform": platform,
+            "updater_package": updater_package,
+        })
+
+    def addLocaleToRelease(self, build_id, locale, from_path=None):
+        r = self.getRelease(build_id, from_path)
+        if not r:
+            raise UpdateVerifyError("Couldn't add '%s' to release identified by build_id '%s' and from_path '%s': '%s' doesn't exist in this config." % (locale, build_id, from_path, build_id))
+        r["locales"].append(locale)
+        r["locales"] = sorted(r["locales"])
+
+    def getRelease(self, build_id, from_path):
+        for r in self.releases:
+            if r["build_id"] == build_id and r["from"] == from_path:
+                return r
+        return {}
+
+    def getFullReleaseTests(self):
+        return [r for r in self.releases if r["from"] is not None]
+
+    def getQuickReleaseTests(self):
+        return [r for r in self.releases if r["from"] is None]
+
+    def getChunk(self, chunks, thisChunk):
+        fullTests = []
+        quickTests = []
+        for test in self.getFullReleaseTests():
+            for locale in test["locales"]:
+                fullTests.append([test["build_id"], locale, test["from"]])
+        for test in self.getQuickReleaseTests():
+            for locale in test["locales"]:
+                quickTests.append([test["build_id"], locale, test["from"]])
+        allTests = getChunk(fullTests, chunks, thisChunk)
+        allTests.extend(getChunk(quickTests, chunks, thisChunk))
+
+        newConfig = UpdateVerifyConfig(self.product, self.channel,
+                                       self.aus_server, self.to,
+                                       self.to_build_id,
+                                       self.to_display_version,
+                                       self.to_app_version)
+        for t in allTests:
+            build_id, locale, from_path = t
+            if from_path == "None":
+                from_path = None
+            r = self.getRelease(build_id, from_path)
+            try:
+                newConfig.addRelease(r["release"], build_id, locales=[],
+                                     ftp_server_from=r["ftp_server_from"],
+                                     ftp_server_to=r["ftp_server_to"],
+                                     patch_types=r["patch_types"], from_path=from_path,
+                                     mar_channel_IDs=r["mar_channel_IDs"],
+                                     platform=r["platform"],
+                                     updater_package=r["updater_package"])
+            except UpdateVerifyError:
+                pass
+            newConfig.addLocaleToRelease(build_id, locale, from_path)
+        return newConfig

python/mozrelease/mozrelease/versions.py

@@ -0,0 +1,44 @@
+from __future__ import absolute_import
+
+from distutils.version import StrictVersion
+import re
+
+
+class ModernMozillaVersion(StrictVersion):
+    """A version class that is slightly less restrictive than StrictVersion.
+       Instead of just allowing "a" or "b" as prerelease tags, it allows any
+       alpha. This allows us to support the once-shipped "3.6.3plugin1" and
+       similar versions."""
+    version_re = re.compile(r"""^(\d+) \. (\d+) (\. (\d+))?
+                                ([a-zA-Z]+(\d+))?$""", re.VERBOSE)
+
+
+class AncientMozillaVersion(StrictVersion):
+    """A version class that is slightly less restrictive than StrictVersion.
+       Instead of just allowing "a" or "b" as prerelease tags, it allows any
+       alpha. This allows us to support the once-shipped "3.6.3plugin1" and
+       similar versions.
+       It also supports versions w.x.y.z by transmuting to w.x.z, which
+       is useful for versions like 1.5.0.x and 2.0.0.y"""
+    version_re = re.compile(r"""^(\d+) \. (\d+) \. \d (\. (\d+))
+                                ([a-zA-Z]+(\d+))?$""", re.VERBOSE)
+
+
+def MozillaVersion(version):
+    try:
+        return ModernMozillaVersion(version)
+    except ValueError:
+        pass
+    try:
+        if version.count('.') == 3:
+            return AncientMozillaVersion(version)
+    except ValueError:
+        pass
+    raise ValueError("Version number %s is invalid." % version)
+
+
+def getPrettyVersion(version):
+    version = re.sub(r'a([0-9]+)$', r' Alpha \1', version)
+    version = re.sub(r'b([0-9]+)$', r' Beta \1', version)
+    version = re.sub(r'rc([0-9]+)$', r' RC \1', version)
+    return version

python/mozrelease/setup.py

@@ -0,0 +1,27 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+from setuptools import setup, find_packages
+
+VERSION = '0.1'
+
+setup(
+    author='Mozilla Foundation',
+    author_email='Mozilla Release Engineering',
+    name='mozrelease',
+    description='Common functionality used by Mozilla Release Automation',
+    license='MPL 2.0',
+    packages=find_packages(),
+    version=VERSION,
+    classifiers=[
+        'Development Status :: 3 - Alpha',
+        'Topic :: Software Development :: Build Tools',
+        'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: Implementation :: CPython',
+    ],
+    keywords='mozilla',
+)

taskcluster/ci/release-secondary-update-verify-config/kind.yml

@@ -0,0 +1,125 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.transform:loader
+
+transforms:
+   - taskgraph.transforms.update_verify_config:transforms
+   - taskgraph.transforms.release_notifications:transforms
+   - taskgraph.transforms.job:transforms
+   - taskgraph.transforms.task:transforms
+
+job-defaults:
+   name: secondary-update-verify-config
+   run-on-projects: []  # to make sure this never runs as part of CI
+   shipping-product: firefox
+   shipping-phase: promote
+   worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+   worker:
+      docker-image:
+         in-tree: "update-verify"
+      max-run-time: 3600
+      artifacts:
+         - name: public/build/update-verify.cfg
+           path: /builds/worker/checkouts/gecko/update-verify.cfg
+           type: file
+   run:
+      using: run-task
+      sparse-profile: update-verify
+   extra:
+      app-name: browser
+      product: firefox
+      archive-prefix:
+         by-project:
+            birch: "http://ftp.stage.mozaws.net/pub"
+            default: "https://archive.mozilla.org/pub"
+      previous-archive-prefix:
+         by-project:
+            birch: "https://archive.mozilla.org/pub"
+            default: null
+      aus-server:
+         by-project:
+            birch: "https://aus4.stage.mozaws.net"
+            default: "https://aus5.mozilla.org"
+      include-version:
+         by-project:
+            birch: nonbeta
+            mozilla-release: nonbeta
+            default: null
+      last-watershed:
+         by-project:
+            birch: "57.0"
+            mozilla-release: "57.0"
+            default: null
+      mar-channel-id-override:
+         by-project:
+            birch: beta
+            mozilla-release: beta
+            default: null
+      channel:
+         by-project:
+            birch: "beta-localtest"
+            mozilla-release: "beta-localtest"
+            default: "default"
+
+jobs:
+   firefox-secondary-linux:
+      treeherder:
+         symbol: UVCS
+         platform: linux/opt
+         kind: test
+         tier: 1
+      attributes:
+         build_platform: linux
+      extra:
+         platform: linux-i686
+         updater-platform: linux-x86_64
+
+   firefox-secondary-linux64:
+      treeherder:
+         symbol: UVCS
+         platform: linux64/opt
+         kind: test
+         tier: 1
+      attributes:
+         build_platform: linux64
+      extra:
+         platform: linux-x86_64
+         updater-platform: linux-x86_64
+
+   firefox-secondary-macosx64:
+      treeherder:
+         symbol: UVCS
+         platform: macosx64/opt
+         kind: test
+         tier: 1
+      attributes:
+         build_platform: macosx64
+      extra:
+         platform: mac
+         updater-platform: linux-x86_64
+
+   firefox-secondary-win32:
+      treeherder:
+         symbol: UVCS
+         platform: win32/opt
+         kind: test
+         tier: 1
+      attributes:
+         build_platform: win32
+      extra:
+         platform: win32
+         updater-platform: linux-x86_64
+
+   firefox-secondary-win64:
+      treeherder:
+         symbol: UVCS
+         platform: win64/opt
+         kind: test
+         tier: 1
+      attributes:
+         build_platform: win64
+      extra:
+         platform: win64
+         updater-platform: linux-x86_64

taskcluster/ci/release-secondary-update-verify/kind.yml

@@ -8,6 +8,7 @@ kind-dependencies:
    - post-balrog-dummy
    - post-beetmover-dummy
    - release-updates-builder
+   - release-secondary-update-verify-config
 
 transforms:
    - taskgraph.transforms.release_deps:transforms
@@ -32,67 +33,18 @@ job-defaults:
          NO_BBCONFIG: "1"
          BUILD_TOOLS_REPO:
             by-project:
+               birch: https://hg.mozilla.org/users/bhearsum_mozilla.com/tools
                jamun: https://hg.mozilla.org/users/stage-ffxbld/tools
                maple: https://hg.mozilla.org/users/asasaki_mozilla.com/tools
                default: https://hg.mozilla.org/build/tools
-         CHANNEL:
-            by-project:
-               mozilla-release: "beta-localtest"
-               default: "default"
+         CHANNEL: "beta-localtest"
    extra:
       chunks: 12
-   notifications:
-      completed:
-         subject: "COMPLETED: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task"
-         message: "COMPLETED: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task"
-         plugins:
-            by-project:
-               mozilla-beta: ["log_collect"]
-               mozilla-release: ["log_collect"]
-               default: []
-
-      failed:
-         subject: "FAILED: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task"
-         message: "FAILED: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task"
-         plugins:
-            by-project:
-               mozilla-beta: ["log_collect", "ses"]
-               mozilla-release: ["log_collect", "ses"]
-               default: ["ses"]
-         emails:
-            by-project:
-               mozilla-beta: ["release-automation-notifications@mozilla.com"]
-               mozilla-release: ["release-automation-notifications@mozilla.com"]
-               try: ["{task_def[metadata][owner]}"]
-               maple: ["release+tcstaging@mozilla.com"]
-               default: []
-
-      exception:
-         subject: "EXCEPTION: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task"
-         message: "EXCEPTION: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task"
-         plugins:
-            by-project:
-               mozilla-beta: ["log_collect", "ses"]
-               mozilla-release: ["log_collect", "ses"]
-               default: ["ses"]
-         emails:
-            by-project:
-               mozilla-beta: ["release-automation-notifications@mozilla.com"]
-               mozilla-release: ["release-automation-notifications@mozilla.com"]
-               try: ["{task_def[metadata][owner]}"]
-               maple: ["release+tcstaging@mozilla.com"]
-               default: []
 
 jobs:
    firefox-secondary-linux64:
       description: linux64 secondary channel update verify
       shipping-product: firefox
-      worker:
-         env:
-            VERIFY_CONFIG:
-               by-project:
-                  mozilla-release: "beta-firefox-linux64.cfg"
-                  default: "none"
       treeherder:
          symbol: UVS
          platform: linux64/opt
@@ -104,12 +56,6 @@ jobs:
    firefox-secondary-linux:
       description: linux secondary channel update verify
       shipping-product: firefox
-      worker:
-         env:
-            VERIFY_CONFIG:
-               by-project:
-                  mozilla-release: "beta-firefox-linux.cfg"
-                  default: "none"
       treeherder:
          symbol: UVS
          platform: linux/opt
@@ -121,12 +67,6 @@ jobs:
    firefox-secondary-win64:
       description: win64 secondary channel update verify
       shipping-product: firefox
-      worker:
-         env:
-            VERIFY_CONFIG:
-               by-project:
-                  mozilla-release: "beta-firefox-win64.cfg"
-                  default: "none"
       treeherder:
          symbol: UVS
          platform: win64/opt
@@ -138,12 +78,6 @@ jobs:
    firefox-secondary-win32:
       description: win32 secondary channel update verify
       shipping-product: firefox
-      worker:
-         env:
-            VERIFY_CONFIG:
-               by-project:
-                  mozilla-release: "beta-firefox-win32.cfg"
-                  default: "none"
       treeherder:
          symbol: UVS
          platform: win32/opt
@@ -155,12 +89,6 @@ jobs:
    firefox-secondary-macosx64:
       description: macosx64 secondary channel update verify
       shipping-product: firefox
-      worker:
-         env:
-            VERIFY_CONFIG:
-               by-project:
-                  mozilla-release: "beta-firefox-macosx64.cfg"
-                  default: "none"
       treeherder:
          symbol: UVS
          platform: macosx64/opt

taskcluster/ci/release-update-verify-config/kind.yml

@@ -0,0 +1,293 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.transform:loader
+
+transforms:
+   - taskgraph.transforms.update_verify_config:transforms
+   - taskgraph.transforms.release_notifications:transforms
+   - taskgraph.transforms.job:transforms
+   - taskgraph.transforms.task:transforms
+
+job-defaults:
+   name: update-verify-config
+   run-on-projects: []  # to make sure this never runs as part of CI
+   shipping-phase: promote
+   worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+   worker:
+      docker-image:
+         in-tree: "update-verify"
+      max-run-time: 3600
+      artifacts:
+         - name: public/build/update-verify.cfg
+           path: /builds/worker/checkouts/gecko/update-verify.cfg
+           type: file
+   run:
+      using: run-task
+      sparse-profile: update-verify
+   extra:
+      app-name: browser
+      archive-prefix:
+         by-project:
+            birch: "http://ftp.stage.mozaws.net/pub"
+            jamun: "http://ftp.stage.mozaws.net/pub"
+            maple: "http://ftp.stage.mozaws.net/pub"
+            default: "https://archive.mozilla.org/pub"
+      previous-archive-prefix:
+         by-project:
+            birch: "https://archive.mozilla.org/pub"
+            jamun: "https://archive.mozilla.org/pub"
+            maple: "https://archive.mozilla.org/pub"
+            default: null
+      aus-server:
+         by-project:
+            birch: "https://aus4.stage.mozaws.net"
+            jamun: "https://aus4.stage.mozaws.net"
+            maple: "https://aus4.stage.mozaws.net"
+            default: "https://aus5.mozilla.org"
+      # This is overridden for devedition to exclude 58.0b1
+      # because of the special case added by
+      # https://bugzilla.mozilla.org/show_bug.cgi?id=1419189
+      # The devedition override can be removed after 58.0b1
+      # is behind a watershed
+      include-version:
+         by-project:
+            birch: nonbeta
+            jamun: beta
+            maple: beta
+            mozilla-beta: beta
+            mozilla-release: nonbeta
+            default: null
+      last-watershed:
+         by-project:
+            # TODO: add esr here when setting up mozilla-esr60
+            # let's put mozilla-esr52 in this comment as well, in case
+            # somebody is grepping the tree for things they need to do.
+            birch: "57.0"
+            jamun: "56.0b10"
+            maple: "56.0b10"
+            mozilla-beta: "56.0b10"
+            mozilla-release: "57.0"
+            default: null
+
+jobs:
+   firefox-linux:
+      shipping-product: firefox
+      treeherder:
+         symbol: UVC
+         platform: linux/opt
+         kind: test
+         tier: 1
+      attributes:
+         build_platform: linux
+      extra:
+         product: firefox
+         platform: linux-i686
+         updater-platform: linux-x86_64
+         channel:
+            by-project:
+               birch: "release-localtest"
+               jamun: "beta-localtest"
+               maple: "beta-localtest"
+               mozilla-beta: "beta-localtest"
+               mozilla-release: "release-localtest"
+               mozilla-esr52: "esr-localtest"
+               default: "default"
+         mar-channel-id-override:
+            by-project:
+               maple: beta
+               mozilla-beta: beta
+               default: null
+
+   firefox-linux64:
+      shipping-product: firefox
+      treeherder:
+         symbol: UVC
+         platform: linux64/opt
+         kind: test
+         tier: 1
+      attributes:
+         build_platform: linux64
+      extra:
+         product: firefox
+         platform: linux-x86_64
+         updater-platform: linux-x86_64
+         channel:
+            by-project:
+               birch: "release-localtest"
+               jamun: "beta-localtest"
+               maple: "beta-localtest"
+               mozilla-beta: "beta-localtest"
+               mozilla-release: "release-localtest"
+               mozilla-esr52: "esr-localtest"
+               default: "default"
+         mar-channel-id-override:
+            by-project:
+               maple: beta
+               mozilla-beta: beta
+               default: null
+
+   firefox-macosx64:
+      shipping-product: firefox
+      treeherder:
+         symbol: UVC
+         platform: macosx64/opt
+         kind: test
+         tier: 1
+      attributes:
+         build_platform: macosx64
+      extra:
+         product: firefox
+         platform: mac
+         updater-platform: linux-x86_64
+         channel:
+            by-project:
+               birch: "release-localtest"
+               jamun: "beta-localtest"
+               maple: "beta-localtest"
+               mozilla-beta: "beta-localtest"
+               mozilla-release: "release-localtest"
+               mozilla-esr52: "esr-localtest"
+               default: "default"
+         mar-channel-id-override:
+            by-project:
+               maple: beta
+               mozilla-beta: beta
+               default: null
+
+   firefox-win32:
+      shipping-product: firefox
+      treeherder:
+         symbol: UVC
+         platform: win32/opt
+         kind: test
+         tier: 1
+      attributes:
+         build_platform: win32
+      extra:
+         product: firefox
+         platform: win32
+         updater-platform: linux-x86_64
+         channel:
+            by-project:
+               birch: "release-localtest"
+               jamun: "beta-localtest"
+               maple: "beta-localtest"
+               mozilla-beta: "beta-localtest"
+               mozilla-release: "release-localtest"
+               mozilla-esr52: "esr-localtest"
+               default: "default"
+         mar-channel-id-override:
+            by-project:
+               maple: beta
+               mozilla-beta: beta
+               default: null
+
+   firefox-win64:
+      shipping-product: firefox
+      treeherder:
+         symbol: UVC
+         platform: win64/opt
+         kind: test
+         tier: 1
+      attributes:
+         build_platform: win64
+      extra:
+         product: firefox
+         platform: win64
+         updater-platform: linux-x86_64
+         channel:
+            by-project:
+               birch: "release-localtest"
+               jamun: "beta-localtest"
+               maple: "beta-localtest"
+               mozilla-beta: "beta-localtest"
+               mozilla-release: "release-localtest"
+               mozilla-esr52: "esr-localtest"
+               default: "default"
+         mar-channel-id-override:
+            by-project:
+               maple: beta
+               mozilla-beta: beta
+               default: null
+
+   devedition-linux:
+      shipping-product: devedition
+      treeherder:
+         symbol: UVC
+         platform: linux-devedition/opt
+         kind: test
+         tier: 1
+      attributes:
+         build_platform: linux-devedition
+      extra:
+         product: firefox
+         platform: linux-i686
+         updater-platform: linux-x86_64
+         channel: "aurora-localtest"
+         include-version: devedition_hack
+
+   devedition-linux64:
+      shipping-product: devedition
+      treeherder:
+         symbol: UVC
+         platform: linux64-devedition/opt
+         kind: test
+         tier: 1
+      attributes:
+         build_platform: linux64-devedition
+      extra:
+         product: firefox
+         platform: linux-x86_64
+         updater-platform: linux-x86_64
+         channel: "aurora-localtest"
+         include-version: devedition_hack
+
+   devedition-macosx64:
+      shipping-product: devedition
+      treeherder:
+         symbol: UVC
+         platform: macosx64-devedition/opt
+         kind: test
+         tier: 1
+      attributes:
+         build_platform: macosx64-devedition
+      extra:
+         product: firefox
+         platform: mac
+         updater-platform: linux-x86_64
+         channel: "aurora-localtest"
+         include-version: devedition_hack
+
+   devedition-win32:
+      shipping-product: devedition
+      treeherder:
+         symbol: UVC
+         platform: win32-devedition/opt
+         kind: test
+         tier: 1
+      attributes:
+         build_platform: win32-devedition
+      extra:
+         product: firefox
+         platform: win32
+         updater-platform: linux-x86_64
+         channel: "aurora-localtest"
+         include-version: devedition_hack
+
+   devedition-win64:
+      shipping-product: devedition
+      treeherder:
+         symbol: UVC
+         platform: win64-devedition/opt
+         kind: test
+         tier: 1
+      attributes:
+         build_platform: win64-devedition
+      extra:
+         product: firefox
+         platform: win64
+         updater-platform: linux-x86_64
+         channel: "aurora-localtest"
+         include-version: devedition_hack

taskcluster/ci/release-update-verify/kind.yml

@@ -8,6 +8,7 @@ kind-dependencies:
    - post-balrog-dummy
    - post-beetmover-dummy
    - release-updates-builder
+   - release-update-verify-config
 
 transforms:
    - taskgraph.transforms.release_deps:transforms
@@ -32,6 +33,7 @@ job-defaults:
          NO_BBCONFIG: "1"
          BUILD_TOOLS_REPO:
             by-project:
+               birch: https://hg.mozilla.org/users/bhearsum_mozilla.com/tools
                jamun: https://hg.mozilla.org/users/stage-ffxbld/tools
                maple: https://hg.mozilla.org/users/asasaki_mozilla.com/tools
                default: https://hg.mozilla.org/build/tools
@@ -46,20 +48,12 @@ jobs:
          env:
             CHANNEL:
                by-project:
+                  birch: "release-localtest"
                   jamun: "beta-localtest"
                   maple: "beta-localtest"
                   mozilla-beta: "beta-localtest"
                   mozilla-release: "release-localtest"
-                  mozilla-esr52: "esr-localtest"
                   default: "default"
-            VERIFY_CONFIG:
-               by-project:
-                  jamun: "beta-firefox-linux64.cfg"
-                  maple: "beta-firefox-linux64.cfg"
-                  mozilla-beta: "beta-firefox-linux64.cfg"
-                  mozilla-release: "release-firefox-linux64.cfg"
-                  mozilla-esr52: "esr-firefox-linux64.cfg"
-                  default: "none"
       treeherder:
          symbol: UV
          platform: linux64/opt
@@ -75,20 +69,12 @@ jobs:
          env:
             CHANNEL:
                by-project:
+                  birch: "release-localtest"
                   jamun: "beta-localtest"
                   maple: "beta-localtest"
                   mozilla-beta: "beta-localtest"
                   mozilla-release: "release-localtest"
-                  mozilla-esr52: "esr-localtest"
                   default: "default"
-            VERIFY_CONFIG:
-               by-project:
-                  jamun: "beta-firefox-linux.cfg"
-                  maple: "beta-firefox-linux.cfg"
-                  mozilla-beta: "beta-firefox-linux.cfg"
-                  mozilla-release: "release-firefox-linux.cfg"
-                  mozilla-esr52: "esr-firefox-linux.cfg"
-                  default: "none"
       treeherder:
          symbol: UV
          platform: linux/opt
@@ -104,20 +90,12 @@ jobs:
          env:
             CHANNEL:
                by-project:
+                  birch: "release-localtest"
                   jamun: "beta-localtest"
                   maple: "beta-localtest"
                   mozilla-beta: "beta-localtest"
                   mozilla-release: "release-localtest"
-                  mozilla-esr52: "esr-localtest"
                   default: "default"
-            VERIFY_CONFIG:
-               by-project:
-                  jamun: "beta-firefox-win64.cfg"
-                  maple: "beta-firefox-win64.cfg"
-                  mozilla-beta: "beta-firefox-win64.cfg"
-                  mozilla-release: "release-firefox-win64.cfg"
-                  mozilla-esr52: "esr-firefox-win64.cfg"
-                  default: "none"
       treeherder:
          symbol: UV
          platform: win64/opt
@@ -133,20 +111,12 @@ jobs:
          env:
             CHANNEL:
                by-project:
+                  birch: "release-localtest"
                   jamun: "beta-localtest"
                   maple: "beta-localtest"
                   mozilla-beta: "beta-localtest"
                   mozilla-release: "release-localtest"
-                  mozilla-esr52: "esr-localtest"
                   default: "default"
-            VERIFY_CONFIG:
-               by-project:
-                  jamun: "beta-firefox-win32.cfg"
-                  maple: "beta-firefox-win32.cfg"
-                  mozilla-beta: "beta-firefox-win32.cfg"
-                  mozilla-release: "release-firefox-win32.cfg"
-                  mozilla-esr52: "esr-firefox-win32.cfg"
-                  default: "none"
       treeherder:
          symbol: UV
          platform: win32/opt
@@ -162,20 +132,12 @@ jobs:
          env:
             CHANNEL:
                by-project:
+                  birch: "release-localtest"
                   jamun: "beta-localtest"
                   maple: "beta-localtest"
                   mozilla-beta: "beta-localtest"
                   mozilla-release: "release-localtest"
-                  mozilla-esr52: "esr-localtest"
                   default: "default"
-            VERIFY_CONFIG:
-               by-project:
-                  jamun: "beta-firefox-macosx64.cfg"
-                  maple: "beta-firefox-macosx64.cfg"
-                  mozilla-beta: "beta-firefox-macosx64.cfg"
-                  mozilla-release: "release-firefox-macosx64.cfg"
-                  mozilla-esr52: "esr-firefox-macosx64.cfg"
-                  default: "none"
       treeherder:
          symbol: UV
          platform: macosx64/opt
@@ -189,22 +151,7 @@ jobs:
       shipping-product: devedition
       worker:
          env:
-            CHANNEL:
-               by-project:
-                  jamun: "aurora-localtest"
-                  maple: "aurora-localtest"
-                  mozilla-beta: "aurora-localtest"
-                  mozilla-release: "release-localtest"
-                  mozilla-esr52: "esr-localtest"
-                  default: "default"
-            VERIFY_CONFIG:
-               by-project:
-                  jamun: "aurora-devedition-linux64.cfg"
-                  maple: "aurora-devedition-linux64.cfg"
-                  mozilla-beta: "aurora-devedition-linux64.cfg"
-                  mozilla-release: "release-devedition-linux64.cfg"
-                  mozilla-esr52: "esr-devedition-linux64.cfg"
-                  default: "none"
+            CHANNEL: "aurora-localtest"
       treeherder:
          symbol: UV
          platform: linux64-devedition/opt
@@ -218,22 +165,7 @@ jobs:
       shipping-product: devedition
       worker:
          env:
-            CHANNEL:
-               by-project:
-                  jamun: "aurora-localtest"
-                  maple: "aurora-localtest"
-                  mozilla-beta: "aurora-localtest"
-                  mozilla-release: "release-localtest"
-                  mozilla-esr52: "esr-localtest"
-                  default: "default"
-            VERIFY_CONFIG:
-               by-project:
-                  jamun: "aurora-devedition-linux.cfg"
-                  maple: "aurora-devedition-linux.cfg"
-                  mozilla-beta: "aurora-devedition-linux.cfg"
-                  mozilla-release: "release-devedition-linux.cfg"
-                  mozilla-esr52: "esr-devedition-linux.cfg"
-                  default: "none"
+            CHANNEL: "aurora-localtest"
       treeherder:
          symbol: UV
          platform: linux-devedition/opt
@@ -247,22 +179,7 @@ jobs:
       shipping-product: devedition
       worker:
          env:
-            CHANNEL:
-               by-project:
-                  jamun: "aurora-localtest"
-                  maple: "aurora-localtest"
-                  mozilla-beta: "aurora-localtest"
-                  mozilla-release: "release-localtest"
-                  mozilla-esr52: "esr-localtest"
-                  default: "default"
-            VERIFY_CONFIG:
-               by-project:
-                  jamun: "aurora-devedition-win64.cfg"
-                  maple: "aurora-devedition-win64.cfg"
-                  mozilla-beta: "aurora-devedition-win64.cfg"
-                  mozilla-release: "release-devedition-win64.cfg"
-                  mozilla-esr52: "esr-devedition-win64.cfg"
-                  default: "none"
+            CHANNEL: "aurora-localtest"
       treeherder:
          symbol: UV
          platform: win64-devedition/opt
@@ -276,22 +193,7 @@ jobs:
       shipping-product: devedition
       worker:
          env:
-            CHANNEL:
-               by-project:
-                  jamun: "aurora-localtest"
-                  maple: "aurora-localtest"
-                  mozilla-beta: "aurora-localtest"
-                  mozilla-release: "release-localtest"
-                  mozilla-esr52: "esr-localtest"
-                  default: "default"
-            VERIFY_CONFIG:
-               by-project:
-                  jamun: "aurora-devedition-win32.cfg"
-                  maple: "aurora-devedition-win32.cfg"
-                  mozilla-beta: "aurora-devedition-win32.cfg"
-                  mozilla-release: "release-devedition-win32.cfg"
-                  mozilla-esr52: "esr-devedition-win32.cfg"
-                  default: "none"
+            CHANNEL: "aurora-localtest"
       treeherder:
          symbol: UV
          platform: win32-devedition/opt
@@ -305,22 +207,7 @@ jobs:
       shipping-product: devedition
       worker:
          env:
-            CHANNEL:
-               by-project:
-                  jamun: "aurora-localtest"
-                  maple: "aurora-localtest"
-                  mozilla-beta: "aurora-localtest"
-                  mozilla-release: "release-localtest"
-                  mozilla-esr52: "esr-localtest"
-                  default: "default"
-            VERIFY_CONFIG:
-               by-project:
-                  jamun: "aurora-devedition-macosx64.cfg"
-                  maple: "aurora-devedition-macosx64.cfg"
-                  mozilla-beta: "aurora-devedition-macosx64.cfg"
-                  mozilla-release: "release-devedition-macosx64.cfg"
-                  mozilla-esr52: "esr-devedition-macosx64.cfg"
-                  default: "none"
+            CHANNEL: "aurora-localtest"
       treeherder:
          symbol: UV
          platform: macosx64-devedition/opt

taskcluster/docs/kinds.rst

@@ -303,6 +303,14 @@ release-secondary-update-verify
 ---------------------
 Verifies the contents and package of release update MARs.
 
+release-update-verify-config
+----------------------------
+Creates configs for release-update-verify tasks
+
+release-secondary-update-verify-config
+--------------------------------------
+Creates configs for release-secondary-update-verify tasks
+
 release-updates-builder
 -----------------------
 Top level Balrog blob submission & patcher/update verify config updates.

taskcluster/taskgraph/target_tasks.py

@@ -334,6 +334,7 @@ def target_tasks_promote_firefox(full_task_graph, parameters, graph_config):
         # RCs
         if parameters.get('release_type') != 'rc':
             if task.kind in ('release-secondary-update-verify',
+                             'release-secondary-update-verify-config',
                              'release-secondary-final-verify'):
                 return False
 

taskcluster/taskgraph/transforms/update_verify.py

@@ -11,7 +11,7 @@ from copy import deepcopy
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.schema import resolve_keyed_by
-from taskgraph.util.scriptworker import get_release_config
+from taskgraph.util.taskcluster import get_taskcluster_artifact_prefix
 
 transforms = TransformSequence()
 
@@ -20,11 +20,6 @@ transforms = TransformSequence()
 def add_command(config, tasks):
     for task in tasks:
         total_chunks = task["extra"]["chunks"]
-        release_config = get_release_config(config)
-        release_tag = "{}_{}_RELEASE_RUNTIME".format(
-            task["shipping-product"].upper(),
-            release_config["version"].replace(".", "_")
-        )
 
         for this_chunk in range(1, total_chunks+1):
             chunked = deepcopy(task)
@@ -37,10 +32,7 @@ def add_command(config, tasks):
             chunked["worker"]["command"] = [
                 "/bin/bash",
                 "-c",
-                "hg clone $BUILD_TOOLS_REPO tools && cd tools && " +
-                "hg up -r {} && cd .. && ".format(
-                    release_tag,
-                ) +
+                "hg clone $BUILD_TOOLS_REPO tools && " +
                 "tools/scripts/release/updates/chunked-verify.sh " +
                 "UNUSED UNUSED {} {}".format(
                     total_chunks,
@@ -50,4 +42,18 @@ def add_command(config, tasks):
             for thing in ("CHANNEL", "VERIFY_CONFIG", "BUILD_TOOLS_REPO"):
                 thing = "worker.env.{}".format(thing)
                 resolve_keyed_by(chunked, thing, thing, **config.params)
+
+            update_verify_config = None
+            for upstream in chunked.get("dependencies", {}).keys():
+                if 'update-verify-config' in upstream:
+                    update_verify_config = "{}update-verify.cfg".format(
+                        get_taskcluster_artifact_prefix("<{}>".format(upstream))
+                    )
+            if not update_verify_config:
+                raise Exception("Couldn't find upate verify config")
+
+            chunked["worker"]["env"]["TASKCLUSTER_VERIFY_CONFIG"] = {
+                "task-reference": update_verify_config
+            }
+
             yield chunked

taskcluster/taskgraph/transforms/update_verify_config.py

@@ -0,0 +1,103 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the beetmover task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import urlparse
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import resolve_keyed_by
+from taskgraph.util.scriptworker import get_release_config
+
+transforms = TransformSequence()
+
+
+# The beta regexes do not match point releases.
+# In the rare event that we do ship a point
+# release to beta, we need to either:
+# 1) update these regexes to match that specific version
+# 2) pass a second include version that matches that specifivc version
+INCLUDE_VERSION_REGEXES = {
+    "beta": r"'^(\d+\.\d+(b\d+)?)$'",
+    "nonbeta": r"'^\d+\.\d+(\.\d+)?$'",
+    # Same as beta, except excludes 58.0b1 due to issues with it not being able
+    # to update to latest
+    "devedition_hack": r"'^((?!58\.0b1$)\d+\.\d+(b\d+)?)$'",
+}
+
+MAR_CHANNEL_ID_OVERRIDE_REGEXES = {
+    "beta": r"'^\d+\.\d+(\.\d+)?$$,firefox-mozilla-beta,firefox-mozilla-release'",
+}
+
+
+@transforms.add
+def add_command(config, tasks):
+    keyed_by_args = [
+        "channel",
+        "archive-prefix",
+        "previous-archive-prefix",
+        "aus-server",
+        "include-version",
+        "mar-channel-id-override",
+        "last-watershed",
+    ]
+    optional_args = [
+        "updater-platform",
+    ]
+
+    for task in tasks:
+        release_config = get_release_config(config)
+        task["description"] = "generate update verify config for {}".format(
+            task["attributes"]["build_platform"]
+        )
+
+        command = [
+            "cd", "/builds/worker/checkouts/gecko", "&&"
+            "./mach", "python",
+            "testing/mozharness/scripts/release/update-verify-config-creator.py",
+            "--config", "internal_pypi.py",
+            "--product", task["extra"]["product"],
+            "--stage-product", task["shipping-product"],
+            "--app-name", task["extra"]["app-name"],
+            "--platform", task["extra"]["platform"],
+            "--to-version", release_config["version"],
+            "--to-app-version", release_config["appVersion"],
+            "--to-build-number", str(release_config["build_number"]),
+            "--to-buildid", config.params["moz_build_date"],
+            "--to-revision", config.params["head_rev"],
+            "--output-file", "update-verify.cfg",
+        ]
+
+        repo_path = urlparse.urlsplit(config.params["head_repository"]).path.lstrip("/")
+        command.extend(["--repo-path", repo_path])
+
+        if release_config.get("partial_versions"):
+            for partial in release_config["partial_versions"].split(","):
+                command.extend(["--partial-version", partial.split("build")[0]])
+
+        for arg in optional_args:
+            if task["extra"].get(arg):
+                command.append("--{}".format(arg))
+                command.append(task["extra"][arg])
+
+        for arg in keyed_by_args:
+            thing = "extra.{}".format(arg)
+            resolve_keyed_by(task, thing, thing, **config.params)
+            # ignore things that resolved to null
+            if not task["extra"].get(arg):
+                continue
+            if arg == "include-version":
+                task["extra"][arg] = INCLUDE_VERSION_REGEXES[task["extra"][arg]]
+            if arg == "mar-channel-id-override":
+                task["extra"][arg] = MAR_CHANNEL_ID_OVERRIDE_REGEXES[task["extra"][arg]]
+
+            command.append("--{}".format(arg))
+            command.append(task["extra"][arg])
+
+        task["run"]["command"] = " ".join(command)
+
+        yield task

taskcluster/taskgraph/util/scriptworker.py

@@ -457,6 +457,8 @@ def get_release_config(config):
     if partial_updates != "" and config.kind in ('release-bouncer-sub',
                                                  'release-bouncer-check',
                                                  'release-updates-builder',
+                                                 'release-update-verify-config',
+                                                 'release-secondary-update-verify-config',
                                                  ):
         partial_updates = json.loads(partial_updates)
         release_config['partial_versions'] = ', '.join([

testing/mozharness/configs/internal_pypi.py

@@ -0,0 +1,7 @@
+config = {
+    "find_links": [
+        "http://pypi.pvt.build.mozilla.org/pub",
+        "http://pypi.pub.build.mozilla.org/pub",
+    ],
+    'pip_index': False,
+}

testing/mozharness/scripts/release/update-verify-config-creator.py

@@ -0,0 +1,452 @@
+from distutils.version import LooseVersion
+import json
+import math
+import os
+import pprint
+import re
+import sys
+from urlparse import urljoin
+
+
+sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
+
+from mozharness.base.log import DEBUG, INFO, FATAL
+from mozharness.base.script import BaseScript
+from mozharness.base.python import VirtualenvMixin
+
+
+def is_triangualar(x):
+    """Check if a number is triangular (0, 1, 3, 6, 10, 15, ...)
+    see: https://en.wikipedia.org/wiki/Triangular_number#Triangular_roots_and_tests_for_triangular_numbers # noqa
+
+    >>> is_triangualar(0)
+    True
+    >>> is_triangualar(1)
+    True
+    >>> is_triangualar(2)
+    False
+    >>> is_triangualar(3)
+    True
+    >>> is_triangualar(4)
+    False
+    >>> all(is_triangualar(x) for x in [0, 1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 66, 78, 91, 105])
+    True
+    >>> all(not is_triangualar(x) for x in [4, 5, 8, 9, 11, 17, 25, 29, 39, 44, 59, 61, 72, 98, 112])
+    True
+    """
+    n = (math.sqrt(8*x + 1) - 1)/2
+    return n == int(n)
+
+
+class UpdateVerifyConfigCreator(BaseScript, VirtualenvMixin):
+    config_options = [
+        [["--product"], {
+            "dest": "product",
+            "help": "Product being tested, as used in the update URL and filenames. Eg: firefox",
+        }],
+        [["--stage-product"], {
+            "dest": "stage_product",
+            "help": "Product being tested, as used in stage directories and ship it"
+                    "If not passed this is assumed to be the same as product."
+        }],
+        [["--app-name"], {
+            "dest": "app_name",
+            "help": "App name being tested. Eg: browser",
+        }],
+        [["--channel"], {
+            "dest": "channel",
+            "help": "Channel to run update verify against",
+        }],
+        [["--aus-server"], {
+            "dest": "aus_server",
+            "default": "https://aus5.mozilla.org",
+            "help": "AUS server to run update verify against",
+        }],
+        [["--to-version"], {
+            "dest": "to_version",
+            "help": "The version of the release being updated to. Eg: 59.0b5",
+        }],
+        [["--to-app-version"], {
+            "dest": "to_app_version",
+            "help": "The in-app version of the release being updated to. Eg: 59.0",
+        }],
+        [["--to-display-version"], {
+            "dest": "to_display_version",
+            "help": "The human-readable version of the release being updated to. Eg: 59.0 Beta 9",
+        }],
+        [["--to-build-number"], {
+            "dest": "to_build_number",
+            "help": "The build number of the release being updated to",
+        }],
+        [["--to-buildid"], {
+            "dest": "to_buildid",
+            "help": "The buildid of the release being updated to",
+        }],
+        [["--to-revision"], {
+            "dest": "to_revision",
+            "help": "The revision that the release being updated to was built against",
+        }],
+        [["--partial-version"], {
+            "dest": "partial_versions",
+            "action": "append",
+            "help": "A previous release version that is expected to receive a partial update. "
+                    "Eg: 59.0b4. May be specified multiple times."
+        }],
+        [["--last-watershed"], {
+            "dest": "last_watershed",
+            "help": "The earliest version to include in the update verify config. Eg: 57.0b10",
+        }],
+        [["--include-version"], {
+            "dest": "include_versions",
+            "default": [],
+            "action": "append",
+            "help": "Only include versions that match one of these regexes. "
+                    "May be passed multiple times",
+        }],
+        [["--mar-channel-id-override"], {
+            "dest": "mar_channel_id_options",
+            "default": [],
+            "action": "append",
+            "help": "A version regex and channel id string to override those versions with."
+                    "Eg: ^\\d+\\.\\d+(\\.\\d+)?$,firefox-mozilla-beta,firefox-mozilla-release "
+                    "will set accepted mar channel ids to 'firefox-mozilla-beta' and "
+                    "'firefox-mozilla-release for x.y and x.y.z versions. "
+                    "May be passed multiple times"
+        }],
+        [["--platform"], {
+            "dest": "platform",
+            "help": "The platform to generate the update verify config for, in FTP-style",
+        }],
+        [["--updater-platform"], {
+            "dest": "updater_platform",
+            "help": "The platform to run the updater on, in FTP-style."
+                    "If not specified, this is assumed to be the same as platform",
+        }],
+        [["--archive-prefix"], {
+            "dest": "archive_prefix",
+            "help": "The server/path to pull the current release from. "
+                    "Eg: https://archive.mozilla.org/pub",
+        }],
+        [["--previous-archive-prefix"], {
+            "dest": "previous_archive_prefix",
+            "help": "The server/path to pull the previous releases from"
+                    "If not specified, this is assumed to be the same as --archive-prefix"
+        }],
+        [["--repo-path"], {
+            "dest": "repo_path",
+            "help": "The repository (relative to the hg server root) that the current release was "
+                    "built from Eg: releases/mozilla-beta"
+        }],
+        [["--output-file"], {
+            "dest": "output_file",
+            "help": "Where to write the update verify config to",
+        }],
+        [["--product-details-server"], {
+            "dest": "product_details_server",
+            "default": "https://product-details.mozilla.org",
+            "help": "Product Details server to pull previous release info from. "
+                    "Using anything other than the production server is likely to "
+                    "cause issues with update verify."
+        }],
+        [["--hg-server"], {
+            "dest": "hg_server",
+            "default": "https://hg.mozilla.org",
+            "help": "Mercurial server to pull various previous and current version info from",
+        }],
+        [["--full-check-locale"], {
+            "dest": "full_check_locales",
+            "default": ["de", "en-US", "ru"],
+            "action": "append",
+            "help": "A list of locales to generate full update verify checks for",
+        }],
+    ]
+
+    def __init__(self):
+        BaseScript.__init__(
+            self,
+            config_options=self.config_options,
+            config={
+                "virtualenv_modules": [
+                    "mozrelease",
+                ],
+                "virtualenv_path": "venv",
+            },
+            all_actions=[
+                "create-virtualenv",
+                "activate-virtualenv",
+                "gather-info",
+                "create-config",
+                "write-config",
+            ],
+            default_actions=[
+                "create-virtualenv",
+                "activate-virtualenv",
+                "gather-info",
+                "create-config",
+                "write-config",
+            ],
+        )
+
+    def _pre_config_lock(self, rw_config):
+        super(UpdateVerifyConfigCreator, self)._pre_config_lock(rw_config)
+
+        if "updater_platform" not in self.config:
+            self.config["updater_platform"] = self.config["platform"]
+        if "stage_product" not in self.config:
+            self.config["stage_product"] = self.config["product"]
+        if "previous_archive_prefix" not in self.config:
+            self.config["previous_archive_prefix"] = self.config["archive_prefix"]
+        self.config["archive_prefix"].rstrip("/")
+        self.config["previous_archive_prefix"].rstrip("/")
+        self.config["mar_channel_id_overrides"] = {}
+        for override in self.config["mar_channel_id_options"]:
+            pattern, override_str = override.split(",", 1)
+            self.config["mar_channel_id_overrides"][pattern] = override_str
+
+    def _get_update_paths(self):
+        from mozrelease.l10n import getPlatformLocales
+        from mozrelease.paths import getCandidatesDir
+        from mozrelease.platforms import ftp2infoFile
+        from mozrelease.versions import MozillaVersion
+
+        self.update_paths = {}
+
+        ret = self._retry_download(
+            "{}/1.0/{}.json".format(
+                self.config["product_details_server"],
+                self.config["stage_product"],
+            ),
+            "WARNING",
+        )
+        releases = json.load(ret)["releases"]
+        for release_name, release_info in reversed(sorted(releases.items())):
+            product, version = release_name.split("-", 1)
+            version = version.rstrip("esr")
+            tag = "{}_{}_RELEASE".format(product.upper(), version.replace(".", "_"))
+            # Product details has a "category" for releases that we can use to
+            # determine the repo path. This will fail if any previous releases
+            # were built from a project branch - but that's not something we do
+            # at the time of writing.
+            branch = None
+            if release_info["category"] == "dev":
+                branch = "releases/mozilla-beta"
+            elif release_info["category"] == "esr":
+                branch = "releases/mozilla-esr{}".format(version[:2])
+            elif release_info["category"] in ("major", "stability"):
+                branch = "releases/mozilla-release"
+            if not branch:
+                raise Exception("Cannot determine branch, cannot continue!")
+
+            # Exclude any releases that don't match one of our include version
+            # regexes. This is generally to avoid including versions from other
+            # channels. Eg: including betas when testing releases
+            for v in self.config["include_versions"]:
+                if re.match(v, version):
+                    break
+            else:
+                self.log("Skipping release whose version doesn't match any "
+                         "include_version pattern: %s" % release_name,
+                         level=INFO)
+                continue
+
+            # We also have to trim out previous releases that aren't in the same
+            # product line, too old, etc.
+            if self.config["stage_product"] != product:
+                self.log("Skipping release that doesn't match product name: %s" % release_name,
+                         level=INFO)
+                continue
+            if MozillaVersion(version) < MozillaVersion(self.config["last_watershed"]):
+                self.log("Skipping release that's behind the last watershed: %s" % release_name,
+                         level=INFO)
+                continue
+            if version == self.config["to_version"]:
+                self.log("Skipping release that is the same as to version: %s" % release_name,
+                         level=INFO)
+                continue
+            if MozillaVersion(version) > MozillaVersion(self.config["to_version"]):
+                self.log("Skipping release that's newer than to version: %s" % release_name,
+                         level=INFO)
+                continue
+
+            if version in self.update_paths:
+                raise Exception("Found duplicate release for version: %s", version)
+
+            # This is a crappy place to get buildids from, but we don't have a better one.
+            # This will start to fail if old info files are deleted.
+            info_file_url = "{}{}/{}_info.txt".format(
+                self.config["previous_archive_prefix"],
+                getCandidatesDir(
+                    self.config["stage_product"],
+                    version,
+                    release_info["build_number"],
+                ),
+                ftp2infoFile(self.config["platform"])
+            )
+            self.log("Retrieving buildid from info file: %s" % info_file_url, level=DEBUG)
+            ret = self._retry_download(info_file_url, "WARNING")
+            buildID = ret.read().split("=")[1].strip()
+
+            shipped_locales_url = urljoin(
+                self.config["hg_server"],
+                "{}/raw-file/{}/{}/locales/shipped-locales".format(
+                    branch,
+                    tag,
+                    self.config["app_name"],
+                ),
+            )
+            ret = self._retry_download(shipped_locales_url, "WARNING")
+            shipped_locales = ret.read().strip()
+
+            app_version_url = urljoin(
+                self.config["hg_server"],
+                "{}/raw-file/{}/{}/config/version.txt".format(
+                    branch,
+                    tag,
+                    self.config["app_name"],
+                ),
+            )
+            app_version = self._retry_download(app_version_url, "WARNING").read().strip()
+
+            self.log("Adding {} to update paths".format(version), level=INFO)
+            self.update_paths[version] = {
+                "appVersion": app_version,
+                "locales": getPlatformLocales(shipped_locales, self.config["platform"]),
+                "buildID": buildID,
+            }
+            for pattern, mar_channel_ids in self.config["mar_channel_id_overrides"].items():
+                if re.match(pattern, version):
+                    self.update_paths[version]["marChannelIds"] = mar_channel_ids
+
+    def gather_info(self):
+        self._get_update_paths()
+        if self.update_paths:
+            self.log("Found update paths:", level=DEBUG)
+            self.log(pprint.pformat(self.update_paths), level=DEBUG)
+        else:
+            self.log("Didn't find any update paths, cannot continue", level=FATAL)
+
+    def create_config(self):
+        from mozrelease.l10n import getPlatformLocales
+        from mozrelease.platforms import ftp2updatePlatforms
+        from mozrelease.update_verify import UpdateVerifyConfig
+        from mozrelease.paths import getCandidatesDir, getReleasesDir, getReleaseInstallerPath
+        from mozrelease.versions import getPrettyVersion
+
+        candidates_dir = getCandidatesDir(
+            self.config["stage_product"], self.config["to_version"],
+            self.config["to_build_number"],
+        )
+        to_ = getReleaseInstallerPath(
+            self.config["product"], self.config["product"].title(),
+            self.config["to_version"], self.config["platform"],
+            locale="%locale%"
+        )
+        to_path = "{}/{}".format(candidates_dir, to_)
+
+        to_display_version = self.config.get("to_display_version")
+        if not to_display_version:
+            to_display_version = getPrettyVersion(self.config["to_version"])
+
+        self.update_verify_config = UpdateVerifyConfig(
+            product=self.config["product"].title(), channel=self.config["channel"],
+            aus_server=self.config["aus_server"], to=to_path,
+            to_build_id=self.config["to_buildid"],
+            to_app_version=self.config["to_app_version"],
+            to_display_version=to_display_version,
+        )
+
+        to_shipped_locales_url = urljoin(
+            self.config["hg_server"],
+            "{}/raw-file/{}/{}/locales/shipped-locales".format(
+                self.config["repo_path"],
+                self.config["to_revision"],
+                self.config["app_name"],
+            ),
+        )
+        to_shipped_locales = self._retry_download(to_shipped_locales_url, "WARNING").read().strip()
+        to_locales = set(getPlatformLocales(to_shipped_locales, self.config["platform"]))
+
+        completes_only_index = 0
+        for fromVersion in reversed(sorted(self.update_paths, key=LooseVersion)):
+            from_ = self.update_paths[fromVersion]
+            locales = sorted(list(set(from_["locales"]).intersection(to_locales)))
+            appVersion = from_["appVersion"]
+            build_id = from_["buildID"]
+            mar_channel_IDs = from_.get('marChannelIds')
+
+            # Use new build targets for Windows, but only on compatible
+            #  versions (42+). See bug 1185456 for additional context.
+            if self.config["platform"] not in ("win32", "win64") or \
+                    LooseVersion(fromVersion) < LooseVersion("42.0"):
+                update_platform = ftp2updatePlatforms(self.config["platform"])[0]
+            else:
+                update_platform = ftp2updatePlatforms(self.config["platform"])[1]
+
+            release_dir = getReleasesDir(
+                self.config["stage_product"], fromVersion
+            )
+            path_ = getReleaseInstallerPath(
+                self.config["product"], self.config["product"].title(),
+                fromVersion, self.config["platform"], locale="%locale%",
+            )
+            from_path = "{}/{}".format(release_dir, path_)
+
+            updater_package = "{}/{}".format(
+                release_dir,
+                getReleaseInstallerPath(
+                    self.config["product"], self.config["product"].title(),
+                    fromVersion, self.config["updater_platform"],
+                    locale="%locale%",
+                )
+            )
+
+            # Exclude locales being full checked
+            quick_check_locales = [l for l in locales
+                                   if l not in self.config["full_check_locales"]]
+            # Get the intersection of from and to full_check_locales
+            this_full_check_locales = [l for l in self.config["full_check_locales"]
+                                       if l in locales]
+
+            if fromVersion in self.config["partial_versions"]:
+                self.info("Generating configs for partial update checks for %s" % fromVersion)
+                self.update_verify_config.addRelease(
+                    release=appVersion, build_id=build_id, locales=locales,
+                    patch_types=["complete", "partial"], from_path=from_path,
+                    ftp_server_from=self.config["previous_archive_prefix"],
+                    ftp_server_to=self.config["archive_prefix"],
+                    mar_channel_IDs=mar_channel_IDs, platform=update_platform,
+                    updater_package=updater_package
+                )
+            else:
+                if this_full_check_locales and is_triangualar(completes_only_index):
+                    self.info("Generating full check configs for %s" % fromVersion)
+                    self.update_verify_config.addRelease(
+                        release=appVersion, build_id=build_id, locales=this_full_check_locales,
+                        from_path=from_path,
+                        ftp_server_from=self.config["previous_archive_prefix"],
+                        ftp_server_to=self.config["archive_prefix"],
+                        mar_channel_IDs=mar_channel_IDs, platform=update_platform,
+                        updater_package=updater_package
+                    )
+                # Quick test for other locales, no download
+                if len(quick_check_locales) > 0:
+                    self.info("Generating quick check configs for %s" % fromVersion)
+                    if not is_triangualar(completes_only_index):
+                        # Assuming we skipped full check locales, using all locales
+                        _locales = locales
+                    else:
+                        # Excluding full check locales from the quick check
+                        _locales = quick_check_locales
+                    self.update_verify_config.addRelease(
+                        release=appVersion, build_id=build_id,
+                        locales=_locales, platform=update_platform
+                    )
+                completes_only_index += 1
+
+    def write_config(self):
+        with open(self.config["output_file"], "w+") as fh:
+            self.update_verify_config.write(fh)
+
+
+if __name__ == "__main__":
+    UpdateVerifyConfigCreator().run_and_exit()

toolkit/components/places/PlacesDBUtils.jsm

@@ -898,6 +898,9 @@ this.PlacesDBUtils = {
    *
    */
   async telemetry() {
+    // First deal with some scalars for feeds:
+    await this._telemetryForFeeds();
+
     // This will be populated with one integer property for each probe result,
     // using the histogram name as key.
     let probeValues = {};
@@ -1045,6 +1048,16 @@ this.PlacesDBUtils = {
     }
   },
 
+  async _telemetryForFeeds() {
+    let db = await PlacesUtils.promiseDBConnection();
+    let livebookmarkCount = await db.execute(
+      `SELECT count(*) FROM moz_items_annos a
+                       JOIN moz_anno_attributes aa ON a.anno_attribute_id = aa.id
+                       WHERE aa.name = 'livemark/feedURI'`);
+    livebookmarkCount = livebookmarkCount[0].getResultByIndex(0);
+    Services.telemetry.scalarSet("browser.feeds.livebookmark_count", livebookmarkCount);
+  },
+
   /**
    * Runs a list of tasks, returning a Map when done.
    *

toolkit/components/telemetry/Scalars.yaml

@@ -262,6 +262,74 @@ browser.engagement.navigation:
     record_in_processes:
       - 'main'
 
+# The following section contains (RSS/Atom) feed usage
+browser.feeds:
+  preview_loaded:
+    bug_numbers:
+      - 1350349
+    description: >
+      The number of times a feed preview page is loaded.
+    expires: "65"
+    kind: uint
+    notification_emails:
+      - gijs@mozilla.com
+    release_channel_collection: opt-out
+    record_in_processes:
+      - 'main'
+      - 'content'
+
+  feed_subscribed:
+    bug_numbers:
+      - 1350349
+    description: >
+      The number of times a user subscribes to a feed.
+    expires: "65"
+    kind: uint
+    notification_emails:
+      - gijs@mozilla.com
+    release_channel_collection: opt-out
+    record_in_processes:
+      - 'main'
+
+  livebookmark_opened:
+    bug_numbers:
+      - 1350349
+    description: >
+      The number of times a user opens a live bookmark dropdown.
+    expires: "65"
+    kind: uint
+    notification_emails:
+      - gijs@mozilla.com
+    release_channel_collection: opt-out
+    record_in_processes:
+      - 'main'
+
+  livebookmark_item_opened:
+    bug_numbers:
+      - 1350349
+    description: >
+      The number of times a user opens an item that is part of a live bookmark.
+    expires: "65"
+    kind: uint
+    notification_emails:
+      - gijs@mozilla.com
+    release_channel_collection: opt-out
+    record_in_processes:
+      - 'main'
+
+  livebookmark_count:
+    bug_numbers:
+      - 1350349
+    description: >
+      The number of live bookmark folders the user has (ie not individual items).
+    expires: "65"
+    kind: uint
+    notification_emails:
+      - gijs@mozilla.com
+    release_channel_collection: opt-out
+    record_in_processes:
+      - 'main'
+
 # The following section contains the browser usage scalars.
 browser.usage:
   graphite: