зеркало из https://github.com/mozilla/gecko-dev.git
bug 1398799: mozharness script to create update verify configs without relying on patcher configs. r=nthomas
This commit is contained in:
Ben Hearsum
Родитель
955ff5438d
Коммит
0d4b578055
|
|
@ -29,6 +29,9 @@ with Files('mozversioncontrol/**'):
|
|||
with Files('l10n/**'):
|
||||
BUG_COMPONENT = ('Core', 'Localization')
|
||||
|
||||
with Files('mozrelease/**'):
|
||||
BUG_COMPONENT = ('Release Engineering', 'Release Automation')
|
||||
|
||||
with Files('mach_commands.py'):
|
||||
BUG_COMPONENT = ('Testing', 'Python Test')
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,24 @@
|
|||
from __future__ import absolute_import
|
||||
|
||||
from copy import copy
|
||||
|
||||
|
||||
class ChunkingError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def getChunk(things, chunks, thisChunk):
|
||||
if thisChunk > chunks:
|
||||
raise ChunkingError("thisChunk (%d) is greater than total chunks (%d)" %
|
||||
(thisChunk, chunks))
|
||||
possibleThings = copy(things)
|
||||
nThings = len(possibleThings)
|
||||
for c in range(1, chunks + 1):
|
||||
n = nThings / chunks
|
||||
# If our things aren't evenly divisible by the number of chunks
|
||||
# we need to append one more onto some of them
|
||||
if c <= (nThings % chunks):
|
||||
n += 1
|
||||
if c == thisChunk:
|
||||
return possibleThings[0:n]
|
||||
del possibleThings[0:n]
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
from __future__ import absolute_import
|
||||
|
||||
from .platforms import shippedLocales2ftp
|
||||
|
||||
|
||||
def getPlatformLocales(shipped_locales, platform):
|
||||
platform_locales = []
|
||||
for line in shipped_locales.splitlines():
|
||||
entry = line.strip().split()
|
||||
locale = entry[0]
|
||||
if len(entry) > 1:
|
||||
for sl_platform in entry[1:]:
|
||||
if platform in shippedLocales2ftp(sl_platform):
|
||||
platform_locales.append(locale)
|
||||
else:
|
||||
platform_locales.append(locale)
|
||||
return platform_locales
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
from __future__ import absolute_import
|
||||
|
||||
from urlparse import urlunsplit
|
||||
|
||||
product_ftp_map = {
|
||||
"fennec": "mobile",
|
||||
}
|
||||
|
||||
def product2ftp(product):
|
||||
return product_ftp_map.get(product, product)
|
||||
|
||||
|
||||
def getCandidatesDir(product, version, buildNumber, protocol=None, server=None):
|
||||
if protocol:
|
||||
assert server is not None, "server is required with protocol"
|
||||
|
||||
product = product2ftp(product)
|
||||
directory = "/{}/candidates/{}-candidates/build{}".format(
|
||||
product, str(version), str(buildNumber)
|
||||
)
|
||||
|
||||
if protocol:
|
||||
return urlunsplit((protocol, server, directory, None, None))
|
||||
else:
|
||||
return directory
|
||||
|
||||
|
||||
def getReleasesDir(product, version=None, protocol=None, server=None):
|
||||
if protocol:
|
||||
assert server is not None, "server is required with protocol"
|
||||
|
||||
directory = "/{}/releases".format(product)
|
||||
if version:
|
||||
directory = "{}/{}".format(directory, version)
|
||||
|
||||
if protocol:
|
||||
return urlunsplit((protocol, server, directory, None, None))
|
||||
else:
|
||||
return directory
|
||||
|
||||
|
||||
def getReleaseInstallerPath(productName, brandName, version, platform,
|
||||
locale='en-US'):
|
||||
if productName not in ('fennec',):
|
||||
if platform.startswith('linux'):
|
||||
return '/'.join([p.strip('/') for p in [
|
||||
platform, locale, '%s-%s.tar.bz2' % (productName, version)]])
|
||||
elif 'mac' in platform:
|
||||
return '/'.join([p.strip('/') for p in [
|
||||
platform, locale, '%s %s.dmg' % (brandName, version)]])
|
||||
elif platform.startswith('win'):
|
||||
return '/'.join([p.strip('/') for p in [
|
||||
platform, locale, '%s Setup %s.exe' % (brandName, version)]])
|
||||
else:
|
||||
raise "Unsupported platform"
|
||||
else:
|
||||
if platform.startswith('android'):
|
||||
filename = '%s-%s.%s.android-arm.apk' % (
|
||||
productName, version, locale)
|
||||
return '/'.join([p.strip('/') for p in [
|
||||
platform, locale, filename]])
|
||||
else:
|
||||
raise "Unsupported platform"
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
from __future__ import absolute_import
|
||||
|
||||
# ftp -> update platform map
|
||||
update_platform_map = {
|
||||
"android": ["Android_arm-eabi-gcc3"],
|
||||
"android-api-11": ["Android_arm-eabi-gcc3"],
|
||||
"android-api-15": ["Android_arm-eabi-gcc3"],
|
||||
"android-api-15-old-id": ["Android_arm-eabi-gcc3"],
|
||||
"android-api-16": ["Android_arm-eabi-gcc3"],
|
||||
"android-api-16-old-id": ["Android_arm-eabi-gcc3"],
|
||||
"android-x86": ["Android_x86-gcc3"],
|
||||
"android-x86-old-id": ["Android_x86-gcc3"],
|
||||
"android-aarch64": ["Android_aarch64-gcc3"],
|
||||
"linux-i686": ["Linux_x86-gcc3"],
|
||||
"linux-x86_64": ["Linux_x86_64-gcc3"],
|
||||
"mac": ["Darwin_x86_64-gcc3-u-i386-x86_64", # The main platofrm
|
||||
"Darwin_x86-gcc3-u-i386-x86_64",
|
||||
# We don"t ship builds with these build targets, but some users
|
||||
# modify their builds in a way that has them report like these.
|
||||
# See bug 1071576 for details.
|
||||
"Darwin_x86-gcc3", "Darwin_x86_64-gcc3"],
|
||||
"win32": ["WINNT_x86-msvc", "WINNT_x86-msvc-x86", "WINNT_x86-msvc-x64"],
|
||||
"win64": ["WINNT_x86_64-msvc", "WINNT_x86_64-msvc-x64"],
|
||||
}
|
||||
|
||||
# ftp -> shipped locales map
|
||||
sl_platform_map = {
|
||||
"linux-i686": "linux",
|
||||
"linux-x86_64": "linux",
|
||||
"mac": "osx",
|
||||
"win32": "win32",
|
||||
"win64": "win64",
|
||||
}
|
||||
|
||||
# ftp -> info file platform map
|
||||
info_file_platform_map = {
|
||||
"linux-i686": "linux",
|
||||
"linux-x86_64": "linux64",
|
||||
"mac": "macosx64",
|
||||
"win32": "win32",
|
||||
"win64": "win64",
|
||||
}
|
||||
|
||||
def ftp2updatePlatforms(platform):
|
||||
return update_platform_map.get(platform, platform)
|
||||
|
||||
def ftp2shippedLocales(platform):
|
||||
return sl_platform_map.get(platform, platform)
|
||||
|
||||
def shippedLocales2ftp(platform):
|
||||
matches = []
|
||||
try:
|
||||
[matches.append(
|
||||
k) for k, v in sl_platform_map.iteritems() if v == platform][0]
|
||||
return matches
|
||||
except IndexError:
|
||||
return [platform]
|
||||
|
||||
def ftp2infoFile(platform):
|
||||
return info_file_platform_map.get(platform, platform)
|
||||
|
|
@ -0,0 +1,190 @@
|
|||
from __future__ import absolute_import
|
||||
|
||||
import os
|
||||
import re
|
||||
|
||||
from .chunking import getChunk
|
||||
|
||||
|
||||
class UpdateVerifyError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class UpdateVerifyConfig(object):
|
||||
comment_regex = re.compile("^#")
|
||||
key_write_order = ("release", "product", "platform", "build_id", "locales",
|
||||
"channel", "patch_types", "from", "aus_server",
|
||||
"ftp_server_from", "ftp_server_to", "to",
|
||||
"mar_channel_IDs", "to_build_id", "to_display_version",
|
||||
"to_app_version", "updater_package")
|
||||
global_keys = ("product", "channel", "aus_server", "to", "to_build_id",
|
||||
"to_display_version", "to_app_version")
|
||||
release_keys = ("release", "build_id", "locales", "patch_types", "from",
|
||||
"ftp_server_from", "ftp_server_to", "mar_channel_IDs",
|
||||
"platform", "updater_package")
|
||||
first_only_keys = ("from", "aus_server", "to", "to_build_id",
|
||||
"to_display_version", "to_app_version")
|
||||
compare_attrs = global_keys + ("releases",)
|
||||
|
||||
def __init__(self, product=None, channel=None,
|
||||
aus_server=None, to=None, to_build_id=None,
|
||||
to_display_version=None, to_app_version=None):
|
||||
self.product = product
|
||||
self.channel = channel
|
||||
self.aus_server = aus_server
|
||||
self.to = to
|
||||
self.to_build_id = to_build_id
|
||||
self.to_display_version = to_display_version
|
||||
self.to_app_version = to_app_version
|
||||
self.releases = []
|
||||
|
||||
def __eq__(self, other):
|
||||
self_list = [getattr(self, attr) for attr in self.compare_attrs]
|
||||
other_list = [getattr(other, attr) for attr in self.compare_attrs]
|
||||
return self_list == other_list
|
||||
|
||||
def __ne__(self, other):
|
||||
return not self.__eq__(other)
|
||||
|
||||
def _parseLine(self, line):
|
||||
entry = {}
|
||||
items = re.findall("\w+=[\"'][^\"']*[\"']", line)
|
||||
for i in items:
|
||||
m = re.search(
|
||||
"(?P<key>\w+)=[\"'](?P<value>.+)[\"']", i).groupdict()
|
||||
if m["key"] not in self.global_keys and m["key"] not in self.release_keys:
|
||||
raise UpdateVerifyError(
|
||||
"Unknown key '%s' found on line:\n%s" % (m["key"], line))
|
||||
if m["key"] in entry:
|
||||
raise UpdateVerifyError("Multiple values found for key '%s' on line:\n%s" % (m["key"], line))
|
||||
entry[m["key"]] = m["value"]
|
||||
if not entry:
|
||||
raise UpdateVerifyError("No parseable data in line '%s'" % line)
|
||||
return entry
|
||||
|
||||
def _addEntry(self, entry, first):
|
||||
releaseKeys = {}
|
||||
for k, v in entry.items():
|
||||
if k in self.global_keys:
|
||||
setattr(self, k, entry[k])
|
||||
elif k in self.release_keys:
|
||||
# "from" is reserved in Python
|
||||
if k == "from":
|
||||
releaseKeys["from_path"] = v
|
||||
else:
|
||||
releaseKeys[k] = v
|
||||
self.addRelease(**releaseKeys)
|
||||
|
||||
def read(self, config):
|
||||
f = open(config)
|
||||
# Only the first non-comment line of an update verify config should
|
||||
# have a "from" and"ausServer". Ignore any subsequent lines with them.
|
||||
first = True
|
||||
for line in f.readlines():
|
||||
# Skip comment lines
|
||||
if self.comment_regex.search(line):
|
||||
continue
|
||||
self._addEntry(self._parseLine(line), first)
|
||||
first = False
|
||||
|
||||
def write(self, fh):
|
||||
first = True
|
||||
for releaseInfo in self.releases:
|
||||
for key in self.key_write_order:
|
||||
if key in self.global_keys and (first or key not in self.first_only_keys):
|
||||
value = getattr(self, key)
|
||||
elif key in self.release_keys:
|
||||
value = releaseInfo[key]
|
||||
else:
|
||||
value = None
|
||||
if value is not None:
|
||||
fh.write(key)
|
||||
fh.write("=")
|
||||
if isinstance(value, (list, tuple)):
|
||||
fh.write('"%s" ' % " ".join(value))
|
||||
else:
|
||||
fh.write('"%s" ' % str(value))
|
||||
# Rewind one character to avoid having a trailing space
|
||||
fh.seek(-1, os.SEEK_CUR)
|
||||
fh.write("\n")
|
||||
first = False
|
||||
|
||||
def addRelease(self, release=None, build_id=None, locales=[],
|
||||
patch_types=['complete'], from_path=None,
|
||||
ftp_server_from=None, ftp_server_to=None,
|
||||
mar_channel_IDs=None, platform=None, updater_package=None):
|
||||
"""Locales and patch_types can be passed as either a string or a list.
|
||||
If a string is passed, they will be converted to a list for internal
|
||||
storage"""
|
||||
if self.getRelease(build_id, from_path):
|
||||
raise UpdateVerifyError("Couldn't add release identified by build_id '%s' and from_path '%s': already exists in config" % (build_id, from_path))
|
||||
if isinstance(locales, basestring):
|
||||
locales = sorted(list(locales.split()))
|
||||
if isinstance(patch_types, basestring):
|
||||
patch_types = list(patch_types.split())
|
||||
self.releases.append({
|
||||
"release": release,
|
||||
"build_id": build_id,
|
||||
"locales": locales,
|
||||
"patch_types": patch_types,
|
||||
"from": from_path,
|
||||
"ftp_server_from": ftp_server_from,
|
||||
"ftp_server_to": ftp_server_to,
|
||||
"mar_channel_IDs": mar_channel_IDs,
|
||||
"platform": platform,
|
||||
"updater_package": updater_package,
|
||||
})
|
||||
|
||||
def addLocaleToRelease(self, build_id, locale, from_path=None):
|
||||
r = self.getRelease(build_id, from_path)
|
||||
if not r:
|
||||
raise UpdateVerifyError("Couldn't add '%s' to release identified by build_id '%s' and from_path '%s': '%s' doesn't exist in this config." % (locale, build_id, from_path, build_id))
|
||||
r["locales"].append(locale)
|
||||
r["locales"] = sorted(r["locales"])
|
||||
|
||||
def getRelease(self, build_id, from_path):
|
||||
for r in self.releases:
|
||||
if r["build_id"] == build_id and r["from"] == from_path:
|
||||
return r
|
||||
return {}
|
||||
|
||||
def getFullReleaseTests(self):
|
||||
return [r for r in self.releases if r["from"] is not None]
|
||||
|
||||
def getQuickReleaseTests(self):
|
||||
return [r for r in self.releases if r["from"] is None]
|
||||
|
||||
def getChunk(self, chunks, thisChunk):
|
||||
fullTests = []
|
||||
quickTests = []
|
||||
for test in self.getFullReleaseTests():
|
||||
for locale in test["locales"]:
|
||||
fullTests.append([test["build_id"], locale, test["from"]])
|
||||
for test in self.getQuickReleaseTests():
|
||||
for locale in test["locales"]:
|
||||
quickTests.append([test["build_id"], locale, test["from"]])
|
||||
allTests = getChunk(fullTests, chunks, thisChunk)
|
||||
allTests.extend(getChunk(quickTests, chunks, thisChunk))
|
||||
|
||||
newConfig = UpdateVerifyConfig(self.product, self.channel,
|
||||
self.aus_server, self.to,
|
||||
self.to_build_id,
|
||||
self.to_display_version,
|
||||
self.to_app_version)
|
||||
for t in allTests:
|
||||
build_id, locale, from_path = t
|
||||
if from_path == "None":
|
||||
from_path = None
|
||||
r = self.getRelease(build_id, from_path)
|
||||
try:
|
||||
newConfig.addRelease(r["release"], build_id, locales=[],
|
||||
ftp_server_from=r["ftp_server_from"],
|
||||
ftp_server_to=r["ftp_server_to"],
|
||||
patch_types=r["patch_types"], from_path=from_path,
|
||||
mar_channel_IDs=r["mar_channel_IDs"],
|
||||
platform=r["platform"],
|
||||
updater_package=r["updater_package"])
|
||||
except UpdateVerifyError:
|
||||
pass
|
||||
newConfig.addLocaleToRelease(build_id, locale, from_path)
|
||||
return newConfig
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
from __future__ import absolute_import
|
||||
|
||||
from distutils.version import StrictVersion
|
||||
import re
|
||||
|
||||
|
||||
class ModernMozillaVersion(StrictVersion):
|
||||
"""A version class that is slightly less restrictive than StrictVersion.
|
||||
Instead of just allowing "a" or "b" as prerelease tags, it allows any
|
||||
alpha. This allows us to support the once-shipped "3.6.3plugin1" and
|
||||
similar versions."""
|
||||
version_re = re.compile(r"""^(\d+) \. (\d+) (\. (\d+))?
|
||||
([a-zA-Z]+(\d+))?$""", re.VERBOSE)
|
||||
|
||||
|
||||
class AncientMozillaVersion(StrictVersion):
|
||||
"""A version class that is slightly less restrictive than StrictVersion.
|
||||
Instead of just allowing "a" or "b" as prerelease tags, it allows any
|
||||
alpha. This allows us to support the once-shipped "3.6.3plugin1" and
|
||||
similar versions.
|
||||
It also supports versions w.x.y.z by transmuting to w.x.z, which
|
||||
is useful for versions like 1.5.0.x and 2.0.0.y"""
|
||||
version_re = re.compile(r"""^(\d+) \. (\d+) \. \d (\. (\d+))
|
||||
([a-zA-Z]+(\d+))?$""", re.VERBOSE)
|
||||
|
||||
|
||||
def MozillaVersion(version):
|
||||
try:
|
||||
return ModernMozillaVersion(version)
|
||||
except ValueError:
|
||||
pass
|
||||
try:
|
||||
if version.count('.') == 3:
|
||||
return AncientMozillaVersion(version)
|
||||
except ValueError:
|
||||
pass
|
||||
raise ValueError("Version number %s is invalid." % version)
|
||||
|
||||
|
||||
def getPrettyVersion(version):
|
||||
version = re.sub(r'a([0-9]+)$', r' Alpha \1', version)
|
||||
version = re.sub(r'b([0-9]+)$', r' Beta \1', version)
|
||||
version = re.sub(r'rc([0-9]+)$', r' RC \1', version)
|
||||
return version
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
from setuptools import setup, find_packages
|
||||
|
||||
VERSION = '0.1'
|
||||
|
||||
setup(
|
||||
author='Mozilla Foundation',
|
||||
author_email='Mozilla Release Engineering',
|
||||
name='mozrelease',
|
||||
description='Common functionality used by Mozilla Release Automation',
|
||||
license='MPL 2.0',
|
||||
packages=find_packages(),
|
||||
version=VERSION,
|
||||
classifiers=[
|
||||
'Development Status :: 3 - Alpha',
|
||||
'Topic :: Software Development :: Build Tools',
|
||||
'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
|
||||
'Programming Language :: Python :: 2.7',
|
||||
'Programming Language :: Python :: Implementation :: CPython',
|
||||
],
|
||||
keywords='mozilla',
|
||||
)
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
config = {
|
||||
"find_links": [
|
||||
"http://pypi.pvt.build.mozilla.org/pub",
|
||||
"http://pypi.pub.build.mozilla.org/pub",
|
||||
],
|
||||
'pip_index': False,
|
||||
}
|
||||
|
|
@ -0,0 +1,452 @@
|
|||
from distutils.version import LooseVersion
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import pprint
|
||||
import re
|
||||
import sys
|
||||
from urlparse import urljoin
|
||||
|
||||
|
||||
sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
|
||||
|
||||
from mozharness.base.log import DEBUG, INFO, FATAL
|
||||
from mozharness.base.script import BaseScript
|
||||
from mozharness.base.python import VirtualenvMixin
|
||||
|
||||
|
||||
def is_triangualar(x):
|
||||
"""Check if a number is triangular (0, 1, 3, 6, 10, 15, ...)
|
||||
see: https://en.wikipedia.org/wiki/Triangular_number#Triangular_roots_and_tests_for_triangular_numbers # noqa
|
||||
|
||||
>>> is_triangualar(0)
|
||||
True
|
||||
>>> is_triangualar(1)
|
||||
True
|
||||
>>> is_triangualar(2)
|
||||
False
|
||||
>>> is_triangualar(3)
|
||||
True
|
||||
>>> is_triangualar(4)
|
||||
False
|
||||
>>> all(is_triangualar(x) for x in [0, 1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 66, 78, 91, 105])
|
||||
True
|
||||
>>> all(not is_triangualar(x) for x in [4, 5, 8, 9, 11, 17, 25, 29, 39, 44, 59, 61, 72, 98, 112])
|
||||
True
|
||||
"""
|
||||
n = (math.sqrt(8*x + 1) - 1)/2
|
||||
return n == int(n)
|
||||
|
||||
|
||||
class UpdateVerifyConfigCreator(BaseScript, VirtualenvMixin):
|
||||
config_options = [
|
||||
[["--product"], {
|
||||
"dest": "product",
|
||||
"help": "Product being tested, as used in the update URL and filenames. Eg: firefox",
|
||||
}],
|
||||
[["--stage-product"], {
|
||||
"dest": "stage_product",
|
||||
"help": "Product being tested, as used in stage directories and ship it"
|
||||
"If not passed this is assumed to be the same as product."
|
||||
}],
|
||||
[["--app-name"], {
|
||||
"dest": "app_name",
|
||||
"help": "App name being tested. Eg: browser",
|
||||
}],
|
||||
[["--channel"], {
|
||||
"dest": "channel",
|
||||
"help": "Channel to run update verify against",
|
||||
}],
|
||||
[["--aus-server"], {
|
||||
"dest": "aus_server",
|
||||
"default": "https://aus5.mozilla.org",
|
||||
"help": "AUS server to run update verify against",
|
||||
}],
|
||||
[["--to-version"], {
|
||||
"dest": "to_version",
|
||||
"help": "The version of the release being updated to. Eg: 59.0b5",
|
||||
}],
|
||||
[["--to-app-version"], {
|
||||
"dest": "to_app_version",
|
||||
"help": "The in-app version of the release being updated to. Eg: 59.0",
|
||||
}],
|
||||
[["--to-display-version"], {
|
||||
"dest": "to_display_version",
|
||||
"help": "The human-readable version of the release being updated to. Eg: 59.0 Beta 9",
|
||||
}],
|
||||
[["--to-build-number"], {
|
||||
"dest": "to_build_number",
|
||||
"help": "The build number of the release being updated to",
|
||||
}],
|
||||
[["--to-buildid"], {
|
||||
"dest": "to_buildid",
|
||||
"help": "The buildid of the release being updated to",
|
||||
}],
|
||||
[["--to-revision"], {
|
||||
"dest": "to_revision",
|
||||
"help": "The revision that the release being updated to was built against",
|
||||
}],
|
||||
[["--partial-version"], {
|
||||
"dest": "partial_versions",
|
||||
"action": "append",
|
||||
"help": "A previous release version that is expected to receive a partial update. "
|
||||
"Eg: 59.0b4. May be specified multiple times."
|
||||
}],
|
||||
[["--last-watershed"], {
|
||||
"dest": "last_watershed",
|
||||
"help": "The earliest version to include in the update verify config. Eg: 57.0b10",
|
||||
}],
|
||||
[["--include-version"], {
|
||||
"dest": "include_versions",
|
||||
"default": [],
|
||||
"action": "append",
|
||||
"help": "Only include versions that match one of these regexes. "
|
||||
"May be passed multiple times",
|
||||
}],
|
||||
[["--mar-channel-id-override"], {
|
||||
"dest": "mar_channel_id_options",
|
||||
"default": [],
|
||||
"action": "append",
|
||||
"help": "A version regex and channel id string to override those versions with."
|
||||
"Eg: ^\\d+\\.\\d+(\\.\\d+)?$,firefox-mozilla-beta,firefox-mozilla-release "
|
||||
"will set accepted mar channel ids to 'firefox-mozilla-beta' and "
|
||||
"'firefox-mozilla-release for x.y and x.y.z versions. "
|
||||
"May be passed multiple times"
|
||||
}],
|
||||
[["--platform"], {
|
||||
"dest": "platform",
|
||||
"help": "The platform to generate the update verify config for, in FTP-style",
|
||||
}],
|
||||
[["--updater-platform"], {
|
||||
"dest": "updater_platform",
|
||||
"help": "The platform to run the updater on, in FTP-style."
|
||||
"If not specified, this is assumed to be the same as platform",
|
||||
}],
|
||||
[["--archive-prefix"], {
|
||||
"dest": "archive_prefix",
|
||||
"help": "The server/path to pull the current release from. "
|
||||
"Eg: https://archive.mozilla.org/pub",
|
||||
}],
|
||||
[["--previous-archive-prefix"], {
|
||||
"dest": "previous_archive_prefix",
|
||||
"help": "The server/path to pull the previous releases from"
|
||||
"If not specified, this is assumed to be the same as --archive-prefix"
|
||||
}],
|
||||
[["--repo-path"], {
|
||||
"dest": "repo_path",
|
||||
"help": "The repository (relative to the hg server root) that the current release was "
|
||||
"built from Eg: releases/mozilla-beta"
|
||||
}],
|
||||
[["--output-file"], {
|
||||
"dest": "output_file",
|
||||
"help": "Where to write the update verify config to",
|
||||
}],
|
||||
[["--product-details-server"], {
|
||||
"dest": "product_details_server",
|
||||
"default": "https://product-details.mozilla.org",
|
||||
"help": "Product Details server to pull previous release info from. "
|
||||
"Using anything other than the production server is likely to "
|
||||
"cause issues with update verify."
|
||||
}],
|
||||
[["--hg-server"], {
|
||||
"dest": "hg_server",
|
||||
"default": "https://hg.mozilla.org",
|
||||
"help": "Mercurial server to pull various previous and current version info from",
|
||||
}],
|
||||
[["--full-check-locale"], {
|
||||
"dest": "full_check_locales",
|
||||
"default": ["de", "en-US", "ru"],
|
||||
"action": "append",
|
||||
"help": "A list of locales to generate full update verify checks for",
|
||||
}],
|
||||
]
|
||||
|
||||
def __init__(self):
|
||||
BaseScript.__init__(
|
||||
self,
|
||||
config_options=self.config_options,
|
||||
config={
|
||||
"virtualenv_modules": [
|
||||
"mozrelease",
|
||||
],
|
||||
"virtualenv_path": "venv",
|
||||
},
|
||||
all_actions=[
|
||||
"create-virtualenv",
|
||||
"activate-virtualenv",
|
||||
"gather-info",
|
||||
"create-config",
|
||||
"write-config",
|
||||
],
|
||||
default_actions=[
|
||||
"create-virtualenv",
|
||||
"activate-virtualenv",
|
||||
"gather-info",
|
||||
"create-config",
|
||||
"write-config",
|
||||
],
|
||||
)
|
||||
|
||||
def _pre_config_lock(self, rw_config):
|
||||
super(UpdateVerifyConfigCreator, self)._pre_config_lock(rw_config)
|
||||
|
||||
if "updater_platform" not in self.config:
|
||||
self.config["updater_platform"] = self.config["platform"]
|
||||
if "stage_product" not in self.config:
|
||||
self.config["stage_product"] = self.config["product"]
|
||||
if "previous_archive_prefix" not in self.config:
|
||||
self.config["previous_archive_prefix"] = self.config["archive_prefix"]
|
||||
self.config["archive_prefix"].rstrip("/")
|
||||
self.config["previous_archive_prefix"].rstrip("/")
|
||||
self.config["mar_channel_id_overrides"] = {}
|
||||
for override in self.config["mar_channel_id_options"]:
|
||||
pattern, override_str = override.split(",", 1)
|
||||
self.config["mar_channel_id_overrides"][pattern] = override_str
|
||||
|
||||
def _get_update_paths(self):
|
||||
from mozrelease.l10n import getPlatformLocales
|
||||
from mozrelease.paths import getCandidatesDir
|
||||
from mozrelease.platforms import ftp2infoFile
|
||||
from mozrelease.versions import MozillaVersion
|
||||
|
||||
self.update_paths = {}
|
||||
|
||||
ret = self._retry_download(
|
||||
"{}/1.0/{}.json".format(
|
||||
self.config["product_details_server"],
|
||||
self.config["stage_product"],
|
||||
),
|
||||
"WARNING",
|
||||
)
|
||||
releases = json.load(ret)["releases"]
|
||||
for release_name, release_info in reversed(sorted(releases.items())):
|
||||
product, version = release_name.split("-", 1)
|
||||
version = version.rstrip("esr")
|
||||
tag = "{}_{}_RELEASE".format(product.upper(), version.replace(".", "_"))
|
||||
# Product details has a "category" for releases that we can use to
|
||||
# determine the repo path. This will fail if any previous releases
|
||||
# were built from a project branch - but that's not something we do
|
||||
# at the time of writing.
|
||||
branch = None
|
||||
if release_info["category"] == "dev":
|
||||
branch = "releases/mozilla-beta"
|
||||
elif release_info["category"] == "esr":
|
||||
branch = "releases/mozilla-esr{}".format(version[:2])
|
||||
elif release_info["category"] in ("major", "stability"):
|
||||
branch = "releases/mozilla-release"
|
||||
if not branch:
|
||||
raise Exception("Cannot determine branch, cannot continue!")
|
||||
|
||||
# Exclude any releases that don't match one of our include version
|
||||
# regexes. This is generally to avoid including versions from other
|
||||
# channels. Eg: including betas when testing releases
|
||||
for v in self.config["include_versions"]:
|
||||
if re.match(v, version):
|
||||
break
|
||||
else:
|
||||
self.log("Skipping release whose version doesn't match any "
|
||||
"include_version pattern: %s" % release_name,
|
||||
level=INFO)
|
||||
continue
|
||||
|
||||
# We also have to trim out previous releases that aren't in the same
|
||||
# product line, too old, etc.
|
||||
if self.config["stage_product"] != product:
|
||||
self.log("Skipping release that doesn't match product name: %s" % release_name,
|
||||
level=INFO)
|
||||
continue
|
||||
if MozillaVersion(version) < MozillaVersion(self.config["last_watershed"]):
|
||||
self.log("Skipping release that's behind the last watershed: %s" % release_name,
|
||||
level=INFO)
|
||||
continue
|
||||
if version == self.config["to_version"]:
|
||||
self.log("Skipping release that is the same as to version: %s" % release_name,
|
||||
level=INFO)
|
||||
continue
|
||||
if MozillaVersion(version) > MozillaVersion(self.config["to_version"]):
|
||||
self.log("Skipping release that's newer than to version: %s" % release_name,
|
||||
level=INFO)
|
||||
continue
|
||||
|
||||
if version in self.update_paths:
|
||||
raise Exception("Found duplicate release for version: %s", version)
|
||||
|
||||
# This is a crappy place to get buildids from, but we don't have a better one.
|
||||
# This will start to fail if old info files are deleted.
|
||||
info_file_url = "{}{}/{}_info.txt".format(
|
||||
self.config["previous_archive_prefix"],
|
||||
getCandidatesDir(
|
||||
self.config["stage_product"],
|
||||
version,
|
||||
release_info["build_number"],
|
||||
),
|
||||
ftp2infoFile(self.config["platform"])
|
||||
)
|
||||
self.log("Retrieving buildid from info file: %s" % info_file_url, level=DEBUG)
|
||||
ret = self._retry_download(info_file_url, "WARNING")
|
||||
buildID = ret.read().split("=")[1].strip()
|
||||
|
||||
shipped_locales_url = urljoin(
|
||||
self.config["hg_server"],
|
||||
"{}/raw-file/{}/{}/locales/shipped-locales".format(
|
||||
branch,
|
||||
tag,
|
||||
self.config["app_name"],
|
||||
),
|
||||
)
|
||||
ret = self._retry_download(shipped_locales_url, "WARNING")
|
||||
shipped_locales = ret.read().strip()
|
||||
|
||||
app_version_url = urljoin(
|
||||
self.config["hg_server"],
|
||||
"{}/raw-file/{}/{}/config/version.txt".format(
|
||||
branch,
|
||||
tag,
|
||||
self.config["app_name"],
|
||||
),
|
||||
)
|
||||
app_version = self._retry_download(app_version_url, "WARNING").read().strip()
|
||||
|
||||
self.log("Adding {} to update paths".format(version), level=INFO)
|
||||
self.update_paths[version] = {
|
||||
"appVersion": app_version,
|
||||
"locales": getPlatformLocales(shipped_locales, self.config["platform"]),
|
||||
"buildID": buildID,
|
||||
}
|
||||
for pattern, mar_channel_ids in self.config["mar_channel_id_overrides"].items():
|
||||
if re.match(pattern, version):
|
||||
self.update_paths[version]["marChannelIds"] = mar_channel_ids
|
||||
|
||||
def gather_info(self):
|
||||
self._get_update_paths()
|
||||
if self.update_paths:
|
||||
self.log("Found update paths:", level=DEBUG)
|
||||
self.log(pprint.pformat(self.update_paths), level=DEBUG)
|
||||
else:
|
||||
self.log("Didn't find any update paths, cannot continue", level=FATAL)
|
||||
|
||||
def create_config(self):
|
||||
from mozrelease.l10n import getPlatformLocales
|
||||
from mozrelease.platforms import ftp2updatePlatforms
|
||||
from mozrelease.update_verify import UpdateVerifyConfig
|
||||
from mozrelease.paths import getCandidatesDir, getReleasesDir, getReleaseInstallerPath
|
||||
from mozrelease.versions import getPrettyVersion
|
||||
|
||||
candidates_dir = getCandidatesDir(
|
||||
self.config["stage_product"], self.config["to_version"],
|
||||
self.config["to_build_number"],
|
||||
)
|
||||
to_ = getReleaseInstallerPath(
|
||||
self.config["product"], self.config["product"].title(),
|
||||
self.config["to_version"], self.config["platform"],
|
||||
locale="%locale%"
|
||||
)
|
||||
to_path = "{}/{}".format(candidates_dir, to_)
|
||||
|
||||
to_display_version = self.config.get("to_display_version")
|
||||
if not to_display_version:
|
||||
to_display_version = getPrettyVersion(self.config["to_version"])
|
||||
|
||||
self.update_verify_config = UpdateVerifyConfig(
|
||||
product=self.config["product"].title(), channel=self.config["channel"],
|
||||
aus_server=self.config["aus_server"], to=to_path,
|
||||
to_build_id=self.config["to_buildid"],
|
||||
to_app_version=self.config["to_app_version"],
|
||||
to_display_version=to_display_version,
|
||||
)
|
||||
|
||||
to_shipped_locales_url = urljoin(
|
||||
self.config["hg_server"],
|
||||
"{}/raw-file/{}/{}/locales/shipped-locales".format(
|
||||
self.config["repo_path"],
|
||||
self.config["to_revision"],
|
||||
self.config["app_name"],
|
||||
),
|
||||
)
|
||||
to_shipped_locales = self._retry_download(to_shipped_locales_url, "WARNING").read().strip()
|
||||
to_locales = set(getPlatformLocales(to_shipped_locales, self.config["platform"]))
|
||||
|
||||
completes_only_index = 0
|
||||
for fromVersion in reversed(sorted(self.update_paths, key=LooseVersion)):
|
||||
from_ = self.update_paths[fromVersion]
|
||||
locales = sorted(list(set(from_["locales"]).intersection(to_locales)))
|
||||
appVersion = from_["appVersion"]
|
||||
build_id = from_["buildID"]
|
||||
mar_channel_IDs = from_.get('marChannelIds')
|
||||
|
||||
# Use new build targets for Windows, but only on compatible
|
||||
# versions (42+). See bug 1185456 for additional context.
|
||||
if self.config["platform"] not in ("win32", "win64") or \
|
||||
LooseVersion(fromVersion) < LooseVersion("42.0"):
|
||||
update_platform = ftp2updatePlatforms(self.config["platform"])[0]
|
||||
else:
|
||||
update_platform = ftp2updatePlatforms(self.config["platform"])[1]
|
||||
|
||||
release_dir = getReleasesDir(
|
||||
self.config["stage_product"], fromVersion
|
||||
)
|
||||
path_ = getReleaseInstallerPath(
|
||||
self.config["product"], self.config["product"].title(),
|
||||
fromVersion, self.config["platform"], locale="%locale%",
|
||||
)
|
||||
from_path = "{}/{}".format(release_dir, path_)
|
||||
|
||||
updater_package = "{}/{}".format(
|
||||
release_dir,
|
||||
getReleaseInstallerPath(
|
||||
self.config["product"], self.config["product"].title(),
|
||||
fromVersion, self.config["updater_platform"],
|
||||
locale="%locale%",
|
||||
)
|
||||
)
|
||||
|
||||
# Exclude locales being full checked
|
||||
quick_check_locales = [l for l in locales
|
||||
if l not in self.config["full_check_locales"]]
|
||||
# Get the intersection of from and to full_check_locales
|
||||
this_full_check_locales = [l for l in self.config["full_check_locales"]
|
||||
if l in locales]
|
||||
|
||||
if fromVersion in self.config["partial_versions"]:
|
||||
self.info("Generating configs for partial update checks for %s" % fromVersion)
|
||||
self.update_verify_config.addRelease(
|
||||
release=appVersion, build_id=build_id, locales=locales,
|
||||
patch_types=["complete", "partial"], from_path=from_path,
|
||||
ftp_server_from=self.config["previous_archive_prefix"],
|
||||
ftp_server_to=self.config["archive_prefix"],
|
||||
mar_channel_IDs=mar_channel_IDs, platform=update_platform,
|
||||
updater_package=updater_package
|
||||
)
|
||||
else:
|
||||
if this_full_check_locales and is_triangualar(completes_only_index):
|
||||
self.info("Generating full check configs for %s" % fromVersion)
|
||||
self.update_verify_config.addRelease(
|
||||
release=appVersion, build_id=build_id, locales=this_full_check_locales,
|
||||
from_path=from_path,
|
||||
ftp_server_from=self.config["previous_archive_prefix"],
|
||||
ftp_server_to=self.config["archive_prefix"],
|
||||
mar_channel_IDs=mar_channel_IDs, platform=update_platform,
|
||||
updater_package=updater_package
|
||||
)
|
||||
# Quick test for other locales, no download
|
||||
if len(quick_check_locales) > 0:
|
||||
self.info("Generating quick check configs for %s" % fromVersion)
|
||||
if not is_triangualar(completes_only_index):
|
||||
# Assuming we skipped full check locales, using all locales
|
||||
_locales = locales
|
||||
else:
|
||||
# Excluding full check locales from the quick check
|
||||
_locales = quick_check_locales
|
||||
self.update_verify_config.addRelease(
|
||||
release=appVersion, build_id=build_id,
|
||||
locales=_locales, platform=update_platform
|
||||
)
|
||||
completes_only_index += 1
|
||||
|
||||
def write_config(self):
|
||||
with open(self.config["output_file"], "w+") as fh:
|
||||
self.update_verify_config.write(fh)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
UpdateVerifyConfigCreator().run_and_exit()
|
||||
Загрузка…
Ссылка в новой задаче