Bug 1595368: [firefox-ci] Use consitent logic to determine tooltool url; r=sfraser a=task-fix

Differential Revision: https://phabricator.services.mozilla.com/D52498

Depends on D52497

--HG--
extra : histedit_source : 4b7e40193155303d8757f08aabe670e4dd127924
This commit is contained in:
Tom Prince 2019-11-11 05:50:11 +01:00
Родитель ad179105ef
Коммит 069263ab46
33 изменённых файлов: 28 добавлений и 128 удалений

Просмотреть файл

@ -27,7 +27,6 @@ mk_export_correct_style VC_PATH
if ! which mozmake 2>/dev/null; then
export PATH="$PATH:$SOURCE/.."
if ! which mozmake 2>/dev/null; then
TT_SERVER=${TT_SERVER:-https://tooltool.mozilla-releng.net/}
( cd $SOURCE/..; $SOURCE/mach artifact toolchain -v --tooltool-manifest $SOURCE/browser/config/tooltool-manifests/${platform:-win32}/releng.manifest --tooltool-url $TT_SERVER --retry 4${TOOLTOOL_CACHE:+ --cache-dir ${TOOLTOOL_CACHE}}${MOZ_TOOLCHAINS:+ ${MOZ_TOOLCHAINS}})
( cd $SOURCE/..; $SOURCE/mach artifact toolchain -v --tooltool-manifest $SOURCE/browser/config/tooltool-manifests/${platform:-win32}/releng.manifest --retry 4${TOOLTOOL_CACHE:+ --cache-dir ${TOOLTOOL_CACHE}}${MOZ_TOOLCHAINS:+ ${MOZ_TOOLCHAINS}})
fi
fi

Просмотреть файл

@ -1279,14 +1279,17 @@ def main(argv, _skip_logging=False):
(options_obj, args) = parser.parse_args(argv[1:])
# default the options list if not provided
if not options_obj.base_url:
options_obj.base_url = ['https://tooltool.mozilla-releng.net/']
tooltool_host = os.environ.get('TOOLTOOL_HOST', 'tooltool.mozilla-releng.net')
taskcluster_proxy_url = os.environ.get('TASKCLUSTER_PROXY_URL')
if taskcluster_proxy_url:
tooltool_url = '{}/{}'.format(taskcluster_proxy_url, tooltool_host)
else:
tooltool_url = 'https://{}'.format(tooltool_host)
# ensure all URLs have a trailing slash
def add_slash(url):
return url if url.endswith('/') else (url + '/')
options_obj.base_url = [add_slash(u) for u in options_obj.base_url]
options_obj.base_url = [add_slash(tooltool_url)]
# expand ~ in --authentication-file
if options_obj.auth_file:

Просмотреть файл

@ -165,8 +165,6 @@ class PackageFrontend(MachCommandBase):
help='Explicit tooltool manifest to process')
@CommandArgument('--authentication-file', metavar='FILE',
help='Use the RelengAPI token found in the given file to authenticate')
@CommandArgument('--tooltool-url', metavar='URL',
help='Use the given url as tooltool server')
@CommandArgument('--no-unpack', action='store_true',
help='Do not unpack any downloaded file')
@CommandArgument('--retry', type=int, default=4,
@ -179,7 +177,7 @@ class PackageFrontend(MachCommandBase):
def artifact_toolchain(self, verbose=False, cache_dir=None,
skip_cache=False, from_build=(),
tooltool_manifest=None, authentication_file=None,
tooltool_url=None, no_unpack=False, retry=None,
no_unpack=False, retry=None,
artifact_manifest=None, files=()):
'''Download, cache and install pre-built toolchains.
'''
@ -189,7 +187,6 @@ class PackageFrontend(MachCommandBase):
open_manifest,
unpack_file,
)
from requests.adapters import HTTPAdapter
import redo
import requests
@ -213,26 +210,16 @@ class PackageFrontend(MachCommandBase):
if not cache_dir:
cache_dir = os.path.join(self._mach_context.state_dir, 'toolchains')
tooltool_url = (tooltool_url or
'https://tooltool.mozilla-releng.net').rstrip('/')
tooltool_host = os.environ.get('TOOLTOOL_HOST', 'tooltool.mozilla-releng.net')
taskcluster_proxy_url = os.environ.get('TASKCLUSTER_PROXY_URL')
if taskcluster_proxy_url:
tooltool_url = '{}/{}'.format(taskcluster_proxy_url, tooltool_host)
else:
tooltool_url = 'https://{}'.format(tooltool_host)
cache = ArtifactCache(cache_dir=cache_dir, log=self.log,
skip_cache=skip_cache)
if authentication_file:
with open(authentication_file, 'rb') as f:
token = f.read().strip()
class TooltoolAuthenticator(HTTPAdapter):
def send(self, request, *args, **kwargs):
request.headers['Authorization'] = \
'Bearer {}'.format(token)
return super(TooltoolAuthenticator, self).send(
request, *args, **kwargs)
cache._download_manager.session.mount(
tooltool_url, TooltoolAuthenticator())
class DownloadRecord(FileRecord):
def __init__(self, url, *args, **kwargs):
super(DownloadRecord, self).__init__(*args, **kwargs)

Просмотреть файл

@ -7,9 +7,6 @@ job-defaults:
using: mozharness
use-caches: false
tooltool-downloads: internal
extra-config:
"tooltool_servers": ['http://taskcluster/tooltool.mozilla-releng.net/']
'tooltool_url': 'http://taskcluster/tooltool.mozilla-releng.net/'
win32/debug:
description: "Win32 Debug"

Просмотреть файл

@ -149,8 +149,6 @@ jobs:
tooltool-downloads: internal
extra-config:
stage_platform: win32
"tooltool_servers": ['http://taskcluster/tooltool.mozilla-releng.net/']
'tooltool_url': 'http://taskcluster/tooltool.mozilla-releng.net/'
fetches:
toolchain:
- win64-clang-cl
@ -183,8 +181,6 @@ jobs:
tooltool-downloads: internal
extra-config:
stage_platform: win64
"tooltool_servers": ['http://taskcluster/tooltool.mozilla-releng.net/']
'tooltool_url': 'http://taskcluster/tooltool.mozilla-releng.net/'
fetches:
toolchain:
- win64-clang-cl

Просмотреть файл

@ -120,9 +120,6 @@ jobs:
- builds/taskcluster_base_win64.py
- builds/taskcluster_sub_win64/searchfox_debug.py
tooltool-downloads: internal
extra-config:
"tooltool_servers": ['http://taskcluster/tooltool.mozilla-releng.net/']
'tooltool_url': 'http://taskcluster/tooltool.mozilla-releng.net/'
fetches:
toolchain:
- win64-clang-cl

Просмотреть файл

@ -84,8 +84,6 @@ jobs:
tooltool-downloads: internal
extra-config:
mozconfig_variant: debug
"tooltool_servers": ['http://taskcluster/tooltool.mozilla-releng.net/']
'tooltool_url': 'http://taskcluster/tooltool.mozilla-releng.net/'
fetches:
toolchain:
- win64-clang-cl

Просмотреть файл

@ -2,7 +2,6 @@
set -x
TOOLTOOL_SERVER=${TOOLTOOL_SERVER:-https://tooltool.mozilla-releng.net/}
SPIDERMONKEY_VARIANT=${SPIDERMONKEY_VARIANT:-plain}
UPLOAD_DIR=${UPLOAD_DIR:-$HOME/artifacts/}
WORK=${WORK:-$HOME/workspace}
@ -57,7 +56,7 @@ fi
# manifests.
BROWSER_PLATFORM=$PLATFORM_OS$BITS
(cd $TOOLTOOL_CHECKOUT && ${SRCDIR}/mach artifact toolchain${TOOLTOOL_MANIFEST:+ -v $TOOLTOOL_AUTH_FLAGS --tooltool-url $TOOLTOOL_SERVER --tooltool-manifest $SRCDIR/$TOOLTOOL_MANIFEST}${TOOLTOOL_CACHE:+ --cache-dir $TOOLTOOL_CACHE}${MOZ_TOOLCHAINS:+ ${MOZ_TOOLCHAINS}})
(cd $TOOLTOOL_CHECKOUT && ${SRCDIR}/mach artifact toolchain${TOOLTOOL_MANIFEST:+ -v $TOOLTOOL_AUTH_FLAGS --tooltool-manifest $SRCDIR/$TOOLTOOL_MANIFEST}${TOOLTOOL_CACHE:+ --cache-dir $TOOLTOOL_CACHE}${MOZ_TOOLCHAINS:+ ${MOZ_TOOLCHAINS}})
) || exit 1 # end of set -e scope

Просмотреть файл

@ -18,11 +18,6 @@ if [ -e "$TOOLTOOL_AUTH_FILE" ]; then
TOOLTOOL_DL_FLAGS="${TOOLTOOL_DL_FLAGS=} --authentication-file=$TOOLTOOL_AUTH_FILE"
fi
if [ -n "$TASKCLUSTER_PROXY_URL" ]; then
# When the worker has the relengapi proxy setup, use it.
TOOLTOOL_DL_FLAGS="${TOOLTOOL_DL_FLAGS=} --tooltool-url=${TASKCLUSTER_PROXY_URL}/tooltool.mozilla-releng.net/"
fi
if [ -n "$UPLOAD_DIR" ]; then
TOOLTOOL_DL_FLAGS="${TOOLTOOL_DL_FLAGS=} --artifact-manifest $UPLOAD_DIR/toolchains.json"
fi

Просмотреть файл

@ -43,7 +43,6 @@ config = {
'run-tests',
],
"tooltool_cache": os.environ.get("TOOLTOOL_CACHE"),
"tooltool_servers": ['http://taskcluster/tooltool.mozilla-releng.net/'],
"hostutils_manifest_path": "testing/config/tooltool-manifests/linux64/hostutils.manifest",
"avds_dir": "/builds/worker/workspace/build/.android",
# "log_format": "%(levelname)8s - %(message)s",

Просмотреть файл

@ -22,7 +22,6 @@ config = {
# from android_common.py
"download_tooltool": True,
"minidump_stackwalk_path": "linux64-minidump_stackwalk",
"tooltool_servers": ['https://tooltool.mozilla-releng.net/'],
"minidump_tooltool_manifest_path": "config/tooltool-manifests/linux64/releng.manifest",
"xpcshell_extra": "--remoteTestRoot=/data/local/tests",
}

Просмотреть файл

@ -5,7 +5,6 @@
config = {
"tooltool_manifest_path": "testing/config/tooltool-manifests/androidx86_7_0/releng.manifest",
"tooltool_servers": ['http://taskcluster/tooltool.mozilla-releng.net/'],
"emulator_manifest": """
[
{

Просмотреть файл

@ -3,9 +3,6 @@
config = {
"taskcluster": {
# use the relengapi proxy to talk to tooltool
"tooltool_servers": ['http://taskcluster/tooltool.mozilla-releng.net/'],
"tooltool_url": 'http://taskcluster/tooltool.mozilla-releng.net/',
'upload_env': {
'UPLOAD_PATH': '/builds/worker/artifacts',
},

Просмотреть файл

@ -32,6 +32,4 @@ config = {
'PATH': ('{MOZ_FETCHES_DIR}/clang/bin/:%(PATH)s'
.format(MOZ_FETCHES_DIR=os.environ['MOZ_FETCHES_DIR'])),
},
"tooltool_servers": ['http://taskcluster/tooltool.mozilla-releng.net/'],
"tooltool_url": 'http://taskcluster/tooltool.mozilla-releng.net/',
}

Просмотреть файл

@ -15,7 +15,6 @@ config = {
"tooltool_cache": "/builds/tooltool_cache",
"download_tooltool": True,
"minidump_stackwalk_path": "linux64-minidump_stackwalk",
"tooltool_servers": ['https://tooltool.mozilla-releng.net/'],
"minidump_tooltool_manifest_path": "config/tooltool-manifests/linux64/releng.manifest",
"hostutils_manifest_path": "testing/config/tooltool-manifests/linux64/hostutils.manifest",
}

Просмотреть файл

@ -6,7 +6,6 @@ config = {
"locale": os.environ.get("LOCALE"),
# ToolTool
"tooltool_url": 'http://taskcluster/tooltool.mozilla-releng.net/',
'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'),
'run_configure': False,

Просмотреть файл

@ -6,7 +6,6 @@ config = {
"locale": os.environ.get("LOCALE"),
# ToolTool
"tooltool_url": 'http://taskcluster/tooltool.mozilla-releng.net/',
'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'),
'run_configure': False,

Просмотреть файл

@ -6,6 +6,5 @@ config = {
"repack_id": os.environ.get("REPACK_ID"),
# ToolTool
"tooltool_url": 'http://taskcluster/tooltool.mozilla-releng.net/',
'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'),
}

Просмотреть файл

@ -6,6 +6,5 @@ config = {
"locale": os.environ.get("LOCALE"),
# ToolTool
"tooltool_url": 'http://taskcluster/tooltool.mozilla-releng.net/',
'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'),
}

Просмотреть файл

@ -5,7 +5,6 @@ platform = "win32"
config = {
"repack_id": os.environ.get("REPACK_ID"),
'tooltool_url': 'https://tooltool.mozilla-releng.net/',
'run_configure': False,
'env': {

Просмотреть файл

@ -5,7 +5,6 @@ platform = "win64"
config = {
"repack_id": os.environ.get("REPACK_ID"),
'tooltool_url': 'https://tooltool.mozilla-releng.net/',
'run_configure': False,
'env': {

Просмотреть файл

@ -15,7 +15,6 @@ config = {
"manifest": "mobile/android/config/tooltool-manifests/android/releng.manifest",
"output_dir": "%(abs_work_dir)s/src",
},
"tooltool_servers": ['http://taskcluster/tooltool.mozilla-releng.net/'],
"upload_env": {
'UPLOAD_PATH': '/builds/worker/artifacts/',

Просмотреть файл

@ -13,8 +13,6 @@ config = {
'UPLOAD_PATH': '/builds/worker/artifacts/',
},
"tooltool_url": 'http://taskcluster/tooltool.mozilla-releng.net/',
"vcs_share_base": "/builds/hg-shared",
}

Просмотреть файл

@ -15,7 +15,6 @@ config = {
'UPLOAD_PATH': os.path.join(os.getcwd(), 'public', 'build'),
},
"tooltool_url": 'https://tooltool.mozilla-releng.net/',
'tooltool_manifest_src': "browser/config/tooltool-manifests/win32/releng.manifest",
# use mozmake?

Просмотреть файл

@ -15,7 +15,6 @@ config = {
'UPLOAD_PATH': os.path.join(os.getcwd(), 'public', 'build'),
},
"tooltool_url": 'https://tooltool.mozilla-releng.net/',
'tooltool_manifest_src': "browser/config/tooltool-manifests/win64/releng.manifest",
# use mozmake?

Просмотреть файл

@ -25,5 +25,4 @@ config = {
"minidump_stackwalk_path": "linux64-minidump_stackwalk",
"per_test_category": "web-platform",
"tooltool_cache": os.environ.get("TOOLTOOL_CACHE"),
"tooltool_servers": ['http://taskcluster/tooltool.mozilla-releng.net/'],
}

Просмотреть файл

@ -1278,15 +1278,18 @@ def main(argv, _skip_logging=False):
(options_obj, args) = parser.parse_args(argv[1:])
# default the options list if not provided
if not options_obj.base_url:
options_obj.base_url = ['https://tooltool.mozilla-releng.net/']
tooltool_host = os.environ.get('TOOLTOOL_HOST', 'tooltool.mozilla-releng.net')
taskcluster_proxy_url = os.environ.get('TASKCLUSTER_PROXY_URL')
if taskcluster_proxy_url:
tooltool_url = '{}/{}'.format(taskcluster_proxy_url, tooltool_host)
else:
tooltool_url = 'https://{}'.format(tooltool_host)
# ensure all URLs have a trailing slash
def add_slash(url):
return url if url.endswith('/') else (url + '/')
options_obj.base_url = [add_slash(u) for u in options_obj.base_url]
options_obj.base_url = [add_slash(tooltool_url)]
# expand ~ in --authentication-file
if options_obj.auth_file:
options_obj.auth_file = os.path.expanduser(options_obj.auth_file)

Просмотреть файл

@ -596,33 +596,6 @@ items from that key's value."
self.info("Both --dump-config and --dump-config-hierarchy don't "
"actually run any actions.")
def _assert_cfg_valid_for_action(self, dependencies, action):
""" assert dependency keys are in config for given action.
Takes a list of dependencies and ensures that each have an
assoctiated key in the config. Displays error messages as
appropriate.
"""
# TODO add type and value checking, not just keys
# TODO solution should adhere to: bug 699343
# TODO add this to BaseScript when the above is done
# for now, let's just use this as a way to save typing...
c = self.config
undetermined_keys = []
err_template = "The key '%s' could not be determined \
and is needed for the action '%s'. Please add this to your config \
or run without that action (ie: --no-{action})"
for dep in dependencies:
if dep not in c:
undetermined_keys.append(dep)
if undetermined_keys:
fatal_msgs = [err_template % (key, action)
for key in undetermined_keys]
self.fatal("".join(fatal_msgs))
# otherwise:
return # all good
def _query_build_prop_from_app_ini(self, prop, app_ini_path=None):
dirs = self.query_abs_dirs()
print_conf_setting_path = os.path.join(dirs['abs_src_dir'],
@ -781,10 +754,6 @@ or run without that action (ie: --no-{action})"
env = self.query_build_env()
env.update(self.query_mach_build_env())
self._assert_cfg_valid_for_action(
['tooltool_url'],
'build'
)
c = self.config
dirs = self.query_abs_dirs()
toolchains = os.environ.get('MOZ_TOOLCHAINS')
@ -807,8 +776,6 @@ or run without that action (ie: --no-{action})"
cmd.extend([
'--tooltool-manifest',
os.path.join(dirs['abs_src_dir'], manifest_src),
'--tooltool-url',
c['tooltool_url'],
])
auth_file = self._get_tooltool_auth_file()
if auth_file:

Просмотреть файл

@ -10,10 +10,6 @@ TooltoolErrorList = PythonErrorList + [{
}]
TOOLTOOL_SERVERS = [
'https://tooltool.mozilla-releng.net/',
]
_here = os.path.abspath(os.path.dirname(__file__))
_external_tools_path = os.path.normpath(os.path.join(_here, '..', '..',
'external_tools'))
@ -21,8 +17,8 @@ _external_tools_path = os.path.normpath(os.path.join(_here, '..', '..',
class TooltoolMixin(object):
"""Mixin class for handling tooltool manifests.
To use a tooltool server other than the Mozilla server, override
config['tooltool_servers']. To specify a different authentication
To use a tooltool server other than the Mozilla server, set
TOOLTOOL_HOST in the environment. To specify a different authentication
file than that used in releng automation,override
config['tooltool_authentication_file']; set it to None to not pass
any authentication information (OK for public files)
@ -67,17 +63,6 @@ class TooltoolMixin(object):
os.path.join(_external_tools_path, 'tooltool.py'),
]
# get the tooltool servers from configuration
default_urls = self.config.get('tooltool_servers', TOOLTOOL_SERVERS)
# add slashes (bug 1155630)
def add_slash(url):
return url if url.endswith('/') else (url + '/')
default_urls = [add_slash(u) for u in default_urls]
for url in default_urls:
cmd.extend(['--tooltool-url' if self.topsrcdir else '--url', url])
# handle authentication file, if given
auth_file = self._get_auth_file()
if auth_file and os.path.exists(auth_file):

Просмотреть файл

@ -465,8 +465,6 @@ class DesktopSingleLocale(LocalesMixin, AutomationMixin,
cmd.extend([
'--tooltool-manifest',
os.path.join(dirs['abs_mozilla_dir'], manifest_src),
'--tooltool-url',
config['tooltool_url'],
])
auth_file = self._get_tooltool_auth_file()
if auth_file and os.path.exists(auth_file):

Просмотреть файл

@ -52,7 +52,6 @@ class FxDesktopBuild(BuildScript, TryToolsMixin, object):
# jobs have a minimal `hg pull`.
"clone_upstream_url": "https://hg.mozilla.org/mozilla-unified",
"repo_base": "https://hg.mozilla.org",
'tooltool_url': 'https://tooltool.mozilla-releng.net/',
"graph_selector": "/server/collect.cgi",
# only used for make uploadsymbols
'old_packages': [

Просмотреть файл

@ -120,8 +120,6 @@ class Repackage(BaseScript):
cmd.extend([
'--tooltool-manifest',
os.path.join(dirs['abs_mozilla_dir'], manifest_src),
'--tooltool-url',
config['tooltool_url'],
])
auth_file = self._get_tooltool_auth_file()
if auth_file:

Просмотреть файл

@ -56,10 +56,10 @@ tar cvz --exclude=eslint-plugin-mozilla --exclude=eslint-plugin-spidermonkey-js
echo "Adding eslint.tar.gz to tooltool..."
rm tools/lint/eslint/manifest.tt
./python/mozbuild/mozbuild/action/tooltool.py add --visibility public --unpack eslint.tar.gz --url="https://tooltool.mozilla-releng.net/"
./python/mozbuild/mozbuild/action/tooltool.py add --visibility public --unpack eslint.tar.gz
echo "Uploading eslint.tar.gz to tooltool..."
./python/mozbuild/mozbuild/action/tooltool.py upload --authentication-file=~/.tooltool-token --message "node_modules folder update for tools/lint/eslint" --url="https://tooltool.mozilla-releng.net/"
./python/mozbuild/mozbuild/action/tooltool.py upload --authentication-file=~/.tooltool-token --message "node_modules folder update for tools/lint/eslint"
echo "Cleaning up..."
mv manifest.tt tools/lint/eslint/manifest.tt