From c152ccec1d2916599074c79fa52552a38247b24e Mon Sep 17 00:00:00 2001 From: "arthur.iakab" Date: Thu, 31 Jan 2019 23:14:11 +0200 Subject: [PATCH] Backed out 4 changesets (bug 1508381) for multiple Windows build bustages CLOSED TREE Backed out changeset f01cec6f712e (bug 1508381) Backed out changeset ba69e59924de (bug 1508381) Backed out changeset 97fe4e5a665e (bug 1508381) Backed out changeset 0c3065c12bef (bug 1508381) --- .../mozrelease/mozrelease/buglist_creator.py | 8 +- .../docker/funsize-update-generator/runme.sh | 3 +- taskcluster/docker/periodic-updates/runme.sh | 3 +- taskcluster/docker/pipfile-updates/runme.sh | 3 +- taskcluster/scripts/misc/fetch-content | 3 +- taskcluster/taskgraph/transforms/task.py | 24 ++ taskcluster/taskgraph/util/taskcluster.py | 45 ++-- third_party/python/requirements.in | 2 +- third_party/python/requirements.txt | 8 +- third_party/python/taskcluster/PKG-INFO | 2 +- third_party/python/taskcluster/README.md | 243 +++++------------- third_party/python/taskcluster/setup.py | 3 +- .../taskcluster/_client_importer.py | 1 - .../taskcluster/aio/_client_importer.py | 1 - .../taskcluster/aio/asyncclient.py | 24 +- .../taskcluster/taskcluster/aio/asyncutils.py | 2 +- .../taskcluster/taskcluster/aio/auth.py | 3 +- .../taskcluster/taskcluster/aio/authevents.py | 10 +- .../taskcluster/aio/awsprovisioner.py | 3 +- .../taskcluster/aio/awsprovisionerevents.py | 3 +- .../taskcluster/taskcluster/aio/ec2manager.py | 93 ++++--- .../taskcluster/taskcluster/aio/github.py | 8 +- .../taskcluster/aio/githubevents.py | 41 +-- .../taskcluster/taskcluster/aio/hooks.py | 5 +- .../taskcluster/taskcluster/aio/index.py | 3 +- .../taskcluster/taskcluster/aio/login.py | 33 ++- .../taskcluster/taskcluster/aio/notify.py | 3 +- .../taskcluster/taskcluster/aio/pulse.py | 74 +++--- .../taskcluster/taskcluster/aio/purgecache.py | 6 +- .../taskcluster/aio/purgecacheevents.py | 4 +- .../taskcluster/taskcluster/aio/queue.py | 3 +- .../taskcluster/aio/queueevents.py | 4 +- .../taskcluster/taskcluster/aio/secrets.py | 3 +- .../taskcluster/aio/treeherderevents.py | 8 +- .../python/taskcluster/taskcluster/auth.py | 3 +- .../taskcluster/taskcluster/authevents.py | 10 +- .../taskcluster/taskcluster/awsprovisioner.py | 3 +- .../taskcluster/awsprovisionerevents.py | 3 +- .../python/taskcluster/taskcluster/client.py | 51 ++-- .../taskcluster/taskcluster/ec2manager.py | 93 ++++--- .../python/taskcluster/taskcluster/github.py | 8 +- .../taskcluster/taskcluster/githubevents.py | 41 +-- .../python/taskcluster/taskcluster/hooks.py | 5 +- .../python/taskcluster/taskcluster/index.py | 3 +- .../python/taskcluster/taskcluster/login.py | 33 ++- .../python/taskcluster/taskcluster/notify.py | 3 +- .../python/taskcluster/taskcluster/pulse.py | 74 +++--- .../taskcluster/taskcluster/purgecache.py | 6 +- .../taskcluster/purgecacheevents.py | 4 +- .../python/taskcluster/taskcluster/queue.py | 3 +- .../taskcluster/taskcluster/queueevents.py | 4 +- .../python/taskcluster/taskcluster/secrets.py | 3 +- .../taskcluster/treeherderevents.py | 8 +- .../python/taskcluster/taskcluster/utils.py | 105 ++++++-- .../python/taskcluster/test/test_async.py | 6 +- .../python/taskcluster/test/test_client.py | 187 ++++++-------- .../python/taskcluster/test/test_utils.py | 99 ------- 57 files changed, 595 insertions(+), 842 deletions(-) diff --git a/python/mozrelease/mozrelease/buglist_creator.py b/python/mozrelease/mozrelease/buglist_creator.py index 135af3168ff3..a88de290ffc3 100644 --- a/python/mozrelease/mozrelease/buglist_creator.py +++ b/python/mozrelease/mozrelease/buglist_creator.py @@ -10,8 +10,6 @@ import os import re import requests from taskcluster.notify import Notify -from taskcluster import optionsFromEnvironment -from taskgraph.util.taskcluster import get_root_url from operator import itemgetter from mozilla_version.gecko import GeckoVersion @@ -218,9 +216,11 @@ Task group: [{task_group_id}](https://tools.taskcluster.net/groups/{task_group_i subject = '{} Build of {} {} build {}'.format(subject_prefix, product, version, build_number) - notify_options = optionsFromEnvironment({'rootUrl': get_root_url()}) + notify_options = {} if 'TASKCLUSTER_PROXY_URL' in os.environ: - notify_options['rootUrl'] = os.environ['TASKCLUSTER_PROXY_URL'] + # Until bug 1460015 is finished, use the old baseUrl style of proxy URL + base_url = os.environ['TASKCLUSTER_PROXY_URL'].rstrip('/') + notify_options['baseUrl'] = '{}/notify/v1'.format(base_url) notify = Notify(notify_options) for address in addresses: notify.email({ diff --git a/taskcluster/docker/funsize-update-generator/runme.sh b/taskcluster/docker/funsize-update-generator/runme.sh index 0810c008b9d6..d2a17d990408 100644 --- a/taskcluster/docker/funsize-update-generator/runme.sh +++ b/taskcluster/docker/funsize-update-generator/runme.sh @@ -33,7 +33,8 @@ then test "${AWS_BUCKET_NAME}" set +x # Don't echo these. - secret_url="${TASKCLUSTER_PROXY_URL}/api/auth/v1/aws/s3/read-write/${AWS_BUCKET_NAME}/${S3_PATH}" + # Until bug 1460015 is finished, use baseUrl-style proxy URLs + secret_url="${TASKCLUSTER_PROXY_URL}/auth/v1/aws/s3/read-write/${AWS_BUCKET_NAME}/${S3_PATH}" AUTH=$(curl "${secret_url}") AWS_ACCESS_KEY_ID=$(echo "${AUTH}" | jq -r '.credentials.accessKeyId') AWS_SECRET_ACCESS_KEY=$(echo "${AUTH}" | jq -r '.credentials.secretAccessKey') diff --git a/taskcluster/docker/periodic-updates/runme.sh b/taskcluster/docker/periodic-updates/runme.sh index ecdb5169ebfd..c6760377f7e0 100755 --- a/taskcluster/docker/periodic-updates/runme.sh +++ b/taskcluster/docker/periodic-updates/runme.sh @@ -68,7 +68,8 @@ fi if [ -n "${ARC_SECRET}" ] && getent hosts taskcluster then set +x # Don't echo these - secrets_url="${TASKCLUSTER_PROXY_URL}/api/secrets/v1/secret/${ARC_SECRET}" + # Until bug 1460015 is finished, use baseUrl-style proxy URLs + secrets_url="${TASKCLUSTER_PROXY_URL}/secrets/v1/secret/${ARC_SECRET}" SECRET=$(curl "${secrets_url}") TOKEN=$(echo "${SECRET}" | jq -r '.secret.token') elif [ -n "${ARC_TOKEN}" ] # Allow for local testing. diff --git a/taskcluster/docker/pipfile-updates/runme.sh b/taskcluster/docker/pipfile-updates/runme.sh index 3a80751e009b..de21737c02a2 100755 --- a/taskcluster/docker/pipfile-updates/runme.sh +++ b/taskcluster/docker/pipfile-updates/runme.sh @@ -36,7 +36,8 @@ fi if [ -n "${ARC_SECRET}" ] && getent hosts taskcluster then set +x # Don't echo these - secrets_url="${TASKCLUSTER_PROXY_URL}/api/secrets/v1/secret/${ARC_SECRET}" + # Until bug 1460015 is finished, use the old, baseUrl-style proxy URLs + secrets_url="${TASKCLUSTER_PROXY_URL}/secrets/v1/secret/${ARC_SECRET}" SECRET=$(curl "${secrets_url}") TOKEN=$(echo "${SECRET}" | jq -r '.secret.token') elif [ -n "${ARC_TOKEN}" ] # Allow for local testing. diff --git a/taskcluster/scripts/misc/fetch-content b/taskcluster/scripts/misc/fetch-content index 00248ca58ed7..735447155725 100755 --- a/taskcluster/scripts/misc/fetch-content +++ b/taskcluster/scripts/misc/fetch-content @@ -396,7 +396,8 @@ def command_task_artifacts(args): task=fetch['task'], artifact=fetch['artifact']) url = api(root_url, 'queue', 'v1', path) else: - url = ('{proxy_url}/api/queue/v1/task/{task}/artifacts/{artifact}').format( + # Until bug 1460015 is finished, use the old baseUrl style proxy URLs + url = ('{proxy_url}/queue/v1/task/{task}/artifacts/{artifact}').format( proxy_url=os.environ['TASKCLUSTER_PROXY_URL'], task=fetch['task'], artifact=fetch['artifact']) diff --git a/taskcluster/taskgraph/transforms/task.py b/taskcluster/taskgraph/transforms/task.py index f5c455d94e73..412360486ad5 100644 --- a/taskcluster/taskgraph/transforms/task.py +++ b/taskcluster/taskgraph/transforms/task.py @@ -21,6 +21,7 @@ from taskgraph.util.attributes import TRUNK_PROJECTS from taskgraph.util.hash import hash_path from taskgraph.util.treeherder import split_symbol from taskgraph.transforms.base import TransformSequence +from taskgraph.util.taskcluster import get_root_url from taskgraph.util.schema import ( validate_schema, Schema, @@ -492,6 +493,11 @@ def build_docker_worker_payload(config, task, task_def): else: raise Exception("unknown docker image type") + # propagate our TASKCLUSTER_ROOT_URL to the task; note that this will soon + # be provided directly by the worker, making this redundant: + # https://bugzilla.mozilla.org/show_bug.cgi?id=1460015 + worker['env']['TASKCLUSTER_ROOT_URL'] = get_root_url() + features = {} if worker.get('relengapi-proxy'): @@ -499,6 +505,7 @@ def build_docker_worker_payload(config, task, task_def): if worker.get('taskcluster-proxy'): features['taskclusterProxy'] = True + worker['env']['TASKCLUSTER_PROXY_URL'] = 'http://taskcluster/' if worker.get('allow-ptrace'): features['allowPtrace'] = True @@ -523,6 +530,11 @@ def build_docker_worker_payload(config, task, task_def): else: worker['env']['SCCACHE_DISABLE'] = '1' + # this will soon be provided directly by the worker: + # https://bugzilla.mozilla.org/show_bug.cgi?id=1460015 + if features.get('taskclusterProxy'): + worker['env']['TASKCLUSTER_PROXY_URL'] = 'http://taskcluster' + capabilities = {} for lo in 'audio', 'video': @@ -755,6 +767,11 @@ def build_generic_worker_payload(config, task, task_def): env = worker.get('env', {}) + # propagate our TASKCLUSTER_ROOT_URL to the task; note that this will soon + # be provided directly by the worker, making this redundant: + # https://bugzilla.mozilla.org/show_bug.cgi?id=1460015 + env['TASKCLUSTER_ROOT_URL'] = get_root_url() + if task.get('needs-sccache'): env['USE_SCCACHE'] = '1' # Disable sccache idle shutdown. @@ -809,6 +826,9 @@ def build_generic_worker_payload(config, task, task_def): if worker.get('taskcluster-proxy'): features['taskclusterProxy'] = True + # this will soon be provided directly by the worker: + # https://bugzilla.mozilla.org/show_bug.cgi?id=1460015 + worker['env']['TASKCLUSTER_PROXY_URL'] = 'http://taskcluster' if worker.get('run-as-administrator', False): features['runAsAdministrator'] = True @@ -1307,6 +1327,10 @@ def build_always_optimized_payload(config, task, task_def): def build_macosx_engine_payload(config, task, task_def): worker = task['worker'] + # propagate our TASKCLUSTER_ROOT_URL to the task; note that this will soon + # be provided directly by the worker, making this redundant + worker.setdefault('env', {})['TASKCLUSTER_ROOT_URL'] = get_root_url() + artifacts = map(lambda artifact: { 'name': artifact['name'], 'path': artifact['path'], diff --git a/taskcluster/taskgraph/util/taskcluster.py b/taskcluster/taskgraph/util/taskcluster.py index ea0911750e0e..4212a30bab3e 100644 --- a/taskcluster/taskgraph/util/taskcluster.py +++ b/taskcluster/taskgraph/util/taskcluster.py @@ -31,23 +31,10 @@ CONCURRENCY = 50 @memoize -def get_root_url(use_proxy=False): +def get_root_url(): """Get the current TASKCLUSTER_ROOT_URL. When running in a task, this must come from $TASKCLUSTER_ROOT_URL; when run on the command line, we apply a - defualt that points to the production deployment of Taskcluster. If use_proxy - is set, this attempts to get TASKCLUSTER_PROXY_URL instead, failing if it - is not set.""" - if use_proxy: - try: - return os.environ['TASKCLUSTER_PROXY_URL'] - except KeyError: - if 'TASK_ID' not in os.environ: - raise RuntimeError( - 'taskcluster-proxy is not available when not executing in a task') - else: - raise RuntimeError( - 'taskcluster-proxy is not enabled for this task') - + defualt that points to the production deployment of Taskcluster.""" if 'TASKCLUSTER_ROOT_URL' not in os.environ: if 'TASK_ID' in os.environ: raise RuntimeError('$TASKCLUSTER_ROOT_URL must be set when running in a task') @@ -154,7 +141,11 @@ def get_artifact_path(task, path): def get_index_url(index_path, use_proxy=False, multiple=False): - index_tmpl = liburls.api(get_root_url(use_proxy), 'index', 'v1', 'task{}/{}') + if use_proxy: + # Until bug 1460015 is finished, use the old baseUrl style of proxy URL + index_tmpl = os.environ['TASKCLUSTER_PROXY_URL'] + '/index/v1/task{}/{}' + else: + index_tmpl = liburls.api(get_root_url(), 'index', 'v1', 'task{}/{}') return index_tmpl.format('s' if multiple else '', index_path) @@ -204,7 +195,11 @@ def parse_time(timestamp): def get_task_url(task_id, use_proxy=False): - task_tmpl = liburls.api(get_root_url(use_proxy), 'queue', 'v1', 'task/{}') + if use_proxy: + # Until bug 1460015 is finished, use the old baseUrl style of proxy URL + task_tmpl = os.environ['TASKCLUSTER_PROXY_URL'] + '/queue/v1/task/{}' + else: + task_tmpl = liburls.api(get_root_url(), 'queue', 'v1', 'task/{}') return task_tmpl.format(task_id) @@ -245,13 +240,17 @@ def rerun_task(task_id): def get_current_scopes(): """Get the current scopes. This only makes sense in a task with the Taskcluster proxy enabled, where it returns the actual scopes accorded to the task.""" - auth_url = liburls.api(get_root_url(use_proxy=True), 'auth', 'v1', 'scopes/current') - resp = _do_request(auth_url) + # Until bug 1460015 is finished, use the old baseUrl style of proxy URL + resp = _do_request(os.environ['TASKCLUSTER_PROXY_URL'] + '/auth/v1/scopes/current') return resp.json().get("scopes", []) def get_purge_cache_url(provisioner_id, worker_type, use_proxy=False): - url_tmpl = liburls.api(get_root_url(use_proxy), 'purge-cache', 'v1', 'purge-cache/{}/{}') + if use_proxy: + # Until bug 1460015 is finished, use the old baseUrl style of proxy URL + url_tmpl = os.environ['TASKCLUSTER_PROXY_URL'] + '/purge-cache/v1/purge-cache/{}/{}' + else: + url_tmpl = liburls.api(get_root_url(), 'purge-cache', 'v1', 'purge-cache/{}/{}') return url_tmpl.format(provisioner_id, worker_type) @@ -268,7 +267,11 @@ def purge_cache(provisioner_id, worker_type, cache_name, use_proxy=False): def send_email(address, subject, content, link, use_proxy=False): """Sends an email using the notify service""" logger.info('Sending email to {}.'.format(address)) - url = liburls.api(get_root_url(use_proxy), 'notify', 'v1', 'email') + if use_proxy: + # Until bug 1460015 is finished, use the old baseUrl style of proxy URL + url = os.environ['TASKCLUSTER_PROXY_URL'] + '/notify/v1/email' + else: + url = liburls.api(get_root_url(), 'notify', 'v1', 'email') _do_request(url, json={ 'address': address, 'subject': subject, diff --git a/third_party/python/requirements.in b/third_party/python/requirements.in index 29862cd3e129..427bbed65d84 100644 --- a/third_party/python/requirements.in +++ b/third_party/python/requirements.in @@ -12,7 +12,7 @@ python-hglib==2.4 redo==2.0.2 requests==2.9.1 six==1.10.0 -taskcluster==6.0.0 +taskcluster==4.0.1 taskcluster-urls==11.0.0 virtualenv==15.2.0 voluptuous==0.11.5 diff --git a/third_party/python/requirements.txt b/third_party/python/requirements.txt index 68fda00b0c87..78da7fe7cd5f 100644 --- a/third_party/python/requirements.txt +++ b/third_party/python/requirements.txt @@ -104,10 +104,10 @@ taskcluster-urls==11.0.0 \ --hash=sha256:18dcaa9c2412d34ff6c78faca33f0dd8f2384e3f00a98d5832c62d6d664741f0 \ --hash=sha256:2aceab7cf5b1948bc197f2e5e50c371aa48181ccd490b8bada00f1e3baf0c5cc \ --hash=sha256:74bd2110b5daaebcec5e1d287bf137b61cb8cf6b2d8f5f2b74183e32bc4e7c87 -taskcluster==6.0.0 \ - --hash=sha256:48ecd4898c7928deddfb34cb1cfe2b2505c68416e6c503f8a7f3dd0572425e96 \ - --hash=sha256:6d5cf7bdbc09dc48b2d376b418b95c1c157a2d359c4b6b231c1fb14a323c0cc5 \ - --hash=sha256:e409fce7a72808e4f87dc7baca7a79d8b64d5c5045264b9e197c120cc40e219b +taskcluster==4.0.1 \ + --hash=sha256:27256511044346ac71a495d3c636f2add95c102b9b09f90d6fb1ea3e9949d311 \ + --hash=sha256:99dd90bc1c566968868c8b07ede32f8e031cbccd52c7195a61e802679d461447 \ + --hash=sha256:d0360063c1a3fcaaa514bb31c03954ba573d2b671df40a2ecfdfd9339cc8e93e virtualenv-clone==0.3.0 \ --hash=sha256:4507071d81013fd03ea9930ec26bc8648b997927a11fa80e8ee81198b57e0ac7 \ --hash=sha256:b5cfe535d14dc68dfc1d1bb4ac1209ea28235b91156e2bba8e250d291c3fb4f8 \ diff --git a/third_party/python/taskcluster/PKG-INFO b/third_party/python/taskcluster/PKG-INFO index 9a20850cdcaa..4b46124fb017 100644 --- a/third_party/python/taskcluster/PKG-INFO +++ b/third_party/python/taskcluster/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: taskcluster -Version: 6.0.0 +Version: 4.0.1 Summary: Python client for Taskcluster Home-page: https://github.com/taskcluster/taskcluster-client.py Author: John Ford diff --git a/third_party/python/taskcluster/README.md b/third_party/python/taskcluster/README.md index 9a0cee7b4a5b..7bb951eff305 100644 --- a/third_party/python/taskcluster/README.md +++ b/third_party/python/taskcluster/README.md @@ -70,7 +70,7 @@ python2-compatible and operate synchronously. The objects under `taskcluster.aio` (e.g., `taskcluster.aio.Queue`) require -`python>=3.6`. The async objects use asyncio coroutines for concurrency; this +`python>=3.5`. The async objects use asyncio coroutines for concurrency; this allows us to put I/O operations in the background, so operations that require the cpu can happen sooner. Given dozens of operations that can run concurrently (e.g., cancelling a medium-to-large task graph), this can result in significant @@ -101,10 +101,7 @@ Here's a slide deck for an [introduction to async python](https://gitpitch.com/e ```python import taskcluster - index = taskcluster.Index({ - 'rootUrl': 'https://tc.example.com', - 'credentials': {'clientId': 'id', 'accessToken': 'accessToken'}, - }) + index = taskcluster.Index({'credentials': {'clientId': 'id', 'accessToken': 'accessToken'}}) index.ping() ``` @@ -122,34 +119,18 @@ Here's a slide deck for an [introduction to async python](https://gitpitch.com/e ```python from taskcluster import client - qEvt = client.QueueEvents({rootUrl: 'https://tc.example.com'}) + qEvt = client.QueueEvents() # The following calls are equivalent qEvt.taskCompleted({'taskId': 'atask'}) qEvt.taskCompleted(taskId='atask') ``` -## Root URL - -This client requires a `rootUrl` argument to identify the Taskcluster -deployment to talk to. As of this writing, the production cluster has rootUrl -`https://taskcluster.net`. - -## Environment Variables - -As of version 6.0.0, the client does not read the standard `TASKCLUSTER_…` -environment variables automatically. To fetch their values explicitly, use -`taskcluster.optionsFromEnvironment()`: - -```python -auth = taskcluster.Auth(taskcluster.optionsFromEnvironment()) -``` - ## Pagination There are two ways to accomplish pagination easily with the python client. The first is to implement pagination in your code: ```python import taskcluster -queue = taskcluster.Queue({'rootUrl': 'https://tc.example.com'}) +queue = taskcluster.Queue() i = 0 tasks = 0 outcome = queue.listTaskGroup('JzTGxwxhQ76_Tt1dxkaG5g') @@ -172,7 +153,7 @@ built and then counted: ```python import taskcluster -queue = taskcluster.Queue({'rootUrl': 'https://tc.example.com'}) +queue = taskcluster.Queue() responses = [] @@ -1030,9 +1011,9 @@ await asyncAuth.testAuthenticateGet() # -> result import taskcluster authEvents = taskcluster.AuthEvents(options) ``` -The auth service is responsible for storing credentials, managing -assignment of scopes, and validation of request signatures from other -services. +The auth service, typically available at `auth.taskcluster.net` +is responsible for storing credentials, managing assignment of scopes, +and validation of request signatures from other services. These exchanges provides notifications when credentials or roles are updated. This is mostly so that multiple instances of the auth service @@ -1507,23 +1488,11 @@ session = taskcluster.aio.createSession(loop=loop) asyncEC2Manager = taskcluster.aio.EC2Manager(options, session=session) ``` A taskcluster service which manages EC2 instances. This service does not understand any taskcluster concepts intrinsicaly other than using the name `workerType` to refer to a group of associated instances. Unless you are working on building a provisioner for AWS, you almost certainly do not want to use this service -#### Ping Server -Respond without doing anything. -This endpoint is used to check that the service is up. - - -```python -# Sync calls -eC2Manager.ping() # -> None` -# Async call -await asyncEC2Manager.ping() # -> None -``` - #### See the list of worker types which are known to be managed This method is only for debugging the ec2-manager -Required [output schema](v1/list-worker-types.json#) +Required [output schema](http://schemas.taskcluster.net/ec2-manager/v1/list-worker-types.json#) ```python # Sync calls @@ -1541,7 +1510,7 @@ Takes the following arguments: * `workerType` -Required [input schema](v1/run-instance-request.json#) +Required [input schema](http://schemas.taskcluster.net/ec2-manager/v1/run-instance-request.json#) ```python # Sync calls @@ -1579,7 +1548,7 @@ Takes the following arguments: * `workerType` -Required [output schema](v1/worker-type-resources.json#) +Required [output schema](http://schemas.taskcluster.net/ec2-manager/v1/worker-type-resources.json#) ```python # Sync calls @@ -1599,7 +1568,7 @@ Takes the following arguments: * `workerType` -Required [output schema](v1/health.json#) +Required [output schema](http://schemas.taskcluster.net/ec2-manager/v1/health.json#) ```python # Sync calls @@ -1619,7 +1588,7 @@ Takes the following arguments: * `workerType` -Required [output schema](v1/errors.json#) +Required [output schema](http://schemas.taskcluster.net/ec2-manager/v1/errors.json#) ```python # Sync calls @@ -1639,7 +1608,7 @@ Takes the following arguments: * `workerType` -Required [output schema](v1/worker-type-state.json#) +Required [output schema](http://schemas.taskcluster.net/ec2-manager/v1/worker-type-state.json#) ```python # Sync calls @@ -1659,7 +1628,7 @@ Takes the following arguments: * `name` -Required [input schema](v1/create-key-pair.json#) +Required [input schema](http://schemas.taskcluster.net/ec2-manager/v1/create-key-pair.json#) ```python # Sync calls @@ -1711,7 +1680,7 @@ await asyncEC2Manager.terminateInstance(region='value', instanceId='value') # -> Return a list of possible prices for EC2 -Required [output schema](v1/prices.json#) +Required [output schema](http://schemas.taskcluster.net/ec2-manager/v1/prices.json#) ```python # Sync calls @@ -1724,9 +1693,9 @@ await asyncEC2Manager.getPrices() # -> result Return a list of possible prices for EC2 -Required [input schema](v1/prices-request.json#) +Required [input schema](http://schemas.taskcluster.net/ec2-manager/v1/prices-request.json#) -Required [output schema](v1/prices.json#) +Required [output schema](http://schemas.taskcluster.net/ec2-manager/v1/prices.json#) ```python # Sync calls @@ -1739,7 +1708,7 @@ await asyncEC2Manager.getSpecificPrices(payload) # -> result Give some basic stats on the health of our EC2 account -Required [output schema](v1/health.json#) +Required [output schema](http://schemas.taskcluster.net/ec2-manager/v1/health.json#) ```python # Sync calls @@ -1752,7 +1721,7 @@ await asyncEC2Manager.getHealth() # -> result Return a list of recent errors encountered -Required [output schema](v1/errors.json#) +Required [output schema](http://schemas.taskcluster.net/ec2-manager/v1/errors.json#) ```python # Sync calls @@ -1852,6 +1821,29 @@ eC2Manager.purgeQueues() # -> None` await asyncEC2Manager.purgeQueues() # -> None ``` +#### API Reference +Generate an API reference for this service + + +```python +# Sync calls +eC2Manager.apiReference() # -> None` +# Async call +await asyncEC2Manager.apiReference() # -> None +``` + +#### Ping Server +Respond without doing anything. +This endpoint is used to check that the service is up. + + +```python +# Sync calls +eC2Manager.ping() # -> None` +# Async call +await asyncEC2Manager.ping() # -> None +``` + @@ -1868,8 +1860,9 @@ loop = asyncio.get_event_loop() session = taskcluster.aio.createSession(loop=loop) asyncGithub = taskcluster.aio.Github(options, session=session) ``` -The github service is responsible for creating tasks in reposnse -to GitHub events, and posting results to the GitHub UI. +The github service, typically available at +`github.taskcluster.net`, is responsible for publishing pulse +messages in response to GitHub events. This document describes the API end-point for consuming GitHub web hooks, as well as some useful consumer APIs. @@ -2063,12 +2056,6 @@ github service * `organization` is required Description: The GitHub `organization` which had an event. All periods have been replaced by % - such that foo.bar becomes foo%bar - and all other special characters aside from - and _ have been stripped. * `repository` is required Description: The GitHub `repository` which had an event.All periods have been replaced by % - such that foo.bar becomes foo%bar - and all other special characters aside from - and _ have been stripped. -#### GitHub release Event - * `githubEvents.taskGroupDefined(routingKeyPattern) -> routingKey` - * `routingKeyKind` is constant of `primary` is required Description: Identifier for the routing-key kind. This is always `"primary"` for the formalized routing key. - * `organization` is required Description: The GitHub `organization` which had an event. All periods have been replaced by % - such that foo.bar becomes foo%bar - and all other special characters aside from - and _ have been stripped. - * `repository` is required Description: The GitHub `repository` which had an event.All periods have been replaced by % - such that foo.bar becomes foo%bar - and all other special characters aside from - and _ have been stripped. - @@ -2102,7 +2089,7 @@ https://www.npmjs.com/package/cron-parser. For example: * `['0 0 9,21 * * 1-5', '0 0 12 * * 0,6']` -- weekdays at 9:00 and 21:00 UTC, weekends at noon The task definition is used as a JSON-e template, with a context depending on how it is fired. See -[/docs/reference/core/taskcluster-hooks/docs/firing-hooks](firing-hooks) +https://docs.taskcluster.net/reference/core/taskcluster-hooks/docs/firing-hooks for more information. #### Ping Server Respond without doing anything. @@ -2636,18 +2623,6 @@ asyncLogin = taskcluster.aio.Login(options, session=session) ``` The Login service serves as the interface between external authentication systems and Taskcluster credentials. -#### Ping Server -Respond without doing anything. -This endpoint is used to check that the service is up. - - -```python -# Sync calls -login.ping() # -> None` -# Async call -await asyncLogin.ping() # -> None -``` - #### Get Taskcluster credentials given a suitable `access_token` Given an OIDC `access_token` from a trusted OpenID provider, return a set of Taskcluster credentials for use on behalf of the identified @@ -2661,7 +2636,7 @@ Authorization: Bearer abc.xyz ``` The `access_token` is first verified against the named -:provider, then passed to the provider's APIBuilder to retrieve a user +:provider, then passed to the provider's API to retrieve a user profile. That profile is then used to generate Taskcluster credentials appropriate to the user. Note that the resulting credentials may or may not include a `certificate` property. Callers should be prepared for either @@ -2677,7 +2652,7 @@ Takes the following arguments: * `provider` -Required [output schema](v1/oidc-credentials-response.json#) +Required [output schema](http://schemas.taskcluster.net/login/v1/oidc-credentials-response.json) ```python # Sync calls @@ -2688,6 +2663,18 @@ await asyncLogin.oidcCredentials(provider) # -> result await asyncLogin.oidcCredentials(provider='value') # -> result ``` +#### Ping Server +Respond without doing anything. +This endpoint is used to check that the service is up. + + +```python +# Sync calls +login.ping() # -> None` +# Async call +await asyncLogin.ping() # -> None +``` + @@ -2774,111 +2761,6 @@ await asyncNotify.irc(payload) # -> None -### Methods in `taskcluster.Pulse` -```python -import asyncio # Only for async -// Create Pulse client instance -import taskcluster -import taskcluster.aio - -pulse = taskcluster.Pulse(options) -# Below only for async instances, assume already in coroutine -loop = asyncio.get_event_loop() -session = taskcluster.aio.createSession(loop=loop) -asyncPulse = taskcluster.aio.Pulse(options, session=session) -``` -The taskcluster-pulse service, typically available at `pulse.taskcluster.net` -manages pulse credentials for taskcluster users. - -A service to manage Pulse credentials for anything using -Taskcluster credentials. This allows for self-service pulse -access and greater control within the Taskcluster project. -#### Ping Server -Respond without doing anything. -This endpoint is used to check that the service is up. - - -```python -# Sync calls -pulse.ping() # -> None` -# Async call -await asyncPulse.ping() # -> None -``` - -#### List Namespaces -List the namespaces managed by this service. - -This will list up to 1000 namespaces. If more namespaces are present a -`continuationToken` will be returned, which can be given in the next -request. For the initial request, do not provide continuation token. - - -Required [output schema](v1/list-namespaces-response.json#) - -```python -# Sync calls -pulse.listNamespaces() # -> result` -# Async call -await asyncPulse.listNamespaces() # -> result -``` - -#### Get a namespace -Get public information about a single namespace. This is the same information -as returned by `listNamespaces`. - - - -Takes the following arguments: - - * `namespace` - -Required [output schema](v1/namespace.json#) - -```python -# Sync calls -pulse.namespace(namespace) # -> result` -pulse.namespace(namespace='value') # -> result -# Async call -await asyncPulse.namespace(namespace) # -> result -await asyncPulse.namespace(namespace='value') # -> result -``` - -#### Claim a namespace -Claim a namespace, returning a connection string with access to that namespace -good for use until the `reclaimAt` time in the response body. The connection -string can be used as many times as desired during this period, but must not -be used after `reclaimAt`. - -Connections made with this connection string may persist beyond `reclaimAt`, -although it should not persist forever. 24 hours is a good maximum, and this -service will terminate connections after 72 hours (although this value is -configurable). - -The specified `expires` time updates any existing expiration times. Connections -for expired namespaces will be terminated. - - - -Takes the following arguments: - - * `namespace` - -Required [input schema](v1/namespace-request.json#) - -Required [output schema](v1/namespace-response.json#) - -```python -# Sync calls -pulse.claimNamespace(namespace, payload) # -> result` -pulse.claimNamespace(payload, namespace='value') # -> result -# Async call -await asyncPulse.claimNamespace(namespace, payload) # -> result -await asyncPulse.claimNamespace(payload, namespace='value') # -> result -``` - - - - ### Methods in `taskcluster.PurgeCache` ```python import asyncio # Only for async @@ -2892,7 +2774,8 @@ loop = asyncio.get_event_loop() session = taskcluster.aio.createSession(loop=loop) asyncPurgeCache = taskcluster.aio.PurgeCache(options, session=session) ``` -The purge-cache service is responsible for publishing a pulse +The purge-cache service, typically available at +`purge-cache.taskcluster.net`, is responsible for publishing a pulse message for workers, so they can purge cache upon request. This document describes the API end-point for publishing the pulse diff --git a/third_party/python/taskcluster/setup.py b/third_party/python/taskcluster/setup.py index c9c7ff4a22c7..f005426af14f 100644 --- a/third_party/python/taskcluster/setup.py +++ b/third_party/python/taskcluster/setup.py @@ -7,7 +7,7 @@ import sys # The VERSION variable is automagically changed # by release.sh. Make sure you understand how # that script works if you want to change this -VERSION = '6.0.0' +VERSION = '4.0.1' tests_require = [ 'nose==1.3.7', @@ -30,7 +30,6 @@ install_requires = [ 'requests>=2.4.3,<3', 'mohawk>=0.3.4,<0.4', 'slugid>=1.0.7,<2', - 'taskcluster-urls>=10.1.0,<12', 'six>=1.10.0,<2', ] diff --git a/third_party/python/taskcluster/taskcluster/_client_importer.py b/third_party/python/taskcluster/taskcluster/_client_importer.py index c32ba5cc6b6d..ed5108122593 100644 --- a/third_party/python/taskcluster/taskcluster/_client_importer.py +++ b/third_party/python/taskcluster/taskcluster/_client_importer.py @@ -9,7 +9,6 @@ from .hooks import Hooks # NOQA from .index import Index # NOQA from .login import Login # NOQA from .notify import Notify # NOQA -from .pulse import Pulse # NOQA from .purgecache import PurgeCache # NOQA from .purgecacheevents import PurgeCacheEvents # NOQA from .queue import Queue # NOQA diff --git a/third_party/python/taskcluster/taskcluster/aio/_client_importer.py b/third_party/python/taskcluster/taskcluster/aio/_client_importer.py index c32ba5cc6b6d..ed5108122593 100644 --- a/third_party/python/taskcluster/taskcluster/aio/_client_importer.py +++ b/third_party/python/taskcluster/taskcluster/aio/_client_importer.py @@ -9,7 +9,6 @@ from .hooks import Hooks # NOQA from .index import Index # NOQA from .login import Login # NOQA from .notify import Notify # NOQA -from .pulse import Pulse # NOQA from .purgecache import PurgeCache # NOQA from .purgecacheevents import PurgeCacheEvents # NOQA from .queue import Queue # NOQA diff --git a/third_party/python/taskcluster/taskcluster/aio/asyncclient.py b/third_party/python/taskcluster/taskcluster/aio/asyncclient.py index 4cafd6f67bc1..21cc1d7170b9 100644 --- a/third_party/python/taskcluster/taskcluster/aio/asyncclient.py +++ b/third_party/python/taskcluster/taskcluster/aio/asyncclient.py @@ -114,7 +114,7 @@ class AsyncBaseClient(BaseClient): the logic about doing failure retry and passes off the actual work of doing an HTTP request to another method.""" - url = self._constructUrl(route) + url = self._joinBaseUrlAndRoute(route) log.debug('Full URL used is: %s', url) hawkExt = self.makeHawkExt() @@ -221,7 +221,7 @@ class AsyncBaseClient(BaseClient): try: await response.release() return await response.json() - except (ValueError, aiohttp.client_exceptions.ContentTypeError): + except ValueError: return {"response": response} # This code-path should be unreachable @@ -239,8 +239,6 @@ class AsyncBaseClient(BaseClient): def createApiClient(name, api): - api = api['reference'] - attributes = dict( name=name, __doc__=api.get('description'), @@ -248,22 +246,12 @@ def createApiClient(name, api): funcinfo={}, ) - # apply a default for apiVersion; this can be removed when all services - # have apiVersion - if 'apiVersion' not in api: - api['apiVersion'] = 'v1' - - copiedOptions = ('exchangePrefix',) + copiedOptions = ('baseUrl', 'exchangePrefix') for opt in copiedOptions: - if opt in api: - attributes['classOptions'][opt] = api[opt] + if opt in api['reference']: + attributes['classOptions'][opt] = api['reference'][opt] - copiedProperties = ('serviceName', 'apiVersion') - for opt in copiedProperties: - if opt in api: - attributes[opt] = api[opt] - - for entry in api['entries']: + for entry in api['reference']['entries']: if entry['type'] == 'function': def addApiCall(e): async def apiCall(self, *args, **kwargs): diff --git a/third_party/python/taskcluster/taskcluster/aio/asyncutils.py b/third_party/python/taskcluster/taskcluster/aio/asyncutils.py index 60c2cbae7ba0..1d3c717628cb 100644 --- a/third_party/python/taskcluster/taskcluster/aio/asyncutils.py +++ b/third_party/python/taskcluster/taskcluster/aio/asyncutils.py @@ -111,6 +111,6 @@ async def putFile(filename, url, contentType, session=None): with open(filename, 'rb') as f: contentLength = os.fstat(f.fileno()).st_size return await makeHttpRequest('put', url, f, headers={ - 'Content-Length': str(contentLength), + 'Content-Length': contentLength, 'Content-Type': contentType, }, session=session) diff --git a/third_party/python/taskcluster/taskcluster/aio/auth.py b/third_party/python/taskcluster/taskcluster/aio/auth.py index 755f89e3e995..97415b1c7e91 100644 --- a/third_party/python/taskcluster/taskcluster/aio/auth.py +++ b/third_party/python/taskcluster/taskcluster/aio/auth.py @@ -55,9 +55,8 @@ class Auth(AsyncBaseClient): """ classOptions = { + "baseUrl": "https://auth.taskcluster.net/v1/" } - serviceName = 'auth' - apiVersion = 'v1' async def ping(self, *args, **kwargs): """ diff --git a/third_party/python/taskcluster/taskcluster/aio/authevents.py b/third_party/python/taskcluster/taskcluster/aio/authevents.py index 1d5cc517e383..c50933d1fed6 100644 --- a/third_party/python/taskcluster/taskcluster/aio/authevents.py +++ b/third_party/python/taskcluster/taskcluster/aio/authevents.py @@ -13,9 +13,9 @@ _defaultConfig = config class AuthEvents(AsyncBaseClient): """ - The auth service is responsible for storing credentials, managing - assignment of scopes, and validation of request signatures from other - services. + The auth service, typically available at `auth.taskcluster.net` + is responsible for storing credentials, managing assignment of scopes, + and validation of request signatures from other services. These exchanges provides notifications when credentials or roles are updated. This is mostly so that multiple instances of the auth service @@ -24,10 +24,8 @@ class AuthEvents(AsyncBaseClient): """ classOptions = { - "exchangePrefix": "exchange/taskcluster-auth/v1/", + "exchangePrefix": "exchange/taskcluster-auth/v1/" } - serviceName = 'auth' - apiVersion = 'v1' def clientCreated(self, *args, **kwargs): """ diff --git a/third_party/python/taskcluster/taskcluster/aio/awsprovisioner.py b/third_party/python/taskcluster/taskcluster/aio/awsprovisioner.py index 1ef30306af92..985674ff7c92 100644 --- a/third_party/python/taskcluster/taskcluster/aio/awsprovisioner.py +++ b/third_party/python/taskcluster/taskcluster/aio/awsprovisioner.py @@ -44,9 +44,8 @@ class AwsProvisioner(AsyncBaseClient): """ classOptions = { + "baseUrl": "https://aws-provisioner.taskcluster.net/v1" } - serviceName = 'aws-provisioner' - apiVersion = 'v1' async def listWorkerTypeSummaries(self, *args, **kwargs): """ diff --git a/third_party/python/taskcluster/taskcluster/aio/awsprovisionerevents.py b/third_party/python/taskcluster/taskcluster/aio/awsprovisionerevents.py index d6ec9114a681..1e18af91d1aa 100644 --- a/third_party/python/taskcluster/taskcluster/aio/awsprovisionerevents.py +++ b/third_party/python/taskcluster/taskcluster/aio/awsprovisionerevents.py @@ -17,9 +17,8 @@ class AwsProvisionerEvents(AsyncBaseClient): """ classOptions = { - "exchangePrefix": "exchange/taskcluster-aws-provisioner/v1/", + "exchangePrefix": "exchange/taskcluster-aws-provisioner/v1/" } - apiVersion = 'v1' def workerTypeCreated(self, *args, **kwargs): """ diff --git a/third_party/python/taskcluster/taskcluster/aio/ec2manager.py b/third_party/python/taskcluster/taskcluster/aio/ec2manager.py index 2b055746fe48..789f6deae511 100644 --- a/third_party/python/taskcluster/taskcluster/aio/ec2manager.py +++ b/third_party/python/taskcluster/taskcluster/aio/ec2manager.py @@ -17,21 +17,8 @@ class EC2Manager(AsyncBaseClient): """ classOptions = { + "baseUrl": "https://ec2-manager.taskcluster.net/v1" } - serviceName = 'ec2-manager' - apiVersion = 'v1' - - async def ping(self, *args, **kwargs): - """ - Ping Server - - Respond without doing anything. - This endpoint is used to check that the service is up. - - This method is ``stable`` - """ - - return await self._makeApiCall(self.funcinfo["ping"], *args, **kwargs) async def listWorkerTypes(self, *args, **kwargs): """ @@ -39,7 +26,7 @@ class EC2Manager(AsyncBaseClient): This method is only for debugging the ec2-manager - This method gives output: ``v1/list-worker-types.json#`` + This method gives output: ``http://schemas.taskcluster.net/ec2-manager/v1/list-worker-types.json#`` This method is ``experimental`` """ @@ -52,7 +39,7 @@ class EC2Manager(AsyncBaseClient): Request an instance of a worker type - This method takes input: ``v1/run-instance-request.json#`` + This method takes input: ``http://schemas.taskcluster.net/ec2-manager/v1/run-instance-request.json#`` This method is ``experimental`` """ @@ -76,7 +63,7 @@ class EC2Manager(AsyncBaseClient): Return an object which has a generic state description. This only contains counts of instances - This method gives output: ``v1/worker-type-resources.json#`` + This method gives output: ``http://schemas.taskcluster.net/ec2-manager/v1/worker-type-resources.json#`` This method is ``experimental`` """ @@ -89,7 +76,7 @@ class EC2Manager(AsyncBaseClient): Return a view of the health of a given worker type - This method gives output: ``v1/health.json#`` + This method gives output: ``http://schemas.taskcluster.net/ec2-manager/v1/health.json#`` This method is ``experimental`` """ @@ -102,7 +89,7 @@ class EC2Manager(AsyncBaseClient): Return a list of the most recent errors encountered by a worker type - This method gives output: ``v1/errors.json#`` + This method gives output: ``http://schemas.taskcluster.net/ec2-manager/v1/errors.json#`` This method is ``experimental`` """ @@ -115,7 +102,7 @@ class EC2Manager(AsyncBaseClient): Return state information for a given worker type - This method gives output: ``v1/worker-type-state.json#`` + This method gives output: ``http://schemas.taskcluster.net/ec2-manager/v1/worker-type-state.json#`` This method is ``experimental`` """ @@ -128,7 +115,7 @@ class EC2Manager(AsyncBaseClient): Idempotently ensure that a keypair of a given name exists - This method takes input: ``v1/create-key-pair.json#`` + This method takes input: ``http://schemas.taskcluster.net/ec2-manager/v1/create-key-pair.json#`` This method is ``experimental`` """ @@ -163,7 +150,7 @@ class EC2Manager(AsyncBaseClient): Return a list of possible prices for EC2 - This method gives output: ``v1/prices.json#`` + This method gives output: ``http://schemas.taskcluster.net/ec2-manager/v1/prices.json#`` This method is ``experimental`` """ @@ -176,9 +163,9 @@ class EC2Manager(AsyncBaseClient): Return a list of possible prices for EC2 - This method takes input: ``v1/prices-request.json#`` + This method takes input: ``http://schemas.taskcluster.net/ec2-manager/v1/prices-request.json#`` - This method gives output: ``v1/prices.json#`` + This method gives output: ``http://schemas.taskcluster.net/ec2-manager/v1/prices.json#`` This method is ``experimental`` """ @@ -191,7 +178,7 @@ class EC2Manager(AsyncBaseClient): Give some basic stats on the health of our EC2 account - This method gives output: ``v1/health.json#`` + This method gives output: ``http://schemas.taskcluster.net/ec2-manager/v1/health.json#`` This method is ``experimental`` """ @@ -204,7 +191,7 @@ class EC2Manager(AsyncBaseClient): Return a list of recent errors encountered - This method gives output: ``v1/errors.json#`` + This method gives output: ``http://schemas.taskcluster.net/ec2-manager/v1/errors.json#`` This method is ``experimental`` """ @@ -302,6 +289,29 @@ class EC2Manager(AsyncBaseClient): return await self._makeApiCall(self.funcinfo["purgeQueues"], *args, **kwargs) + async def apiReference(self, *args, **kwargs): + """ + API Reference + + Generate an API reference for this service + + This method is ``experimental`` + """ + + return await self._makeApiCall(self.funcinfo["apiReference"], *args, **kwargs) + + async def ping(self, *args, **kwargs): + """ + Ping Server + + Respond without doing anything. + This endpoint is used to check that the service is up. + + This method is ``stable`` + """ + + return await self._makeApiCall(self.funcinfo["ping"], *args, **kwargs) + funcinfo = { "allState": { 'args': [], @@ -317,6 +327,13 @@ class EC2Manager(AsyncBaseClient): 'route': '/internal/ami-usage', 'stability': 'experimental', }, + "apiReference": { + 'args': [], + 'method': 'get', + 'name': 'apiReference', + 'route': '/internal/api-reference', + 'stability': 'experimental', + }, "dbpoolStats": { 'args': [], 'method': 'get', @@ -333,7 +350,7 @@ class EC2Manager(AsyncBaseClient): }, "ensureKeyPair": { 'args': ['name'], - 'input': 'v1/create-key-pair.json#', + 'input': 'http://schemas.taskcluster.net/ec2-manager/v1/create-key-pair.json#', 'method': 'get', 'name': 'ensureKeyPair', 'route': '/key-pairs/', @@ -343,7 +360,7 @@ class EC2Manager(AsyncBaseClient): 'args': [], 'method': 'get', 'name': 'getHealth', - 'output': 'v1/health.json#', + 'output': 'http://schemas.taskcluster.net/ec2-manager/v1/health.json#', 'route': '/health', 'stability': 'experimental', }, @@ -351,7 +368,7 @@ class EC2Manager(AsyncBaseClient): 'args': [], 'method': 'get', 'name': 'getPrices', - 'output': 'v1/prices.json#', + 'output': 'http://schemas.taskcluster.net/ec2-manager/v1/prices.json#', 'route': '/prices', 'stability': 'experimental', }, @@ -359,16 +376,16 @@ class EC2Manager(AsyncBaseClient): 'args': [], 'method': 'get', 'name': 'getRecentErrors', - 'output': 'v1/errors.json#', + 'output': 'http://schemas.taskcluster.net/ec2-manager/v1/errors.json#', 'route': '/errors', 'stability': 'experimental', }, "getSpecificPrices": { 'args': [], - 'input': 'v1/prices-request.json#', + 'input': 'http://schemas.taskcluster.net/ec2-manager/v1/prices-request.json#', 'method': 'post', 'name': 'getSpecificPrices', - 'output': 'v1/prices.json#', + 'output': 'http://schemas.taskcluster.net/ec2-manager/v1/prices.json#', 'route': '/prices', 'stability': 'experimental', }, @@ -376,7 +393,7 @@ class EC2Manager(AsyncBaseClient): 'args': [], 'method': 'get', 'name': 'listWorkerTypes', - 'output': 'v1/list-worker-types.json#', + 'output': 'http://schemas.taskcluster.net/ec2-manager/v1/list-worker-types.json#', 'route': '/worker-types', 'stability': 'experimental', }, @@ -410,7 +427,7 @@ class EC2Manager(AsyncBaseClient): }, "runInstance": { 'args': ['workerType'], - 'input': 'v1/run-instance-request.json#', + 'input': 'http://schemas.taskcluster.net/ec2-manager/v1/run-instance-request.json#', 'method': 'put', 'name': 'runInstance', 'route': '/worker-types//instance', @@ -441,7 +458,7 @@ class EC2Manager(AsyncBaseClient): 'args': ['workerType'], 'method': 'get', 'name': 'workerTypeErrors', - 'output': 'v1/errors.json#', + 'output': 'http://schemas.taskcluster.net/ec2-manager/v1/errors.json#', 'route': '/worker-types//errors', 'stability': 'experimental', }, @@ -449,7 +466,7 @@ class EC2Manager(AsyncBaseClient): 'args': ['workerType'], 'method': 'get', 'name': 'workerTypeHealth', - 'output': 'v1/health.json#', + 'output': 'http://schemas.taskcluster.net/ec2-manager/v1/health.json#', 'route': '/worker-types//health', 'stability': 'experimental', }, @@ -457,7 +474,7 @@ class EC2Manager(AsyncBaseClient): 'args': ['workerType'], 'method': 'get', 'name': 'workerTypeState', - 'output': 'v1/worker-type-state.json#', + 'output': 'http://schemas.taskcluster.net/ec2-manager/v1/worker-type-state.json#', 'route': '/worker-types//state', 'stability': 'experimental', }, @@ -465,7 +482,7 @@ class EC2Manager(AsyncBaseClient): 'args': ['workerType'], 'method': 'get', 'name': 'workerTypeStats', - 'output': 'v1/worker-type-resources.json#', + 'output': 'http://schemas.taskcluster.net/ec2-manager/v1/worker-type-resources.json#', 'route': '/worker-types//stats', 'stability': 'experimental', }, diff --git a/third_party/python/taskcluster/taskcluster/aio/github.py b/third_party/python/taskcluster/taskcluster/aio/github.py index af6795737b0a..29dd15061912 100644 --- a/third_party/python/taskcluster/taskcluster/aio/github.py +++ b/third_party/python/taskcluster/taskcluster/aio/github.py @@ -13,8 +13,9 @@ _defaultConfig = config class Github(AsyncBaseClient): """ - The github service is responsible for creating tasks in reposnse - to GitHub events, and posting results to the GitHub UI. + The github service, typically available at + `github.taskcluster.net`, is responsible for publishing pulse + messages in response to GitHub events. This document describes the API end-point for consuming GitHub web hooks, as well as some useful consumer APIs. @@ -24,9 +25,8 @@ class Github(AsyncBaseClient): """ classOptions = { + "baseUrl": "https://github.taskcluster.net/v1/" } - serviceName = 'github' - apiVersion = 'v1' async def ping(self, *args, **kwargs): """ diff --git a/third_party/python/taskcluster/taskcluster/aio/githubevents.py b/third_party/python/taskcluster/taskcluster/aio/githubevents.py index aecfcc85ab9f..06727056e7df 100644 --- a/third_party/python/taskcluster/taskcluster/aio/githubevents.py +++ b/third_party/python/taskcluster/taskcluster/aio/githubevents.py @@ -22,10 +22,8 @@ class GithubEvents(AsyncBaseClient): """ classOptions = { - "exchangePrefix": "exchange/taskcluster-github/v1/", + "exchangePrefix": "exchange/taskcluster-github/v1/" } - serviceName = 'github' - apiVersion = 'v1' def pullRequest(self, *args, **kwargs): """ @@ -150,43 +148,6 @@ class GithubEvents(AsyncBaseClient): } return self._makeTopicExchange(ref, *args, **kwargs) - def taskGroupDefined(self, *args, **kwargs): - """ - GitHub release Event - - used for creating status indicators in GitHub UI using Statuses API - - This exchange outputs: ``v1/task-group-defined-message.json#``This exchange takes the following keys: - - * routingKeyKind: Identifier for the routing-key kind. This is always `"primary"` for the formalized routing key. (required) - - * organization: The GitHub `organization` which had an event. All periods have been replaced by % - such that foo.bar becomes foo%bar - and all other special characters aside from - and _ have been stripped. (required) - - * repository: The GitHub `repository` which had an event.All periods have been replaced by % - such that foo.bar becomes foo%bar - and all other special characters aside from - and _ have been stripped. (required) - """ - - ref = { - 'exchange': 'task-group-defined', - 'name': 'taskGroupDefined', - 'routingKey': [ - { - 'constant': 'primary', - 'multipleWords': False, - 'name': 'routingKeyKind', - }, - { - 'multipleWords': False, - 'name': 'organization', - }, - { - 'multipleWords': False, - 'name': 'repository', - }, - ], - 'schema': 'v1/task-group-defined-message.json#', - } - return self._makeTopicExchange(ref, *args, **kwargs) - funcinfo = { } diff --git a/third_party/python/taskcluster/taskcluster/aio/hooks.py b/third_party/python/taskcluster/taskcluster/aio/hooks.py index 029702d484f1..009832dc7a7f 100644 --- a/third_party/python/taskcluster/taskcluster/aio/hooks.py +++ b/third_party/python/taskcluster/taskcluster/aio/hooks.py @@ -30,14 +30,13 @@ class Hooks(AsyncBaseClient): * `['0 0 9,21 * * 1-5', '0 0 12 * * 0,6']` -- weekdays at 9:00 and 21:00 UTC, weekends at noon The task definition is used as a JSON-e template, with a context depending on how it is fired. See - [/docs/reference/core/taskcluster-hooks/docs/firing-hooks](firing-hooks) + https://docs.taskcluster.net/reference/core/taskcluster-hooks/docs/firing-hooks for more information. """ classOptions = { + "baseUrl": "https://hooks.taskcluster.net/v1/" } - serviceName = 'hooks' - apiVersion = 'v1' async def ping(self, *args, **kwargs): """ diff --git a/third_party/python/taskcluster/taskcluster/aio/index.py b/third_party/python/taskcluster/taskcluster/aio/index.py index 2eb440b132a6..df61f39b3b78 100644 --- a/third_party/python/taskcluster/taskcluster/aio/index.py +++ b/third_party/python/taskcluster/taskcluster/aio/index.py @@ -108,9 +108,8 @@ class Index(AsyncBaseClient): """ classOptions = { + "baseUrl": "https://index.taskcluster.net/v1/" } - serviceName = 'index' - apiVersion = 'v1' async def ping(self, *args, **kwargs): """ diff --git a/third_party/python/taskcluster/taskcluster/aio/login.py b/third_party/python/taskcluster/taskcluster/aio/login.py index 83515ee15766..aa03c6963be9 100644 --- a/third_party/python/taskcluster/taskcluster/aio/login.py +++ b/third_party/python/taskcluster/taskcluster/aio/login.py @@ -18,21 +18,8 @@ class Login(AsyncBaseClient): """ classOptions = { + "baseUrl": "https://login.taskcluster.net/v1" } - serviceName = 'login' - apiVersion = 'v1' - - async def ping(self, *args, **kwargs): - """ - Ping Server - - Respond without doing anything. - This endpoint is used to check that the service is up. - - This method is ``stable`` - """ - - return await self._makeApiCall(self.funcinfo["ping"], *args, **kwargs) async def oidcCredentials(self, *args, **kwargs): """ @@ -50,7 +37,7 @@ class Login(AsyncBaseClient): ``` The `access_token` is first verified against the named - :provider, then passed to the provider's APIBuilder to retrieve a user + :provider, then passed to the provider's API to retrieve a user profile. That profile is then used to generate Taskcluster credentials appropriate to the user. Note that the resulting credentials may or may not include a `certificate` property. Callers should be prepared for either @@ -60,19 +47,31 @@ class Login(AsyncBaseClient): monitor this expiration and refresh the credentials if necessary, by calling this endpoint again, if they have expired. - This method gives output: ``v1/oidc-credentials-response.json#`` + This method gives output: ``http://schemas.taskcluster.net/login/v1/oidc-credentials-response.json`` This method is ``experimental`` """ return await self._makeApiCall(self.funcinfo["oidcCredentials"], *args, **kwargs) + async def ping(self, *args, **kwargs): + """ + Ping Server + + Respond without doing anything. + This endpoint is used to check that the service is up. + + This method is ``stable`` + """ + + return await self._makeApiCall(self.funcinfo["ping"], *args, **kwargs) + funcinfo = { "oidcCredentials": { 'args': ['provider'], 'method': 'get', 'name': 'oidcCredentials', - 'output': 'v1/oidc-credentials-response.json#', + 'output': 'http://schemas.taskcluster.net/login/v1/oidc-credentials-response.json', 'route': '/oidc-credentials/', 'stability': 'experimental', }, diff --git a/third_party/python/taskcluster/taskcluster/aio/notify.py b/third_party/python/taskcluster/taskcluster/aio/notify.py index d4aa02eefe0a..d0e8a1a0d269 100644 --- a/third_party/python/taskcluster/taskcluster/aio/notify.py +++ b/third_party/python/taskcluster/taskcluster/aio/notify.py @@ -19,9 +19,8 @@ class Notify(AsyncBaseClient): """ classOptions = { + "baseUrl": "https://notify.taskcluster.net/v1/" } - serviceName = 'notify' - apiVersion = 'v1' async def ping(self, *args, **kwargs): """ diff --git a/third_party/python/taskcluster/taskcluster/aio/pulse.py b/third_party/python/taskcluster/taskcluster/aio/pulse.py index 1ce4966427dc..894a44b286e7 100644 --- a/third_party/python/taskcluster/taskcluster/aio/pulse.py +++ b/third_party/python/taskcluster/taskcluster/aio/pulse.py @@ -22,21 +22,21 @@ class Pulse(AsyncBaseClient): """ classOptions = { + "baseUrl": "https://pulse.taskcluster.net/v1" } - serviceName = 'pulse' - apiVersion = 'v1' - async def ping(self, *args, **kwargs): + async def overview(self, *args, **kwargs): """ - Ping Server + Rabbit Overview - Respond without doing anything. - This endpoint is used to check that the service is up. + Get an overview of the Rabbit cluster. - This method is ``stable`` + This method gives output: ``http://schemas.taskcluster.net/pulse/v1/rabbit-overview.json`` + + This method is ``experimental`` """ - return await self._makeApiCall(self.funcinfo["ping"], *args, **kwargs) + return await self._makeApiCall(self.funcinfo["overview"], *args, **kwargs) async def listNamespaces(self, *args, **kwargs): """ @@ -46,9 +46,9 @@ class Pulse(AsyncBaseClient): This will list up to 1000 namespaces. If more namespaces are present a `continuationToken` will be returned, which can be given in the next - request. For the initial request, do not provide continuation token. + request. For the initial request, do not provide continuation. - This method gives output: ``v1/list-namespaces-response.json#`` + This method gives output: ``http://schemas.taskcluster.net/pulse/v1/list-namespaces-response.json`` This method is ``experimental`` """ @@ -62,7 +62,7 @@ class Pulse(AsyncBaseClient): Get public information about a single namespace. This is the same information as returned by `listNamespaces`. - This method gives output: ``v1/namespace.json#`` + This method gives output: ``http://schemas.taskcluster.net/pulse/v1/namespace.json`` This method is ``experimental`` """ @@ -73,35 +73,43 @@ class Pulse(AsyncBaseClient): """ Claim a namespace - Claim a namespace, returning a connection string with access to that namespace - good for use until the `reclaimAt` time in the response body. The connection - string can be used as many times as desired during this period, but must not - be used after `reclaimAt`. + Claim a namespace, returning a username and password with access to that + namespace good for a short time. Clients should call this endpoint again + at the re-claim time given in the response, as the password will be rotated + soon after that time. The namespace will expire, and any associated queues + and exchanges will be deleted, at the given expiration time. - Connections made with this connection string may persist beyond `reclaimAt`, - although it should not persist forever. 24 hours is a good maximum, and this - service will terminate connections after 72 hours (although this value is - configurable). + The `expires` and `contact` properties can be updated at any time in a reclaim + operation. - The specified `expires` time updates any existing expiration times. Connections - for expired namespaces will be terminated. + This method takes input: ``http://schemas.taskcluster.net/pulse/v1/namespace-request.json`` - This method takes input: ``v1/namespace-request.json#`` - - This method gives output: ``v1/namespace-response.json#`` + This method gives output: ``http://schemas.taskcluster.net/pulse/v1/namespace-response.json`` This method is ``experimental`` """ return await self._makeApiCall(self.funcinfo["claimNamespace"], *args, **kwargs) + async def ping(self, *args, **kwargs): + """ + Ping Server + + Respond without doing anything. + This endpoint is used to check that the service is up. + + This method is ``stable`` + """ + + return await self._makeApiCall(self.funcinfo["ping"], *args, **kwargs) + funcinfo = { "claimNamespace": { 'args': ['namespace'], - 'input': 'v1/namespace-request.json#', + 'input': 'http://schemas.taskcluster.net/pulse/v1/namespace-request.json', 'method': 'post', 'name': 'claimNamespace', - 'output': 'v1/namespace-response.json#', + 'output': 'http://schemas.taskcluster.net/pulse/v1/namespace-response.json', 'route': '/namespace/', 'stability': 'experimental', }, @@ -109,8 +117,8 @@ class Pulse(AsyncBaseClient): 'args': [], 'method': 'get', 'name': 'listNamespaces', - 'output': 'v1/list-namespaces-response.json#', - 'query': ['limit', 'continuationToken'], + 'output': 'http://schemas.taskcluster.net/pulse/v1/list-namespaces-response.json', + 'query': ['limit', 'continuation'], 'route': '/namespaces', 'stability': 'experimental', }, @@ -118,10 +126,18 @@ class Pulse(AsyncBaseClient): 'args': ['namespace'], 'method': 'get', 'name': 'namespace', - 'output': 'v1/namespace.json#', + 'output': 'http://schemas.taskcluster.net/pulse/v1/namespace.json', 'route': '/namespace/', 'stability': 'experimental', }, + "overview": { + 'args': [], + 'method': 'get', + 'name': 'overview', + 'output': 'http://schemas.taskcluster.net/pulse/v1/rabbit-overview.json', + 'route': '/overview', + 'stability': 'experimental', + }, "ping": { 'args': [], 'method': 'get', diff --git a/third_party/python/taskcluster/taskcluster/aio/purgecache.py b/third_party/python/taskcluster/taskcluster/aio/purgecache.py index b36273d30654..2d61ee07923b 100644 --- a/third_party/python/taskcluster/taskcluster/aio/purgecache.py +++ b/third_party/python/taskcluster/taskcluster/aio/purgecache.py @@ -13,7 +13,8 @@ _defaultConfig = config class PurgeCache(AsyncBaseClient): """ - The purge-cache service is responsible for publishing a pulse + The purge-cache service, typically available at + `purge-cache.taskcluster.net`, is responsible for publishing a pulse message for workers, so they can purge cache upon request. This document describes the API end-point for publishing the pulse @@ -21,9 +22,8 @@ class PurgeCache(AsyncBaseClient): """ classOptions = { + "baseUrl": "https://purge-cache.taskcluster.net/v1/" } - serviceName = 'purge-cache' - apiVersion = 'v1' async def ping(self, *args, **kwargs): """ diff --git a/third_party/python/taskcluster/taskcluster/aio/purgecacheevents.py b/third_party/python/taskcluster/taskcluster/aio/purgecacheevents.py index 69b8a305518e..293716107a2d 100644 --- a/third_party/python/taskcluster/taskcluster/aio/purgecacheevents.py +++ b/third_party/python/taskcluster/taskcluster/aio/purgecacheevents.py @@ -22,10 +22,8 @@ class PurgeCacheEvents(AsyncBaseClient): """ classOptions = { - "exchangePrefix": "exchange/taskcluster-purge-cache/v1/", + "exchangePrefix": "exchange/taskcluster-purge-cache/v1/" } - serviceName = 'purge-cache' - apiVersion = 'v1' def purgeCache(self, *args, **kwargs): """ diff --git a/third_party/python/taskcluster/taskcluster/aio/queue.py b/third_party/python/taskcluster/taskcluster/aio/queue.py index 58fd4d9e71ca..ff6dcc015f75 100644 --- a/third_party/python/taskcluster/taskcluster/aio/queue.py +++ b/third_party/python/taskcluster/taskcluster/aio/queue.py @@ -25,9 +25,8 @@ class Queue(AsyncBaseClient): """ classOptions = { + "baseUrl": "https://queue.taskcluster.net/v1/" } - serviceName = 'queue' - apiVersion = 'v1' async def ping(self, *args, **kwargs): """ diff --git a/third_party/python/taskcluster/taskcluster/aio/queueevents.py b/third_party/python/taskcluster/taskcluster/aio/queueevents.py index 3b0b84662ddc..16b87f6647ae 100644 --- a/third_party/python/taskcluster/taskcluster/aio/queueevents.py +++ b/third_party/python/taskcluster/taskcluster/aio/queueevents.py @@ -63,10 +63,8 @@ class QueueEvents(AsyncBaseClient): """ classOptions = { - "exchangePrefix": "exchange/taskcluster-queue/v1/", + "exchangePrefix": "exchange/taskcluster-queue/v1/" } - serviceName = 'queue' - apiVersion = 'v1' def taskDefined(self, *args, **kwargs): """ diff --git a/third_party/python/taskcluster/taskcluster/aio/secrets.py b/third_party/python/taskcluster/taskcluster/aio/secrets.py index abb20dcf82de..27df37ee3770 100644 --- a/third_party/python/taskcluster/taskcluster/aio/secrets.py +++ b/third_party/python/taskcluster/taskcluster/aio/secrets.py @@ -23,9 +23,8 @@ class Secrets(AsyncBaseClient): """ classOptions = { + "baseUrl": "https://secrets.taskcluster.net/v1/" } - serviceName = 'secrets' - apiVersion = 'v1' async def ping(self, *args, **kwargs): """ diff --git a/third_party/python/taskcluster/taskcluster/aio/treeherderevents.py b/third_party/python/taskcluster/taskcluster/aio/treeherderevents.py index 8d0f640182ae..50a8eacc8761 100644 --- a/third_party/python/taskcluster/taskcluster/aio/treeherderevents.py +++ b/third_party/python/taskcluster/taskcluster/aio/treeherderevents.py @@ -23,10 +23,8 @@ class TreeherderEvents(AsyncBaseClient): """ classOptions = { - "exchangePrefix": "exchange/taskcluster-treeherder/v1/", + "exchangePrefix": "exchange/taskcluster-treeherder/v1/" } - serviceName = 'treeherder' - apiVersion = 'v1' def jobs(self, *args, **kwargs): """ @@ -35,7 +33,7 @@ class TreeherderEvents(AsyncBaseClient): When a task run is scheduled or resolved, a message is posted to this exchange in a Treeherder consumable format. - This exchange outputs: ``v1/pulse-job.json#``This exchange takes the following keys: + This exchange outputs: ``http://schemas.taskcluster.net/taskcluster-treeherder/v1/pulse-job.json#``This exchange takes the following keys: * destination: destination (required) @@ -61,7 +59,7 @@ class TreeherderEvents(AsyncBaseClient): 'name': 'reserved', }, ], - 'schema': 'v1/pulse-job.json#', + 'schema': 'http://schemas.taskcluster.net/taskcluster-treeherder/v1/pulse-job.json#', } return self._makeTopicExchange(ref, *args, **kwargs) diff --git a/third_party/python/taskcluster/taskcluster/auth.py b/third_party/python/taskcluster/taskcluster/auth.py index 42dcf411d2a6..ec317df5d723 100644 --- a/third_party/python/taskcluster/taskcluster/auth.py +++ b/third_party/python/taskcluster/taskcluster/auth.py @@ -55,9 +55,8 @@ class Auth(BaseClient): """ classOptions = { + "baseUrl": "https://auth.taskcluster.net/v1/" } - serviceName = 'auth' - apiVersion = 'v1' def ping(self, *args, **kwargs): """ diff --git a/third_party/python/taskcluster/taskcluster/authevents.py b/third_party/python/taskcluster/taskcluster/authevents.py index 6f7dbfa88803..1a8ebd5c4aac 100644 --- a/third_party/python/taskcluster/taskcluster/authevents.py +++ b/third_party/python/taskcluster/taskcluster/authevents.py @@ -13,9 +13,9 @@ _defaultConfig = config class AuthEvents(BaseClient): """ - The auth service is responsible for storing credentials, managing - assignment of scopes, and validation of request signatures from other - services. + The auth service, typically available at `auth.taskcluster.net` + is responsible for storing credentials, managing assignment of scopes, + and validation of request signatures from other services. These exchanges provides notifications when credentials or roles are updated. This is mostly so that multiple instances of the auth service @@ -24,10 +24,8 @@ class AuthEvents(BaseClient): """ classOptions = { - "exchangePrefix": "exchange/taskcluster-auth/v1/", + "exchangePrefix": "exchange/taskcluster-auth/v1/" } - serviceName = 'auth' - apiVersion = 'v1' def clientCreated(self, *args, **kwargs): """ diff --git a/third_party/python/taskcluster/taskcluster/awsprovisioner.py b/third_party/python/taskcluster/taskcluster/awsprovisioner.py index 820bbea0e8a9..f94221bd6e89 100644 --- a/third_party/python/taskcluster/taskcluster/awsprovisioner.py +++ b/third_party/python/taskcluster/taskcluster/awsprovisioner.py @@ -44,9 +44,8 @@ class AwsProvisioner(BaseClient): """ classOptions = { + "baseUrl": "https://aws-provisioner.taskcluster.net/v1" } - serviceName = 'aws-provisioner' - apiVersion = 'v1' def listWorkerTypeSummaries(self, *args, **kwargs): """ diff --git a/third_party/python/taskcluster/taskcluster/awsprovisionerevents.py b/third_party/python/taskcluster/taskcluster/awsprovisionerevents.py index d24d773ff918..cbb929a7fae9 100644 --- a/third_party/python/taskcluster/taskcluster/awsprovisionerevents.py +++ b/third_party/python/taskcluster/taskcluster/awsprovisionerevents.py @@ -17,9 +17,8 @@ class AwsProvisionerEvents(BaseClient): """ classOptions = { - "exchangePrefix": "exchange/taskcluster-aws-provisioner/v1/", + "exchangePrefix": "exchange/taskcluster-aws-provisioner/v1/" } - apiVersion = 'v1' def workerTypeCreated(self, *args, **kwargs): """ diff --git a/third_party/python/taskcluster/taskcluster/client.py b/third_party/python/taskcluster/taskcluster/client.py index 9ef21956c265..e799d90ae375 100644 --- a/third_party/python/taskcluster/taskcluster/client.py +++ b/third_party/python/taskcluster/taskcluster/client.py @@ -2,6 +2,7 @@ from __future__ import absolute_import, division, print_function +import os import json import logging import copy @@ -20,7 +21,6 @@ import mohawk.bewit import taskcluster.exceptions as exceptions import taskcluster.utils as utils -import taskcluster_urls as liburls log = logging.getLogger(__name__) @@ -28,11 +28,10 @@ log = logging.getLogger(__name__) # Default configuration _defaultConfig = config = { 'credentials': { - 'clientId': None, - 'accessToken': None, - 'certificate': None, + 'clientId': os.environ.get('TASKCLUSTER_CLIENT_ID'), + 'accessToken': os.environ.get('TASKCLUSTER_ACCESS_TOKEN'), + 'certificate': os.environ.get('TASKCLUSTER_CERTIFICATE'), }, - 'rootUrl': None, 'maxRetries': 5, 'signedUrlExpiration': 15 * 60, } @@ -53,14 +52,10 @@ class BaseClient(object): """ def __init__(self, options=None, session=None): - if options and options.get('baseUrl'): - raise exceptions.TaskclusterFailure('baseUrl option is no longer allowed') o = copy.deepcopy(self.classOptions) o.update(_defaultConfig) if options: o.update(options) - if not o.get('rootUrl'): - raise exceptions.TaskclusterFailure('rootUrl option is required') credentials = o.get('credentials') if credentials: @@ -72,7 +67,6 @@ class BaseClient(object): except: s = '%s (%s) must be unicode encodable' % (x, credentials[x]) raise exceptions.TaskclusterAuthFailure(s) - self.options = o if 'credentials' in o: log.debug('credentials key scrubbed from logging output') @@ -174,7 +168,7 @@ class BaseClient(object): route = self._subArgsInRoute(entry, routeParams) if query: route += '?' + urllib.parse.urlencode(query) - return liburls.api(self.options['rootUrl'], self.serviceName, self.apiVersion, route) + return self._joinBaseUrlAndRoute(route) def buildSignedUrl(self, methodName, *args, **kwargs): """ Build a signed URL. This URL contains the credentials needed to access @@ -243,14 +237,11 @@ class BaseClient(object): u.fragment, )) - def _constructUrl(self, route): - """Construct a URL for the given route on this service, based on the - rootUrl""" - return liburls.api( - self.options['rootUrl'], - self.serviceName, - self.apiVersion, - route.rstrip('/')) + def _joinBaseUrlAndRoute(self, route): + return urllib.parse.urljoin( + '{}/'.format(self.options['baseUrl'].rstrip('/')), + route.lstrip('/') + ) def _makeApiCall(self, entry, *args, **kwargs): """ This function is used to dispatch calls to other functions @@ -446,7 +437,7 @@ class BaseClient(object): the logic about doing failure retry and passes off the actual work of doing an HTTP request to another method.""" - url = self._constructUrl(route) + url = self._joinBaseUrlAndRoute(route) log.debug('Full URL used is: %s', url) hawkExt = self.makeHawkExt() @@ -554,8 +545,6 @@ class BaseClient(object): def createApiClient(name, api): - api = api['reference'] - attributes = dict( name=name, __doc__=api.get('description'), @@ -563,22 +552,12 @@ def createApiClient(name, api): funcinfo={}, ) - # apply a default for apiVersion; this can be removed when all services - # have apiVersion - if 'apiVersion' not in api: - api['apiVersion'] = 'v1' - - copiedOptions = ('exchangePrefix',) + copiedOptions = ('baseUrl', 'exchangePrefix') for opt in copiedOptions: - if opt in api: - attributes['classOptions'][opt] = api[opt] + if opt in api['reference']: + attributes['classOptions'][opt] = api['reference'][opt] - copiedProperties = ('serviceName', 'apiVersion') - for opt in copiedProperties: - if opt in api: - attributes[opt] = api[opt] - - for entry in api['entries']: + for entry in api['reference']['entries']: if entry['type'] == 'function': def addApiCall(e): def apiCall(self, *args, **kwargs): diff --git a/third_party/python/taskcluster/taskcluster/ec2manager.py b/third_party/python/taskcluster/taskcluster/ec2manager.py index a8707ebe8194..b720a90e4bd4 100644 --- a/third_party/python/taskcluster/taskcluster/ec2manager.py +++ b/third_party/python/taskcluster/taskcluster/ec2manager.py @@ -17,21 +17,8 @@ class EC2Manager(BaseClient): """ classOptions = { + "baseUrl": "https://ec2-manager.taskcluster.net/v1" } - serviceName = 'ec2-manager' - apiVersion = 'v1' - - def ping(self, *args, **kwargs): - """ - Ping Server - - Respond without doing anything. - This endpoint is used to check that the service is up. - - This method is ``stable`` - """ - - return self._makeApiCall(self.funcinfo["ping"], *args, **kwargs) def listWorkerTypes(self, *args, **kwargs): """ @@ -39,7 +26,7 @@ class EC2Manager(BaseClient): This method is only for debugging the ec2-manager - This method gives output: ``v1/list-worker-types.json#`` + This method gives output: ``http://schemas.taskcluster.net/ec2-manager/v1/list-worker-types.json#`` This method is ``experimental`` """ @@ -52,7 +39,7 @@ class EC2Manager(BaseClient): Request an instance of a worker type - This method takes input: ``v1/run-instance-request.json#`` + This method takes input: ``http://schemas.taskcluster.net/ec2-manager/v1/run-instance-request.json#`` This method is ``experimental`` """ @@ -76,7 +63,7 @@ class EC2Manager(BaseClient): Return an object which has a generic state description. This only contains counts of instances - This method gives output: ``v1/worker-type-resources.json#`` + This method gives output: ``http://schemas.taskcluster.net/ec2-manager/v1/worker-type-resources.json#`` This method is ``experimental`` """ @@ -89,7 +76,7 @@ class EC2Manager(BaseClient): Return a view of the health of a given worker type - This method gives output: ``v1/health.json#`` + This method gives output: ``http://schemas.taskcluster.net/ec2-manager/v1/health.json#`` This method is ``experimental`` """ @@ -102,7 +89,7 @@ class EC2Manager(BaseClient): Return a list of the most recent errors encountered by a worker type - This method gives output: ``v1/errors.json#`` + This method gives output: ``http://schemas.taskcluster.net/ec2-manager/v1/errors.json#`` This method is ``experimental`` """ @@ -115,7 +102,7 @@ class EC2Manager(BaseClient): Return state information for a given worker type - This method gives output: ``v1/worker-type-state.json#`` + This method gives output: ``http://schemas.taskcluster.net/ec2-manager/v1/worker-type-state.json#`` This method is ``experimental`` """ @@ -128,7 +115,7 @@ class EC2Manager(BaseClient): Idempotently ensure that a keypair of a given name exists - This method takes input: ``v1/create-key-pair.json#`` + This method takes input: ``http://schemas.taskcluster.net/ec2-manager/v1/create-key-pair.json#`` This method is ``experimental`` """ @@ -163,7 +150,7 @@ class EC2Manager(BaseClient): Return a list of possible prices for EC2 - This method gives output: ``v1/prices.json#`` + This method gives output: ``http://schemas.taskcluster.net/ec2-manager/v1/prices.json#`` This method is ``experimental`` """ @@ -176,9 +163,9 @@ class EC2Manager(BaseClient): Return a list of possible prices for EC2 - This method takes input: ``v1/prices-request.json#`` + This method takes input: ``http://schemas.taskcluster.net/ec2-manager/v1/prices-request.json#`` - This method gives output: ``v1/prices.json#`` + This method gives output: ``http://schemas.taskcluster.net/ec2-manager/v1/prices.json#`` This method is ``experimental`` """ @@ -191,7 +178,7 @@ class EC2Manager(BaseClient): Give some basic stats on the health of our EC2 account - This method gives output: ``v1/health.json#`` + This method gives output: ``http://schemas.taskcluster.net/ec2-manager/v1/health.json#`` This method is ``experimental`` """ @@ -204,7 +191,7 @@ class EC2Manager(BaseClient): Return a list of recent errors encountered - This method gives output: ``v1/errors.json#`` + This method gives output: ``http://schemas.taskcluster.net/ec2-manager/v1/errors.json#`` This method is ``experimental`` """ @@ -302,6 +289,29 @@ class EC2Manager(BaseClient): return self._makeApiCall(self.funcinfo["purgeQueues"], *args, **kwargs) + def apiReference(self, *args, **kwargs): + """ + API Reference + + Generate an API reference for this service + + This method is ``experimental`` + """ + + return self._makeApiCall(self.funcinfo["apiReference"], *args, **kwargs) + + def ping(self, *args, **kwargs): + """ + Ping Server + + Respond without doing anything. + This endpoint is used to check that the service is up. + + This method is ``stable`` + """ + + return self._makeApiCall(self.funcinfo["ping"], *args, **kwargs) + funcinfo = { "allState": { 'args': [], @@ -317,6 +327,13 @@ class EC2Manager(BaseClient): 'route': '/internal/ami-usage', 'stability': 'experimental', }, + "apiReference": { + 'args': [], + 'method': 'get', + 'name': 'apiReference', + 'route': '/internal/api-reference', + 'stability': 'experimental', + }, "dbpoolStats": { 'args': [], 'method': 'get', @@ -333,7 +350,7 @@ class EC2Manager(BaseClient): }, "ensureKeyPair": { 'args': ['name'], - 'input': 'v1/create-key-pair.json#', + 'input': 'http://schemas.taskcluster.net/ec2-manager/v1/create-key-pair.json#', 'method': 'get', 'name': 'ensureKeyPair', 'route': '/key-pairs/', @@ -343,7 +360,7 @@ class EC2Manager(BaseClient): 'args': [], 'method': 'get', 'name': 'getHealth', - 'output': 'v1/health.json#', + 'output': 'http://schemas.taskcluster.net/ec2-manager/v1/health.json#', 'route': '/health', 'stability': 'experimental', }, @@ -351,7 +368,7 @@ class EC2Manager(BaseClient): 'args': [], 'method': 'get', 'name': 'getPrices', - 'output': 'v1/prices.json#', + 'output': 'http://schemas.taskcluster.net/ec2-manager/v1/prices.json#', 'route': '/prices', 'stability': 'experimental', }, @@ -359,16 +376,16 @@ class EC2Manager(BaseClient): 'args': [], 'method': 'get', 'name': 'getRecentErrors', - 'output': 'v1/errors.json#', + 'output': 'http://schemas.taskcluster.net/ec2-manager/v1/errors.json#', 'route': '/errors', 'stability': 'experimental', }, "getSpecificPrices": { 'args': [], - 'input': 'v1/prices-request.json#', + 'input': 'http://schemas.taskcluster.net/ec2-manager/v1/prices-request.json#', 'method': 'post', 'name': 'getSpecificPrices', - 'output': 'v1/prices.json#', + 'output': 'http://schemas.taskcluster.net/ec2-manager/v1/prices.json#', 'route': '/prices', 'stability': 'experimental', }, @@ -376,7 +393,7 @@ class EC2Manager(BaseClient): 'args': [], 'method': 'get', 'name': 'listWorkerTypes', - 'output': 'v1/list-worker-types.json#', + 'output': 'http://schemas.taskcluster.net/ec2-manager/v1/list-worker-types.json#', 'route': '/worker-types', 'stability': 'experimental', }, @@ -410,7 +427,7 @@ class EC2Manager(BaseClient): }, "runInstance": { 'args': ['workerType'], - 'input': 'v1/run-instance-request.json#', + 'input': 'http://schemas.taskcluster.net/ec2-manager/v1/run-instance-request.json#', 'method': 'put', 'name': 'runInstance', 'route': '/worker-types//instance', @@ -441,7 +458,7 @@ class EC2Manager(BaseClient): 'args': ['workerType'], 'method': 'get', 'name': 'workerTypeErrors', - 'output': 'v1/errors.json#', + 'output': 'http://schemas.taskcluster.net/ec2-manager/v1/errors.json#', 'route': '/worker-types//errors', 'stability': 'experimental', }, @@ -449,7 +466,7 @@ class EC2Manager(BaseClient): 'args': ['workerType'], 'method': 'get', 'name': 'workerTypeHealth', - 'output': 'v1/health.json#', + 'output': 'http://schemas.taskcluster.net/ec2-manager/v1/health.json#', 'route': '/worker-types//health', 'stability': 'experimental', }, @@ -457,7 +474,7 @@ class EC2Manager(BaseClient): 'args': ['workerType'], 'method': 'get', 'name': 'workerTypeState', - 'output': 'v1/worker-type-state.json#', + 'output': 'http://schemas.taskcluster.net/ec2-manager/v1/worker-type-state.json#', 'route': '/worker-types//state', 'stability': 'experimental', }, @@ -465,7 +482,7 @@ class EC2Manager(BaseClient): 'args': ['workerType'], 'method': 'get', 'name': 'workerTypeStats', - 'output': 'v1/worker-type-resources.json#', + 'output': 'http://schemas.taskcluster.net/ec2-manager/v1/worker-type-resources.json#', 'route': '/worker-types//stats', 'stability': 'experimental', }, diff --git a/third_party/python/taskcluster/taskcluster/github.py b/third_party/python/taskcluster/taskcluster/github.py index 56a432a8a45e..af962b504a35 100644 --- a/third_party/python/taskcluster/taskcluster/github.py +++ b/third_party/python/taskcluster/taskcluster/github.py @@ -13,8 +13,9 @@ _defaultConfig = config class Github(BaseClient): """ - The github service is responsible for creating tasks in reposnse - to GitHub events, and posting results to the GitHub UI. + The github service, typically available at + `github.taskcluster.net`, is responsible for publishing pulse + messages in response to GitHub events. This document describes the API end-point for consuming GitHub web hooks, as well as some useful consumer APIs. @@ -24,9 +25,8 @@ class Github(BaseClient): """ classOptions = { + "baseUrl": "https://github.taskcluster.net/v1/" } - serviceName = 'github' - apiVersion = 'v1' def ping(self, *args, **kwargs): """ diff --git a/third_party/python/taskcluster/taskcluster/githubevents.py b/third_party/python/taskcluster/taskcluster/githubevents.py index 73d37b608f4f..d54402e2d385 100644 --- a/third_party/python/taskcluster/taskcluster/githubevents.py +++ b/third_party/python/taskcluster/taskcluster/githubevents.py @@ -22,10 +22,8 @@ class GithubEvents(BaseClient): """ classOptions = { - "exchangePrefix": "exchange/taskcluster-github/v1/", + "exchangePrefix": "exchange/taskcluster-github/v1/" } - serviceName = 'github' - apiVersion = 'v1' def pullRequest(self, *args, **kwargs): """ @@ -150,43 +148,6 @@ class GithubEvents(BaseClient): } return self._makeTopicExchange(ref, *args, **kwargs) - def taskGroupDefined(self, *args, **kwargs): - """ - GitHub release Event - - used for creating status indicators in GitHub UI using Statuses API - - This exchange outputs: ``v1/task-group-defined-message.json#``This exchange takes the following keys: - - * routingKeyKind: Identifier for the routing-key kind. This is always `"primary"` for the formalized routing key. (required) - - * organization: The GitHub `organization` which had an event. All periods have been replaced by % - such that foo.bar becomes foo%bar - and all other special characters aside from - and _ have been stripped. (required) - - * repository: The GitHub `repository` which had an event.All periods have been replaced by % - such that foo.bar becomes foo%bar - and all other special characters aside from - and _ have been stripped. (required) - """ - - ref = { - 'exchange': 'task-group-defined', - 'name': 'taskGroupDefined', - 'routingKey': [ - { - 'constant': 'primary', - 'multipleWords': False, - 'name': 'routingKeyKind', - }, - { - 'multipleWords': False, - 'name': 'organization', - }, - { - 'multipleWords': False, - 'name': 'repository', - }, - ], - 'schema': 'v1/task-group-defined-message.json#', - } - return self._makeTopicExchange(ref, *args, **kwargs) - funcinfo = { } diff --git a/third_party/python/taskcluster/taskcluster/hooks.py b/third_party/python/taskcluster/taskcluster/hooks.py index b9608e3ec04b..730eacfab0a5 100644 --- a/third_party/python/taskcluster/taskcluster/hooks.py +++ b/third_party/python/taskcluster/taskcluster/hooks.py @@ -30,14 +30,13 @@ class Hooks(BaseClient): * `['0 0 9,21 * * 1-5', '0 0 12 * * 0,6']` -- weekdays at 9:00 and 21:00 UTC, weekends at noon The task definition is used as a JSON-e template, with a context depending on how it is fired. See - [/docs/reference/core/taskcluster-hooks/docs/firing-hooks](firing-hooks) + https://docs.taskcluster.net/reference/core/taskcluster-hooks/docs/firing-hooks for more information. """ classOptions = { + "baseUrl": "https://hooks.taskcluster.net/v1/" } - serviceName = 'hooks' - apiVersion = 'v1' def ping(self, *args, **kwargs): """ diff --git a/third_party/python/taskcluster/taskcluster/index.py b/third_party/python/taskcluster/taskcluster/index.py index fddda2ef1bee..7aaede360103 100644 --- a/third_party/python/taskcluster/taskcluster/index.py +++ b/third_party/python/taskcluster/taskcluster/index.py @@ -108,9 +108,8 @@ class Index(BaseClient): """ classOptions = { + "baseUrl": "https://index.taskcluster.net/v1/" } - serviceName = 'index' - apiVersion = 'v1' def ping(self, *args, **kwargs): """ diff --git a/third_party/python/taskcluster/taskcluster/login.py b/third_party/python/taskcluster/taskcluster/login.py index 235b60566d9c..0fd2ae9d7bcd 100644 --- a/third_party/python/taskcluster/taskcluster/login.py +++ b/third_party/python/taskcluster/taskcluster/login.py @@ -18,21 +18,8 @@ class Login(BaseClient): """ classOptions = { + "baseUrl": "https://login.taskcluster.net/v1" } - serviceName = 'login' - apiVersion = 'v1' - - def ping(self, *args, **kwargs): - """ - Ping Server - - Respond without doing anything. - This endpoint is used to check that the service is up. - - This method is ``stable`` - """ - - return self._makeApiCall(self.funcinfo["ping"], *args, **kwargs) def oidcCredentials(self, *args, **kwargs): """ @@ -50,7 +37,7 @@ class Login(BaseClient): ``` The `access_token` is first verified against the named - :provider, then passed to the provider's APIBuilder to retrieve a user + :provider, then passed to the provider's API to retrieve a user profile. That profile is then used to generate Taskcluster credentials appropriate to the user. Note that the resulting credentials may or may not include a `certificate` property. Callers should be prepared for either @@ -60,19 +47,31 @@ class Login(BaseClient): monitor this expiration and refresh the credentials if necessary, by calling this endpoint again, if they have expired. - This method gives output: ``v1/oidc-credentials-response.json#`` + This method gives output: ``http://schemas.taskcluster.net/login/v1/oidc-credentials-response.json`` This method is ``experimental`` """ return self._makeApiCall(self.funcinfo["oidcCredentials"], *args, **kwargs) + def ping(self, *args, **kwargs): + """ + Ping Server + + Respond without doing anything. + This endpoint is used to check that the service is up. + + This method is ``stable`` + """ + + return self._makeApiCall(self.funcinfo["ping"], *args, **kwargs) + funcinfo = { "oidcCredentials": { 'args': ['provider'], 'method': 'get', 'name': 'oidcCredentials', - 'output': 'v1/oidc-credentials-response.json#', + 'output': 'http://schemas.taskcluster.net/login/v1/oidc-credentials-response.json', 'route': '/oidc-credentials/', 'stability': 'experimental', }, diff --git a/third_party/python/taskcluster/taskcluster/notify.py b/third_party/python/taskcluster/taskcluster/notify.py index adc5b1d31503..ea736d7e8f89 100644 --- a/third_party/python/taskcluster/taskcluster/notify.py +++ b/third_party/python/taskcluster/taskcluster/notify.py @@ -19,9 +19,8 @@ class Notify(BaseClient): """ classOptions = { + "baseUrl": "https://notify.taskcluster.net/v1/" } - serviceName = 'notify' - apiVersion = 'v1' def ping(self, *args, **kwargs): """ diff --git a/third_party/python/taskcluster/taskcluster/pulse.py b/third_party/python/taskcluster/taskcluster/pulse.py index d6959f46fd79..0696d59ad864 100644 --- a/third_party/python/taskcluster/taskcluster/pulse.py +++ b/third_party/python/taskcluster/taskcluster/pulse.py @@ -22,21 +22,21 @@ class Pulse(BaseClient): """ classOptions = { + "baseUrl": "https://pulse.taskcluster.net/v1" } - serviceName = 'pulse' - apiVersion = 'v1' - def ping(self, *args, **kwargs): + def overview(self, *args, **kwargs): """ - Ping Server + Rabbit Overview - Respond without doing anything. - This endpoint is used to check that the service is up. + Get an overview of the Rabbit cluster. - This method is ``stable`` + This method gives output: ``http://schemas.taskcluster.net/pulse/v1/rabbit-overview.json`` + + This method is ``experimental`` """ - return self._makeApiCall(self.funcinfo["ping"], *args, **kwargs) + return self._makeApiCall(self.funcinfo["overview"], *args, **kwargs) def listNamespaces(self, *args, **kwargs): """ @@ -46,9 +46,9 @@ class Pulse(BaseClient): This will list up to 1000 namespaces. If more namespaces are present a `continuationToken` will be returned, which can be given in the next - request. For the initial request, do not provide continuation token. + request. For the initial request, do not provide continuation. - This method gives output: ``v1/list-namespaces-response.json#`` + This method gives output: ``http://schemas.taskcluster.net/pulse/v1/list-namespaces-response.json`` This method is ``experimental`` """ @@ -62,7 +62,7 @@ class Pulse(BaseClient): Get public information about a single namespace. This is the same information as returned by `listNamespaces`. - This method gives output: ``v1/namespace.json#`` + This method gives output: ``http://schemas.taskcluster.net/pulse/v1/namespace.json`` This method is ``experimental`` """ @@ -73,35 +73,43 @@ class Pulse(BaseClient): """ Claim a namespace - Claim a namespace, returning a connection string with access to that namespace - good for use until the `reclaimAt` time in the response body. The connection - string can be used as many times as desired during this period, but must not - be used after `reclaimAt`. + Claim a namespace, returning a username and password with access to that + namespace good for a short time. Clients should call this endpoint again + at the re-claim time given in the response, as the password will be rotated + soon after that time. The namespace will expire, and any associated queues + and exchanges will be deleted, at the given expiration time. - Connections made with this connection string may persist beyond `reclaimAt`, - although it should not persist forever. 24 hours is a good maximum, and this - service will terminate connections after 72 hours (although this value is - configurable). + The `expires` and `contact` properties can be updated at any time in a reclaim + operation. - The specified `expires` time updates any existing expiration times. Connections - for expired namespaces will be terminated. + This method takes input: ``http://schemas.taskcluster.net/pulse/v1/namespace-request.json`` - This method takes input: ``v1/namespace-request.json#`` - - This method gives output: ``v1/namespace-response.json#`` + This method gives output: ``http://schemas.taskcluster.net/pulse/v1/namespace-response.json`` This method is ``experimental`` """ return self._makeApiCall(self.funcinfo["claimNamespace"], *args, **kwargs) + def ping(self, *args, **kwargs): + """ + Ping Server + + Respond without doing anything. + This endpoint is used to check that the service is up. + + This method is ``stable`` + """ + + return self._makeApiCall(self.funcinfo["ping"], *args, **kwargs) + funcinfo = { "claimNamespace": { 'args': ['namespace'], - 'input': 'v1/namespace-request.json#', + 'input': 'http://schemas.taskcluster.net/pulse/v1/namespace-request.json', 'method': 'post', 'name': 'claimNamespace', - 'output': 'v1/namespace-response.json#', + 'output': 'http://schemas.taskcluster.net/pulse/v1/namespace-response.json', 'route': '/namespace/', 'stability': 'experimental', }, @@ -109,8 +117,8 @@ class Pulse(BaseClient): 'args': [], 'method': 'get', 'name': 'listNamespaces', - 'output': 'v1/list-namespaces-response.json#', - 'query': ['limit', 'continuationToken'], + 'output': 'http://schemas.taskcluster.net/pulse/v1/list-namespaces-response.json', + 'query': ['limit', 'continuation'], 'route': '/namespaces', 'stability': 'experimental', }, @@ -118,10 +126,18 @@ class Pulse(BaseClient): 'args': ['namespace'], 'method': 'get', 'name': 'namespace', - 'output': 'v1/namespace.json#', + 'output': 'http://schemas.taskcluster.net/pulse/v1/namespace.json', 'route': '/namespace/', 'stability': 'experimental', }, + "overview": { + 'args': [], + 'method': 'get', + 'name': 'overview', + 'output': 'http://schemas.taskcluster.net/pulse/v1/rabbit-overview.json', + 'route': '/overview', + 'stability': 'experimental', + }, "ping": { 'args': [], 'method': 'get', diff --git a/third_party/python/taskcluster/taskcluster/purgecache.py b/third_party/python/taskcluster/taskcluster/purgecache.py index a49e6bc83173..a3462c6056e2 100644 --- a/third_party/python/taskcluster/taskcluster/purgecache.py +++ b/third_party/python/taskcluster/taskcluster/purgecache.py @@ -13,7 +13,8 @@ _defaultConfig = config class PurgeCache(BaseClient): """ - The purge-cache service is responsible for publishing a pulse + The purge-cache service, typically available at + `purge-cache.taskcluster.net`, is responsible for publishing a pulse message for workers, so they can purge cache upon request. This document describes the API end-point for publishing the pulse @@ -21,9 +22,8 @@ class PurgeCache(BaseClient): """ classOptions = { + "baseUrl": "https://purge-cache.taskcluster.net/v1/" } - serviceName = 'purge-cache' - apiVersion = 'v1' def ping(self, *args, **kwargs): """ diff --git a/third_party/python/taskcluster/taskcluster/purgecacheevents.py b/third_party/python/taskcluster/taskcluster/purgecacheevents.py index 97637f19049b..bc0a0cf5bc0c 100644 --- a/third_party/python/taskcluster/taskcluster/purgecacheevents.py +++ b/third_party/python/taskcluster/taskcluster/purgecacheevents.py @@ -22,10 +22,8 @@ class PurgeCacheEvents(BaseClient): """ classOptions = { - "exchangePrefix": "exchange/taskcluster-purge-cache/v1/", + "exchangePrefix": "exchange/taskcluster-purge-cache/v1/" } - serviceName = 'purge-cache' - apiVersion = 'v1' def purgeCache(self, *args, **kwargs): """ diff --git a/third_party/python/taskcluster/taskcluster/queue.py b/third_party/python/taskcluster/taskcluster/queue.py index 408e526ad236..1d824bf25b04 100644 --- a/third_party/python/taskcluster/taskcluster/queue.py +++ b/third_party/python/taskcluster/taskcluster/queue.py @@ -25,9 +25,8 @@ class Queue(BaseClient): """ classOptions = { + "baseUrl": "https://queue.taskcluster.net/v1/" } - serviceName = 'queue' - apiVersion = 'v1' def ping(self, *args, **kwargs): """ diff --git a/third_party/python/taskcluster/taskcluster/queueevents.py b/third_party/python/taskcluster/taskcluster/queueevents.py index 0ece2985d7ed..634745c1c127 100644 --- a/third_party/python/taskcluster/taskcluster/queueevents.py +++ b/third_party/python/taskcluster/taskcluster/queueevents.py @@ -63,10 +63,8 @@ class QueueEvents(BaseClient): """ classOptions = { - "exchangePrefix": "exchange/taskcluster-queue/v1/", + "exchangePrefix": "exchange/taskcluster-queue/v1/" } - serviceName = 'queue' - apiVersion = 'v1' def taskDefined(self, *args, **kwargs): """ diff --git a/third_party/python/taskcluster/taskcluster/secrets.py b/third_party/python/taskcluster/taskcluster/secrets.py index 08a4632f3394..be2b0501fd30 100644 --- a/third_party/python/taskcluster/taskcluster/secrets.py +++ b/third_party/python/taskcluster/taskcluster/secrets.py @@ -23,9 +23,8 @@ class Secrets(BaseClient): """ classOptions = { + "baseUrl": "https://secrets.taskcluster.net/v1/" } - serviceName = 'secrets' - apiVersion = 'v1' def ping(self, *args, **kwargs): """ diff --git a/third_party/python/taskcluster/taskcluster/treeherderevents.py b/third_party/python/taskcluster/taskcluster/treeherderevents.py index 42461f2eeb1b..352baf0ab6c0 100644 --- a/third_party/python/taskcluster/taskcluster/treeherderevents.py +++ b/third_party/python/taskcluster/taskcluster/treeherderevents.py @@ -23,10 +23,8 @@ class TreeherderEvents(BaseClient): """ classOptions = { - "exchangePrefix": "exchange/taskcluster-treeherder/v1/", + "exchangePrefix": "exchange/taskcluster-treeherder/v1/" } - serviceName = 'treeherder' - apiVersion = 'v1' def jobs(self, *args, **kwargs): """ @@ -35,7 +33,7 @@ class TreeherderEvents(BaseClient): When a task run is scheduled or resolved, a message is posted to this exchange in a Treeherder consumable format. - This exchange outputs: ``v1/pulse-job.json#``This exchange takes the following keys: + This exchange outputs: ``http://schemas.taskcluster.net/taskcluster-treeherder/v1/pulse-job.json#``This exchange takes the following keys: * destination: destination (required) @@ -61,7 +59,7 @@ class TreeherderEvents(BaseClient): 'name': 'reserved', }, ], - 'schema': 'v1/pulse-job.json#', + 'schema': 'http://schemas.taskcluster.net/taskcluster-treeherder/v1/pulse-job.json#', } return self._makeTopicExchange(ref, *args, **kwargs) diff --git a/third_party/python/taskcluster/taskcluster/utils.py b/third_party/python/taskcluster/taskcluster/utils.py index 8fac855d2b63..fd7817f7ac85 100644 --- a/third_party/python/taskcluster/taskcluster/utils.py +++ b/third_party/python/taskcluster/taskcluster/utils.py @@ -1,4 +1,3 @@ -# -*- coding: UTF-8 -*- from __future__ import absolute_import, division, print_function import re import json @@ -11,6 +10,7 @@ import requests.exceptions import slugid import time import six +import sys import random from . import exceptions @@ -298,7 +298,7 @@ def putFile(filename, url, contentType): with open(filename, 'rb') as f: contentLength = os.fstat(f.fileno()).st_size return makeHttpRequest('put', url, f, headers={ - 'Content-Length': str(contentLength), + 'Content-Length': contentLength, 'Content-Type': contentType, }) @@ -319,30 +319,89 @@ def isExpired(certificate): return expiry < int(time.time() * 1000) + 20 * 60 -def optionsFromEnvironment(defaults=None): - """Fetch root URL and credentials from the standard TASKCLUSTER_… - environment variables and return them in a format suitable for passing to a - client constructor.""" - options = defaults or {} - credentials = options.get('credentials', {}) +def authenticate(description=None): + """ + Open a web-browser to login.taskcluster.net and listen on localhost for + a callback with credentials in query-string. - rootUrl = os.environ.get('TASKCLUSTER_ROOT_URL') - if rootUrl: - options['rootUrl'] = rootUrl + The description will be shown on login.taskcluster.net, if not provided + a default message with script path will be displayed. + """ + # Importing here to avoid loading these 'obscure' module before it's needed. + # Most clients won't use this feature, so we don't want issues with these + # modules to affect the library. Maybe they don't work in some environments + import webbrowser + from six.moves import urllib + from six.moves.urllib.parse import quote + import BaseHTTPServer - clientId = os.environ.get('TASKCLUSTER_CLIENT_ID') - if clientId: - credentials['clientId'] = clientId + if not description: + script = '[interpreter/unknown]' + main = sys.modules.get('__main__', None) + if main and hasattr(main, '__file__'): + script = os.path.abspath(main.__file__) + description = ( + "Python script: `%s`\n\nWould like some temporary credentials." + % script + ) - accessToken = os.environ.get('TASKCLUSTER_ACCESS_TOKEN') - if accessToken: - credentials['accessToken'] = accessToken + creds = [None] - certificate = os.environ.get('TASKCLUSTER_CERTIFICATE') - if certificate: - credentials['certificate'] = certificate + class AuthCallBackRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): + def log_message(format, *args): + pass - if credentials: - options['credentials'] = credentials + def do_GET(self): + url = urllib.parse.urlparse(self.path) + query = urllib.parse.parse_qs(url.query) + clientId = query.get('clientId', [None])[0] + accessToken = query.get('accessToken', [None])[0] + certificate = query.get('certificate', [None])[0] + hasCreds = clientId and accessToken and certificate + if hasCreds: + creds[0] = { + "clientId": clientId, + "accessToken": accessToken, + "certificate": certificate + } + self.send_response(200) + self.send_header('Content-type', 'text/html') + self.end_headers() + if hasCreds: + self.wfile.write(""" +

Credentials transferred successfully

+ You can close this window now. + + """) + else: + self.wfile.write(""" +

Transfer of credentials failed!

+

Something went wrong, you can navigate back and try again...

+ """) + return - return options + # Create server on localhost at random port + retries = 5 + while retries > 0: + try: + server = BaseHTTPServer.HTTPServer(('', 0), AuthCallBackRequestHandler) + except: + retries -= 1 + break + port = server.server_address[1] + + query = "?target=" + quote('http://localhost:' + str(port), '') + query += "&description=" + quote(description, '') + + webbrowser.open('https://login.taskcluster.net' + query, 1, True) + print("") + print("-------------------------------------------------------") + print(" Opening browser window to login.taskcluster.net") + print(" Asking you to grant temporary credentials to:") + print(" http://localhost:" + str(port)) + print("-------------------------------------------------------") + print("") + + while not creds[0]: + server.handle_request() + return creds[0] diff --git a/third_party/python/taskcluster/test/test_async.py b/third_party/python/taskcluster/test/test_async.py index 8a06f422b596..b26f8b371063 100644 --- a/third_party/python/taskcluster/test/test_async.py +++ b/third_party/python/taskcluster/test/test_async.py @@ -21,7 +21,6 @@ class TestAuthenticationAsync(base.TCTest): async def x(): async with subjectAsync.createSession(loop=loop) as session: client = subjectAsync.Auth({ - 'rootUrl': self.real_root_url, 'credentials': { 'clientId': 'tester', 'accessToken': 'no-secret', @@ -50,14 +49,13 @@ class TestAuthenticationAsync(base.TCTest): ['test:xyz'], ) client = subjectAsync.Auth({ - 'rootUrl': self.real_root_url, 'credentials': tempCred, }, session=session) - result = await client.testAuthenticate({ + result = client.testAuthenticate({ 'clientScopes': ['test:*'], 'requiredScopes': ['test:xyz'], }) self.assertEqual(result, {'scopes': ['test:xyz'], 'clientId': 'tester'}) - loop.run_until_complete(x()) + loop.run_until_complete diff --git a/third_party/python/taskcluster/test/test_client.py b/third_party/python/taskcluster/test/test_client.py index afe4b254bca3..8e0f35abdab2 100644 --- a/third_party/python/taskcluster/test/test_client.py +++ b/third_party/python/taskcluster/test/test_client.py @@ -7,7 +7,6 @@ from six.moves import urllib import os import re import json -import copy import mock import httmock @@ -17,7 +16,6 @@ import base import taskcluster.auth as subject import taskcluster.exceptions as exc import taskcluster.utils as utils -import taskcluster_urls as liburls class ClientTest(base.TCTest): @@ -47,7 +45,7 @@ class ClientTest(base.TCTest): ] self.apiRef = base.createApiRef(entries=entries) self.clientClass = subject.createApiClient('testApi', self.apiRef) - self.client = self.clientClass({'rootUrl': self.test_root_url}) + self.client = self.clientClass() # Patch time.sleep so that we don't delay tests sleepPatcher = mock.patch('time.sleep') sleepSleep = sleepPatcher.start() @@ -58,32 +56,6 @@ class ClientTest(base.TCTest): time.sleep = self.realTimeSleep -class TestConstructorOptions(ClientTest): - - def test_baseUrl_not_allowed(self): - with self.assertRaises(exc.TaskclusterFailure): - self.clientClass({'baseUrl': 'https://bogus.net'}) - - def test_rootUrl_set_correctly(self): - client = self.clientClass({'rootUrl': self.test_root_url}) - self.assertEqual(client.options['rootUrl'], self.test_root_url) - - def test_apiVersion_set_correctly(self): - client = self.clientClass({'rootUrl': self.test_root_url}) - self.assertEqual(client.apiVersion, 'v1') - - def test_apiVersion_set_correctly_default(self): - apiRef = copy.deepcopy(self.apiRef) - del apiRef['reference']['apiVersion'] - clientClass = subject.createApiClient('testApi', apiRef) - client = clientClass({'rootUrl': self.test_root_url}) - self.assertEqual(client.apiVersion, 'v1') - - def test_serviceName_set_correctly(self): - client = self.clientClass({'rootUrl': self.test_root_url}) - self.assertEqual(client.serviceName, 'fake') - - class TestSubArgsInRoute(ClientTest): def test_valid_no_subs(self): @@ -196,10 +168,10 @@ class TestProcessArgs(ClientTest): def test_calling_convention_1_with_payload(self): params, payload, query, _, _ = self.client._processArgs( - {'args': ['k1', 'k2'], 'name': 'test', 'input': True}, - 1, - 2, - {'A': 123} + {'args': ['k1', 'k2'], 'name': 'test', 'input': True}, + 1, + 2, + {'A': 123} ) self.assertEqual(params, {'k1': 1, 'k2': 2}) self.assertEqual(payload, {'A': 123}) @@ -213,8 +185,8 @@ class TestProcessArgs(ClientTest): def test_calling_convention_2_with_payload(self): params, payload, query, _, _ = self.client._processArgs( - {'args': ['k1', 'k2'], 'name': 'test', 'input': True}, - {'A': 123}, k1=1, k2=2 + {'args': ['k1', 'k2'], 'name': 'test', 'input': True}, + {'A': 123}, k1=1, k2=2 ) self.assertEqual(params, {'k1': 1, 'k2': 2}) self.assertEqual(payload, {'A': 123}) @@ -222,8 +194,8 @@ class TestProcessArgs(ClientTest): def test_calling_convention_3_without_payload_without_query(self): params, payload, query, _, _ = self.client._processArgs( - {'args': ['k1', 'k2'], 'name': 'test'}, - params={'k1': 1, 'k2': 2} + {'args': ['k1', 'k2'], 'name': 'test'}, + params={'k1': 1, 'k2': 2} ) self.assertEqual(params, {'k1': 1, 'k2': 2}) self.assertEqual(payload, None) @@ -231,9 +203,9 @@ class TestProcessArgs(ClientTest): def test_calling_convention_3_with_payload_without_query(self): params, payload, query, _, _ = self.client._processArgs( - {'args': ['k1', 'k2'], 'name': 'test'}, - params={'k1': 1, 'k2': 2}, - payload={'A': 123} + {'args': ['k1', 'k2'], 'name': 'test'}, + params={'k1': 1, 'k2': 2}, + payload={'A': 123} ) self.assertEqual(params, {'k1': 1, 'k2': 2}) self.assertEqual(payload, {'A': 123}) @@ -241,10 +213,10 @@ class TestProcessArgs(ClientTest): def test_calling_convention_3_with_payload_with_query(self): params, payload, query, _, _ = self.client._processArgs( - {'args': ['k1', 'k2'], 'name': 'test'}, - params={'k1': 1, 'k2': 2}, - payload={'A': 123}, - query={'B': 456} + {'args': ['k1', 'k2'], 'name': 'test'}, + params={'k1': 1, 'k2': 2}, + payload={'A': 123}, + query={'B': 456} ) self.assertEqual(params, {'k1': 1, 'k2': 2}) self.assertEqual(payload, {'A': 123}) @@ -252,9 +224,9 @@ class TestProcessArgs(ClientTest): def test_calling_convention_3_without_payload_with_query(self): params, payload, query, _, _ = self.client._processArgs( - {'args': ['k1', 'k2'], 'name': 'test'}, - params={'k1': 1, 'k2': 2}, - query={'B': 456} + {'args': ['k1', 'k2'], 'name': 'test'}, + params={'k1': 1, 'k2': 2}, + query={'B': 456} ) self.assertEqual(params, {'k1': 1, 'k2': 2}) self.assertEqual(payload, None) @@ -262,11 +234,11 @@ class TestProcessArgs(ClientTest): def test_calling_convention_3_with_positional_arguments_with_payload_with_query(self): params, payload, query, _, _ = self.client._processArgs( - {'args': ['k1', 'k2'], 'name': 'test'}, - 1, - 2, - query={'B': 456}, - payload={'A': 123} + {'args': ['k1', 'k2'], 'name': 'test'}, + 1, + 2, + query={'B': 456}, + payload={'A': 123} ) self.assertEqual(params, {'k1': 1, 'k2': 2}) self.assertEqual(payload, {'A': 123}) @@ -286,12 +258,12 @@ class TestProcessArgs(ClientTest): def test_calling_convention_3_with_pos_args_same_as_param_kwarg_dict_vals_with_payload_with_query(self): with self.assertRaises(exc.TaskclusterFailure): params, payload, query, _, _ = self.client._processArgs( - {'args': ['k1', 'k2'], 'name': 'test'}, - 1, - 2, - params={'k1': 1, 'k2': 2}, - query={'B': 456}, - payload={'A': 123} + {'args': ['k1', 'k2'], 'name': 'test'}, + 1, + 2, + params={'k1': 1, 'k2': 2}, + query={'B': 456}, + payload={'A': 123} ) @@ -312,8 +284,6 @@ class ObjWithDotJson(object): class TestMakeHttpRequest(ClientTest): - apiPath = liburls.api(ClientTest.test_root_url, 'fake', 'v1', 'test') - def setUp(self): ClientTest.setUp(self) @@ -323,8 +293,8 @@ class TestMakeHttpRequest(ClientTest): expected = {'test': 'works'} p.return_value = ObjWithDotJson(200, expected) - v = self.client._makeHttpRequest('GET', 'test', None) - p.assert_called_once_with('GET', self.apiPath, None, mock.ANY) + v = self.client._makeHttpRequest('GET', 'http://www.example.com', None) + p.assert_called_once_with('GET', 'http://www.example.com', None, mock.ANY) self.assertEqual(expected, v) def test_success_first_try_payload(self): @@ -332,8 +302,8 @@ class TestMakeHttpRequest(ClientTest): expected = {'test': 'works'} p.return_value = ObjWithDotJson(200, expected) - v = self.client._makeHttpRequest('GET', 'test', {'payload': 2}) - p.assert_called_once_with('GET', self.apiPath, + v = self.client._makeHttpRequest('GET', 'http://www.example.com', {'payload': 2}) + p.assert_called_once_with('GET', 'http://www.example.com', utils.dumpJson({'payload': 2}), mock.ANY) self.assertEqual(expected, v) @@ -348,10 +318,10 @@ class TestMakeHttpRequest(ClientTest): ObjWithDotJson(200, expected) ] p.side_effect = sideEffect - expectedCalls = [mock.call('GET', self.apiPath, None, mock.ANY) + expectedCalls = [mock.call('GET', 'http://www.example.com', None, mock.ANY) for x in range(self.client.options['maxRetries'])] - v = self.client._makeHttpRequest('GET', 'test', None) + v = self.client._makeHttpRequest('GET', 'http://www.example.com', None) p.assert_has_calls(expectedCalls) self.assertEqual(expected, v) @@ -373,12 +343,12 @@ class TestMakeHttpRequest(ClientTest): ObjWithDotJson(200, {'got this': 'wrong'}) ] p.side_effect = sideEffect - expectedCalls = [mock.call('GET', self.apiPath, None, mock.ANY) + expectedCalls = [mock.call('GET', 'http://www.example.com', None, mock.ANY) for x in range(self.client.options['maxRetries'] + 1)] with self.assertRaises(exc.TaskclusterRestFailure): try: - self.client._makeHttpRequest('GET', 'test', None) + self.client._makeHttpRequest('GET', 'http://www.example.com', None) except exc.TaskclusterRestFailure as err: self.assertEqual('msg', str(err)) self.assertEqual(500, err.status_code) @@ -397,34 +367,43 @@ class TestMakeHttpRequest(ClientTest): ObjWithDotJson(200, expected) ] p.side_effect = sideEffect - expectedCalls = [mock.call('GET', self.apiPath, None, mock.ANY) + expectedCalls = [mock.call('GET', 'http://www.example.com', None, mock.ANY) for x in range(self.client.options['maxRetries'])] - v = self.client._makeHttpRequest('GET', 'test', None) + v = self.client._makeHttpRequest('GET', 'http://www.example.com', None) p.assert_has_calls(expectedCalls) self.assertEqual(expected, v) def test_failure_status_code(self): with mock.patch.object(utils, 'makeSingleHttpRequest') as p: p.return_value = ObjWithDotJson(500, None) - expectedCalls = [mock.call('GET', self.apiPath, None, mock.ANY) + expectedCalls = [mock.call('GET', 'http://www.example.com', None, mock.ANY) for x in range(self.client.options['maxRetries'])] with self.assertRaises(exc.TaskclusterRestFailure): - self.client._makeHttpRequest('GET', 'test', None) + self.client._makeHttpRequest('GET', 'http://www.example.com', None) p.assert_has_calls(expectedCalls) def test_failure_connection_errors(self): with mock.patch.object(utils, 'makeSingleHttpRequest') as p: p.side_effect = requests.exceptions.RequestException - expectedCalls = [mock.call('GET', self.apiPath, None, mock.ANY) + expectedCalls = [mock.call('GET', 'http://www.example.com', None, mock.ANY) for x in range(self.client.options['maxRetries'])] with self.assertRaises(exc.TaskclusterConnectionError): - self.client._makeHttpRequest('GET', 'test', None) + self.client._makeHttpRequest('GET', 'http://www.example.com', None) p.assert_has_calls(expectedCalls) class TestOptions(ClientTest): + def setUp(self): + ClientTest.setUp(self) + self.clientClass2 = subject.createApiClient('testApi', base.createApiRef()) + self.client2 = self.clientClass2({'baseUrl': 'http://notlocalhost:5888/v2'}) + + def test_defaults_should_work(self): + self.assertEqual(self.client.options['baseUrl'], 'https://fake.taskcluster.net/v1') + self.assertEqual(self.client2.options['baseUrl'], 'http://notlocalhost:5888/v2') + def test_change_default_doesnt_change_previous_instances(self): prevMaxRetries = subject._defaultConfig['maxRetries'] with mock.patch.dict(subject._defaultConfig, {'maxRetries': prevMaxRetries + 1}): @@ -436,10 +415,7 @@ class TestOptions(ClientTest): 'clientId': u"\U0001F4A9", } with self.assertRaises(exc.TaskclusterAuthFailure): - subject.Auth({ - 'rootUrl': self.real_root_url, - 'credentials': badCredentials, - }) + subject.Auth({'credentials': badCredentials}) class TestMakeApiCall(ClientTest): @@ -542,13 +518,13 @@ class TestTopicExchange(ClientTest): self.assertEqual(expected, actual['routingKeyPattern']) def test_exchange(self): - expected = 'exchange/taskcluster-fake/v1/topicExchange' + expected = 'test/v1/topicExchange' actual = self.client.topicName('') self.assertEqual(expected, actual['exchange']) def test_exchange_trailing_slash(self): - self.client.options['exchangePrefix'] = 'exchange/taskcluster-fake2/v1/' - expected = 'exchange/taskcluster-fake2/v1/topicExchange' + self.client.options['exchangePrefix'] = 'test/v1/' + expected = 'test/v1/topicExchange' actual = self.client.topicName('') self.assertEqual(expected, actual['exchange']) @@ -580,17 +556,18 @@ class TestTopicExchange(ClientTest): class TestBuildUrl(ClientTest): - apiPath = liburls.api(ClientTest.test_root_url, 'fake', 'v1', 'two_args_no_input/arg0/arg1') - def test_build_url_positional(self): + expected = 'https://fake.taskcluster.net/v1/two_args_no_input/arg0/arg1' actual = self.client.buildUrl('two_args_no_input', 'arg0', 'arg1') - self.assertEqual(self.apiPath, actual) + self.assertEqual(expected, actual) def test_build_url_keyword(self): + expected = 'https://fake.taskcluster.net/v1/two_args_no_input/arg0/arg1' actual = self.client.buildUrl('two_args_no_input', arg0='arg0', arg1='arg1') - self.assertEqual(self.apiPath, actual) + self.assertEqual(expected, actual) def test_build_url_query_string(self): + expected = 'https://fake.taskcluster.net/v1/two_args_no_input/arg0/arg1?qs0=1' actual = self.client.buildUrl( 'two_args_no_input', params={ @@ -599,7 +576,7 @@ class TestBuildUrl(ClientTest): }, query={'qs0': 1} ) - self.assertEqual(self.apiPath + '?qs0=1', actual) + self.assertEqual(expected, actual) def test_fails_to_build_url_for_missing_method(self): with self.assertRaises(exc.TaskclusterFailure): @@ -612,17 +589,17 @@ class TestBuildUrl(ClientTest): class TestBuildSignedUrl(ClientTest): - apiPath = liburls.api(ClientTest.test_root_url, 'fake', 'v1', 'two_args_no_input/arg0/arg1') - def test_builds_surl_positional(self): + expected = 'https://fake.taskcluster.net/v1/two_args_no_input/arg0/arg1?bewit=X' actual = self.client.buildSignedUrl('two_args_no_input', 'arg0', 'arg1') actual = re.sub('bewit=[^&]*', 'bewit=X', actual) - self.assertEqual(self.apiPath + '?bewit=X', actual) + self.assertEqual(expected, actual) def test_builds_surl_keyword(self): + expected = 'https://fake.taskcluster.net/v1/two_args_no_input/arg0/arg1?bewit=X' actual = self.client.buildSignedUrl('two_args_no_input', arg0='arg0', arg1='arg1') actual = re.sub('bewit=[^&]*', 'bewit=X', actual) - self.assertEqual(self.apiPath + '?bewit=X', actual) + self.assertEqual(expected, actual) class TestMockHttpCalls(ClientTest): @@ -645,30 +622,30 @@ class TestMockHttpCalls(ClientTest): def test_no_args_no_input(self): with httmock.HTTMock(self.fakeSite): self.client.no_args_no_input() - self.assertEqual(self.gotUrl, 'https://tc-tests.example.com/api/fake/v1/no_args_no_input') + self.assertEqual(self.gotUrl, 'https://fake.taskcluster.net/v1/no_args_no_input') def test_two_args_no_input(self): with httmock.HTTMock(self.fakeSite): self.client.two_args_no_input('1', '2') - self.assertEqual(self.gotUrl, 'https://tc-tests.example.com/api/fake/v1/two_args_no_input/1/2') + self.assertEqual(self.gotUrl, 'https://fake.taskcluster.net/v1/two_args_no_input/1/2') def test_no_args_with_input(self): with httmock.HTTMock(self.fakeSite): self.client.no_args_with_input({'x': 1}) - self.assertEqual(self.gotUrl, 'https://tc-tests.example.com/api/fake/v1/no_args_with_input') + self.assertEqual(self.gotUrl, 'https://fake.taskcluster.net/v1/no_args_with_input') self.assertEqual(json.loads(self.gotRequest.body), {"x": 1}) def test_no_args_with_empty_input(self): with httmock.HTTMock(self.fakeSite): self.client.no_args_with_input({}) - self.assertEqual(self.gotUrl, 'https://tc-tests.example.com/api/fake/v1/no_args_with_input') + self.assertEqual(self.gotUrl, 'https://fake.taskcluster.net/v1/no_args_with_input') self.assertEqual(json.loads(self.gotRequest.body), {}) def test_two_args_with_input(self): with httmock.HTTMock(self.fakeSite): self.client.two_args_with_input('a', 'b', {'x': 1}) self.assertEqual(self.gotUrl, - 'https://tc-tests.example.com/api/fake/v1/two_args_with_input/a/b') + 'https://fake.taskcluster.net/v1/two_args_with_input/a/b') self.assertEqual(json.loads(self.gotRequest.body), {"x": 1}) def test_kwargs(self): @@ -676,7 +653,7 @@ class TestMockHttpCalls(ClientTest): self.client.two_args_with_input( {'x': 1}, arg0='a', arg1='b') self.assertEqual(self.gotUrl, - 'https://tc-tests.example.com/api/fake/v1/two_args_with_input/a/b') + 'https://fake.taskcluster.net/v1/two_args_with_input/a/b') self.assertEqual(json.loads(self.gotRequest.body), {"x": 1}) @@ -690,14 +667,14 @@ class TestAuthentication(base.TCTest): @httmock.all_requests def auth_response(url, request): self.assertEqual(urllib.parse.urlunsplit(url), - 'https://tc-tests.example.com/api/auth/v1/clients/abc') + 'https://auth.taskcluster.net/v1/clients/abc') self.failIf('Authorization' in request.headers) headers = {'content-type': 'application/json'} content = {"clientId": "abc"} return httmock.response(200, content, headers, None, 5, request) with httmock.HTTMock(auth_response): - client = subject.Auth({"rootUrl": "https://tc-tests.example.com", "credentials": {}}) + client = subject.Auth({"credentials": {}}) result = client.client('abc') self.assertEqual(result, {"clientId": "abc"}) @@ -705,7 +682,6 @@ class TestAuthentication(base.TCTest): """we can call methods which require authentication with valid permacreds""" client = subject.Auth({ - 'rootUrl': self.real_root_url, 'credentials': { 'clientId': 'tester', 'accessToken': 'no-secret', @@ -719,7 +695,6 @@ class TestAuthentication(base.TCTest): def test_permacred_simple_authorizedScopes(self): client = subject.Auth({ - 'rootUrl': self.real_root_url, 'credentials': { 'clientId': 'tester', 'accessToken': 'no-secret', @@ -736,7 +711,6 @@ class TestAuthentication(base.TCTest): def test_unicode_permacred_simple(self): """Unicode strings that encode to ASCII in credentials do not cause issues""" client = subject.Auth({ - 'rootUrl': self.real_root_url, 'credentials': { 'clientId': u'tester', 'accessToken': u'no-secret', @@ -752,7 +726,6 @@ class TestAuthentication(base.TCTest): """Unicode strings that do not encode to ASCII in credentials cause issues""" with self.assertRaises(exc.TaskclusterAuthFailure): subject.Auth({ - 'rootUrl': self.test_root_url, 'credentials': { 'clientId': u"\U0001F4A9", 'accessToken': u"\U0001F4A9", @@ -762,7 +735,6 @@ class TestAuthentication(base.TCTest): def test_permacred_insufficient_scopes(self): """A call with insufficient scopes results in an error""" client = subject.Auth({ - 'rootUrl': self.real_root_url, 'credentials': { 'clientId': 'tester', 'accessToken': 'no-secret', @@ -787,7 +759,6 @@ class TestAuthentication(base.TCTest): ['test:xyz'], ) client = subject.Auth({ - 'rootUrl': self.real_root_url, 'credentials': tempCred, }) @@ -807,7 +778,6 @@ class TestAuthentication(base.TCTest): name='credName' ) client = subject.Auth({ - 'rootUrl': self.real_root_url, 'credentials': tempCred, }) @@ -826,7 +796,6 @@ class TestAuthentication(base.TCTest): ['test:xyz:*'], ) client = subject.Auth({ - 'rootUrl': self.real_root_url, 'credentials': tempCred, 'authorizedScopes': ['test:xyz:abc'], }) @@ -848,7 +817,6 @@ class TestAuthentication(base.TCTest): name='credName' ) client = subject.Auth({ - 'rootUrl': self.real_root_url, 'credentials': tempCred, 'authorizedScopes': ['test:xyz:abc'], }) @@ -863,7 +831,6 @@ class TestAuthentication(base.TCTest): def test_signed_url(self): """we can use a signed url built with the python client""" client = subject.Auth({ - 'rootUrl': self.real_root_url, 'credentials': { 'clientId': 'tester', 'accessToken': 'no-secret', @@ -881,7 +848,6 @@ class TestAuthentication(base.TCTest): def test_signed_url_bad_credentials(self): client = subject.Auth({ - 'rootUrl': self.real_root_url, 'credentials': { 'clientId': 'tester', 'accessToken': 'wrong-secret', @@ -902,7 +868,6 @@ class TestAuthentication(base.TCTest): ['test:*'], ) client = subject.Auth({ - 'rootUrl': self.real_root_url, 'credentials': tempCred, }) signedUrl = client.buildSignedUrl('testAuthenticateGet') @@ -916,7 +881,6 @@ class TestAuthentication(base.TCTest): def test_signed_url_authorizedScopes(self): client = subject.Auth({ - 'rootUrl': self.real_root_url, 'credentials': { 'clientId': 'tester', 'accessToken': 'no-secret', @@ -941,7 +905,6 @@ class TestAuthentication(base.TCTest): ['test:*'], ) client = subject.Auth({ - 'rootUrl': self.real_root_url, 'credentials': tempCred, 'authorizedScopes': ['test:authenticate-get'], }) diff --git a/third_party/python/taskcluster/test/test_utils.py b/third_party/python/taskcluster/test/test_utils.py index 31328199d87e..f23903c11f77 100644 --- a/third_party/python/taskcluster/test/test_utils.py +++ b/third_party/python/taskcluster/test/test_utils.py @@ -1,6 +1,5 @@ import datetime import uuid -import os import taskcluster.utils as subject import dateutil.parser @@ -339,101 +338,3 @@ class TestIsExpired(TestCase): } """) self.assertEqual(isExpired, True) - - -class TestFromEnv(TestCase): - - def clear_env(self): - for v in 'ROOT_URL', 'CLIENT_ID', 'ACCESS_TOKEN', 'CERTIFICATE': - v = 'TASKCLUSTER_' + v - if v in os.environ: - del os.environ[v] - - @mock.patch.dict(os.environ) - def test_empty(self): - self.clear_env() - self.assertEqual(subject.optionsFromEnvironment(), {}) - - @mock.patch.dict(os.environ) - def test_all(self): - os.environ['TASKCLUSTER_ROOT_URL'] = 'https://tc.example.com' - os.environ['TASKCLUSTER_CLIENT_ID'] = 'me' - os.environ['TASKCLUSTER_ACCESS_TOKEN'] = 'shave-and-a-haircut' - os.environ['TASKCLUSTER_CERTIFICATE'] = '{"bits":2}' - self.assertEqual(subject.optionsFromEnvironment(), { - 'rootUrl': 'https://tc.example.com', - 'credentials': { - 'clientId': 'me', - 'accessToken': 'shave-and-a-haircut', - 'certificate': '{"bits":2}', - }, - }) - - @mock.patch.dict(os.environ) - def test_cred_only(self): - os.environ['TASKCLUSTER_ACCESS_TOKEN'] = 'shave-and-a-haircut' - self.assertEqual(subject.optionsFromEnvironment(), { - 'credentials': { - 'accessToken': 'shave-and-a-haircut', - }, - }) - - @mock.patch.dict(os.environ) - def test_rooturl_only(self): - os.environ['TASKCLUSTER_ROOT_URL'] = 'https://tc.example.com' - self.assertEqual(subject.optionsFromEnvironment(), { - 'rootUrl': 'https://tc.example.com', - }) - - @mock.patch.dict(os.environ) - def test_default_rooturl(self): - os.environ['TASKCLUSTER_CLIENT_ID'] = 'me' - os.environ['TASKCLUSTER_ACCESS_TOKEN'] = 'shave-and-a-haircut' - os.environ['TASKCLUSTER_CERTIFICATE'] = '{"bits":2}' - self.assertEqual( - subject.optionsFromEnvironment({'rootUrl': 'https://other.example.com'}), { - 'rootUrl': 'https://other.example.com', - 'credentials': { - 'clientId': 'me', - 'accessToken': 'shave-and-a-haircut', - 'certificate': '{"bits":2}', - }, - }) - - @mock.patch.dict(os.environ) - def test_default_rooturl_overridden(self): - os.environ['TASKCLUSTER_ROOT_URL'] = 'https://tc.example.com' - self.assertEqual( - subject.optionsFromEnvironment({'rootUrl': 'https://other.example.com'}), - {'rootUrl': 'https://tc.example.com'}) - - @mock.patch.dict(os.environ) - def test_default_creds(self): - os.environ['TASKCLUSTER_ROOT_URL'] = 'https://tc.example.com' - os.environ['TASKCLUSTER_ACCESS_TOKEN'] = 'shave-and-a-haircut' - os.environ['TASKCLUSTER_CERTIFICATE'] = '{"bits":2}' - self.assertEqual( - subject.optionsFromEnvironment({'credentials': {'clientId': 'them'}}), { - 'rootUrl': 'https://tc.example.com', - 'credentials': { - 'clientId': 'them', - 'accessToken': 'shave-and-a-haircut', - 'certificate': '{"bits":2}', - }, - }) - - @mock.patch.dict(os.environ) - def test_default_creds_overridden(self): - os.environ['TASKCLUSTER_ROOT_URL'] = 'https://tc.example.com' - os.environ['TASKCLUSTER_CLIENT_ID'] = 'me' - os.environ['TASKCLUSTER_ACCESS_TOKEN'] = 'shave-and-a-haircut' - os.environ['TASKCLUSTER_CERTIFICATE'] = '{"bits":2}' - self.assertEqual( - subject.optionsFromEnvironment({'credentials': {'clientId': 'them'}}), { - 'rootUrl': 'https://tc.example.com', - 'credentials': { - 'clientId': 'me', - 'accessToken': 'shave-and-a-haircut', - 'certificate': '{"bits":2}', - }, - })