зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1277595 - Create initial nightly graph, r=dustin
MozReview-Commit-ID: 2rZYddnGdvh --HG-- rename : taskcluster/ci/legacy/tasks/build.yml => taskcluster/ci/nightly-fennec/build.yml rename : taskcluster/ci/legacy/tasks/docker_build.yml => taskcluster/ci/nightly-fennec/docker_build.yml rename : taskcluster/ci/legacy/tasks/builds/mobile_base.yml => taskcluster/ci/nightly-fennec/mobile_base.yml rename : taskcluster/ci/legacy/routes.json => taskcluster/ci/nightly-fennec/routes.json extra : rebase_source : dcdb4f106bbaa7173e6b49c67135234640c88235
This commit is contained in:
amiyaguchi@mozilla.com
Родитель
2d162586c0
Коммит
0046838ade
|
|
@ -0,0 +1,66 @@
|
|||
$inherits:
|
||||
from: 'mobile_base.yml'
|
||||
variables:
|
||||
build_name: 'android'
|
||||
build_type: 'opt'
|
||||
task:
|
||||
metadata:
|
||||
name: '[TC] Android armv7 API 15+'
|
||||
description: 'Android armv7 API 15+'
|
||||
|
||||
workerType: android-api-15
|
||||
|
||||
routes:
|
||||
- 'index.buildbot.branches.{{project}}.android-api-15'
|
||||
- 'index.buildbot.revisions.{{head_rev}}.{{project}}.android-api-15'
|
||||
|
||||
scopes:
|
||||
- 'docker-worker:cache:level-{{level}}-{{project}}-build-android-api-15-workspace'
|
||||
- 'docker-worker:cache:tooltool-cache'
|
||||
- 'docker-worker:relengapi-proxy:tooltool.download.internal'
|
||||
- 'docker-worker:relengapi-proxy:tooltool.download.public'
|
||||
|
||||
payload:
|
||||
cache:
|
||||
level-{{level}}-{{project}}-build-android-api-15-workspace: '/home/worker/workspace'
|
||||
tooltool-cache: '/home/worker/tooltool-cache'
|
||||
|
||||
features:
|
||||
relengAPIProxy: true
|
||||
|
||||
env:
|
||||
# inputs to mozharness
|
||||
MOZHARNESS_SCRIPT: 'mozharness/scripts/fx_desktop_build.py'
|
||||
# TODO: make these additional configuration files go away
|
||||
MOZHARNESS_CONFIG: >
|
||||
builds/releng_base_android_64_builds.py
|
||||
disable_signing.py
|
||||
platform_supports_post_upload_to_latest.py
|
||||
taskcluster_nightly.py
|
||||
MOZHARNESS_ACTIONS: "get-secrets build multi-l10n update"
|
||||
MH_CUSTOM_BUILD_VARIANT_CFG: api-15
|
||||
MH_BRANCH: {{project}}
|
||||
MH_BUILD_POOL: taskcluster
|
||||
TOOLTOOL_CACHE: '/home/worker/tooltool-cache'
|
||||
|
||||
command: ["/bin/bash", "bin/build.sh"]
|
||||
|
||||
extra:
|
||||
treeherderEnv:
|
||||
- production
|
||||
- staging
|
||||
treeherder:
|
||||
machine:
|
||||
# see https://github.com/mozilla/treeherder/blob/master/ui/js/values.js
|
||||
platform: android-4-0-armv7-api15
|
||||
groupSymbol: tc
|
||||
groupName: Submitted by taskcluster
|
||||
symbol: B
|
||||
tier: 2
|
||||
# Rather then enforcing particular conventions we require that all build
|
||||
# tasks provide the "build" extra field to specify where the build and tests
|
||||
# files are located.
|
||||
locations:
|
||||
build: 'public/build/target.apk'
|
||||
mozharness: 'public/build/mozharness.zip'
|
||||
test_packages: 'public/build/target.test_packages.json'
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
# This is the "base" task which contains the common values all builds must
|
||||
# provide.
|
||||
---
|
||||
taskId: {{build_slugid}}
|
||||
|
||||
task:
|
||||
created:
|
||||
relative-datestamp: "0 seconds"
|
||||
deadline:
|
||||
relative-datestamp: "24 hours"
|
||||
metadata:
|
||||
source: '{{source}}'
|
||||
owner: mozilla-taskcluster-maintenance@mozilla.com
|
||||
|
||||
tags:
|
||||
createdForUser: {{owner}}
|
||||
|
||||
provisionerId: aws-provisioner-v1
|
||||
schedulerId: task-graph-scheduler
|
||||
|
||||
payload:
|
||||
# Two hours is long but covers edge cases (and matches bb based infra)
|
||||
maxRunTime: 7200
|
||||
|
||||
env:
|
||||
# Common environment variables for checking out gecko
|
||||
GECKO_BASE_REPOSITORY: '{{base_repository}}'
|
||||
GECKO_HEAD_REPOSITORY: '{{head_repository}}'
|
||||
GECKO_HEAD_REV: '{{head_rev}}'
|
||||
GECKO_HEAD_REF: '{{head_ref}}'
|
||||
TOOLTOOL_REPO: 'https://github.com/mozilla/build-tooltool'
|
||||
TOOLTOOL_REV: 'master'
|
||||
MOZ_BUILD_DATE: '{{pushdate}}'
|
||||
MOZ_SCM_LEVEL: '{{level}}'
|
||||
|
||||
extra:
|
||||
build_product: '{{build_product}}'
|
||||
build_name: '{{build_name}}'
|
||||
build_type: '{{build_type}}'
|
||||
index:
|
||||
rank: {{rank}}
|
||||
treeherder:
|
||||
jobKind: build
|
||||
groupSymbol: tc
|
||||
groupName: Submitted by taskcluster
|
||||
symbol: B
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
$inherits:
|
||||
from: 'build.yml'
|
||||
|
||||
|
||||
task:
|
||||
workerType: b2gbuild
|
||||
|
||||
routes:
|
||||
- 'index.gecko.v1.{{project}}.revision.linux.{{head_rev}}.{{build_name}}.{{build_type}}'
|
||||
- 'index.gecko.v1.{{project}}.latest.linux.{{build_name}}.{{build_type}}'
|
||||
|
||||
scopes:
|
||||
# docker build tasks use tc-vcs so include the scope.
|
||||
- 'docker-worker:cache:level-{{level}}-{{project}}-tc-vcs'
|
||||
|
||||
payload:
|
||||
|
||||
cache:
|
||||
level-{{level}}-{{project}}-tc-vcs: '/home/worker/.tc-vcs'
|
||||
|
||||
# All docker builds share a common artifact directory for ease of uploading.
|
||||
artifacts:
|
||||
'public/build':
|
||||
type: directory
|
||||
path: '/home/worker/artifacts/'
|
||||
expires:
|
||||
relative-datestamp: '1 year'
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
implementation: 'taskgraph.task.nightly_fennec:NightlyFennecTask'
|
||||
nightly_fennec_path: '.'
|
||||
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
$inherits:
|
||||
from: 'docker_build.yml'
|
||||
variables:
|
||||
build_product: 'mobile'
|
||||
docker-image: desktop-build
|
||||
task:
|
||||
payload:
|
||||
image:
|
||||
type: 'task-image'
|
||||
path: 'public/image.tar'
|
||||
taskId:
|
||||
task-reference: "<docker-image>"
|
||||
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
{
|
||||
"routes": [
|
||||
"{index}.gecko.v2.{project}.revision.{head_rev}.{build_product}.{build_name}-{build_type}",
|
||||
"{index}.gecko.v2.{project}.pushdate.{year}.{month}.{day}.{pushdate}.{build_product}.{build_name}-{build_type}",
|
||||
"{index}.gecko.v2.{project}.latest.{build_product}.{build_name}-{build_type}"
|
||||
],
|
||||
"nightly": [
|
||||
"{index}.gecko.v2.{project}.nightly.{year}.{month}.{day}.revision.{head_rev}.{build_product}.{build_name}-{build_type}",
|
||||
"{index}.gecko.v2.{project}.nightly.{year}.{month}.{day}.latest.{build_product}.{build_name}-{build_type}",
|
||||
"{index}.gecko.v2.{project}.nightly.revision.{head_rev}.{build_product}.{build_name}-{build_type}",
|
||||
"{index}.gecko.v2.{project}.nightly.latest.{build_product}.{build_name}-{build_type}"
|
||||
],
|
||||
"l10n": [
|
||||
"{index}.gecko.v2.{project}.revision.{head_rev}.{build_product}-l10n.{build_name}-{build_type}.{locale}",
|
||||
"{index}.gecko.v2.{project}.pushdate.{year}.{month}.{day}.{pushdate}.{build_product}-l10n.{build_name}-{build_type}.{locale}",
|
||||
"{index}.gecko.v2.{project}.latest.{build_product}-l10n.{build_name}-{build_type}.{locale}"
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
implementation: 'taskgraph.task.signing:SigningTask'
|
||||
signing_path: '.'
|
||||
kind-dependencies:
|
||||
- nightly-fennec
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
task:
|
||||
provisionerId: "aws-provisioner-v1"
|
||||
workerType: "desktop-test"
|
||||
schedulerId: task-graph-scheduler
|
||||
|
||||
created:
|
||||
relative-datestamp: "0 seconds"
|
||||
deadline:
|
||||
relative-datestamp: "24 hours"
|
||||
payload:
|
||||
image: "ubuntu:13.10"
|
||||
command:
|
||||
- "/bin/bash"
|
||||
- "-c"
|
||||
- "echo \"hello World\""
|
||||
maxRunTime: 600
|
||||
metadata:
|
||||
name: "Signing Fennec Nightly Task"
|
||||
description: "Markdown description of **what** this task does"
|
||||
owner: "amiyaguchi@mozilla.com"
|
||||
source: "https://tools.taskcluster.net/task-creator/"
|
||||
|
||||
|
|
@ -84,3 +84,13 @@ def target_tasks_ash_tasks(full_task_graph, parameters):
|
|||
return False
|
||||
return True
|
||||
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
|
||||
|
||||
|
||||
@_target_task('nightly_fennec')
|
||||
def target_tasks_nightly(full_task_graph, parameters):
|
||||
"""Select the set of tasks required for a nightly build of fennec. The
|
||||
nightly build process involves a pipeline of builds, signing,
|
||||
and, eventually, uploading the tasks to balrog."""
|
||||
return [t.label for t in full_task_graph.tasks.itervalues()
|
||||
if t.attributes.get('kind') in ['nightly-fennec', 'signing']]
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,113 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
import logging
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
|
||||
from . import base
|
||||
from .legacy import query_vcs_info, decorate_task_json_routes, gaia_info, \
|
||||
mklabel
|
||||
from taskgraph.util.templates import Templates
|
||||
from taskgraph.util.docker import docker_image
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
|
||||
ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
|
||||
INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'
|
||||
|
||||
class NightlyFennecTask(base.Task):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.task_dict = kwargs.pop('task_dict')
|
||||
super(NightlyFennecTask, self).__init__(*args, **kwargs)
|
||||
|
||||
@classmethod
|
||||
def load_tasks(cls, kind, path, config, params, loaded_tasks):
|
||||
root = os.path.abspath(os.path.join(path, config[
|
||||
'nightly_fennec_path']))
|
||||
|
||||
project = params['project']
|
||||
|
||||
# Set up the parameters, including the time of the build
|
||||
push_epoch = int(time.time())
|
||||
vcs_info = query_vcs_info(params['head_repository'], params['head_rev'])
|
||||
changed_files = set()
|
||||
if vcs_info:
|
||||
push_epoch = vcs_info.pushdate
|
||||
|
||||
logger.debug(
|
||||
'{} commits influencing task scheduling:'.format(len(vcs_info.changesets)))
|
||||
for c in vcs_info.changesets:
|
||||
logger.debug("{cset} {desc}".format(
|
||||
cset=c['node'][0:12],
|
||||
desc=c['desc'].splitlines()[0].encode('ascii', 'ignore')))
|
||||
changed_files |= set(c['files'])
|
||||
|
||||
pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(push_epoch))
|
||||
|
||||
# Template parameters used when expanding the graph
|
||||
parameters = dict(gaia_info().items() + {
|
||||
'index': 'index',
|
||||
'project': project,
|
||||
'pushlog_id': params.get('pushlog_id', 0),
|
||||
'base_repository': params['base_repository'] or
|
||||
params['head_repository'],
|
||||
'docker_image': docker_image,
|
||||
'head_repository': params['head_repository'],
|
||||
'head_ref': params['head_ref'] or params['head_rev'],
|
||||
'head_rev': params['head_rev'],
|
||||
'pushdate': pushdate,
|
||||
'pushtime': pushdate[8:],
|
||||
'year': pushdate[0:4],
|
||||
'month': pushdate[4:6],
|
||||
'day': pushdate[6:8],
|
||||
'rank': push_epoch,
|
||||
'owner': params['owner'],
|
||||
'level': params['level'],
|
||||
'build_slugid': mklabel(),
|
||||
'source': '{repo}file/{rev}/taskcluster/ci/nightly-fennec'.format(
|
||||
repo=params['head_repository'], rev=params['head_rev']),
|
||||
'build_name': 'android',
|
||||
'build_type': 'opt',
|
||||
'build_product': 'mobile'
|
||||
}.items())
|
||||
|
||||
routes_file = os.path.join(root, 'routes.json')
|
||||
with open(routes_file) as f:
|
||||
contents = json.load(f)
|
||||
json_routes = contents['routes']
|
||||
|
||||
tasks = []
|
||||
templates = Templates(root)
|
||||
|
||||
task = templates.load('android-api-15-nightly-build.yml', parameters)
|
||||
decorate_task_json_routes(task['task'], json_routes, parameters)
|
||||
|
||||
attributes = {'kind': 'nightly-fennec'}
|
||||
tasks.append(cls(kind, 'build-nightly-fennec',
|
||||
task=task['task'], attributes=attributes,
|
||||
task_dict=task))
|
||||
|
||||
# Convert to a dictionary of tasks. The process above has invented a
|
||||
# taskId for each task, and we use those as the *labels* for the tasks;
|
||||
# taskgraph will later assign them new taskIds.
|
||||
return tasks
|
||||
|
||||
def get_dependencies(self, taskgraph):
|
||||
deps = [(label, label) for label in self.task_dict.get('requires', [])]
|
||||
|
||||
# add a dependency on an image task, if needed
|
||||
if 'docker-image' in self.task_dict:
|
||||
deps.append(('build-docker-image-{docker-image}'.format(**self.task_dict),
|
||||
'docker-image'))
|
||||
|
||||
return deps
|
||||
|
||||
def optimize(self):
|
||||
return False, None
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
import logging
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
|
||||
from . import base
|
||||
from taskgraph.util.templates import Templates
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
|
||||
ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
|
||||
INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'
|
||||
|
||||
class SigningTask(base.Task):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(SigningTask, self).__init__(*args, **kwargs)
|
||||
|
||||
@classmethod
|
||||
def load_tasks(cls, kind, path, config, params, loaded_tasks):
|
||||
root = os.path.abspath(os.path.join(path, config['signing_path']))
|
||||
|
||||
# get each nightly-fennec and add its name to this task
|
||||
fennec_tasks = [t for t in loaded_tasks if t.attributes.get('kind')
|
||||
== 'nightly-fennec']
|
||||
|
||||
tasks = []
|
||||
for fennec_task in fennec_tasks:
|
||||
templates = Templates(root)
|
||||
task = templates.load('signing.yml', {})
|
||||
attributes = {'kind': 'signing'}
|
||||
tasks.append(cls(kind, 'signing-nightly-fennec', task=task['task'],
|
||||
attributes=attributes))
|
||||
|
||||
return tasks
|
||||
|
||||
def get_dependencies(self, taskgraph):
|
||||
return [('build-nightly-fennec', 'build-nightly-fennec')]
|
||||
|
||||
def optimize(self):
|
||||
return False, None
|
||||
Загрузка…
Ссылка в новой задаче