Backed out changeset cc4009e9cecc (bug 1284005) for causing merge conflicts when merging to m-c

--HG--
extra : rebase_source : 1076f58b4aa748ddc35920d1591bb99b8c1a7358
This commit is contained in:
Carsten "Tomcat" Book 2016-07-06 11:49:59 +02:00
Родитель 96da4695c8
Коммит fb7e5c4f86
31 изменённых файлов: 76 добавлений и 163 удалений

Просмотреть файл

@ -1,9 +1,7 @@
---
task:
created:
relative-datestamp: "0 seconds"
deadline:
relative-datestamp: "24 hours"
created: '{{now}}'
deadline: '{{#from_now}}24 hours{{/from_now}}'
metadata:
name: 'Docker Image Build: {{image_name}}'
description: 'Build the docker image {{image_name}} for use by dependent tasks'
@ -45,8 +43,7 @@ task:
'{{artifact_path}}':
type: 'file'
path: '/artifacts/image.tar'
expires:
relative-datestamp: "1 year"
expires: '{{#from_now}}1 year{{/from_now}}'
extra:
treeherderEnv:
- staging

Просмотреть файл

@ -4,10 +4,8 @@
taskId: {{build_slugid}}
task:
created:
relative-datestamp: "0 seconds"
deadline:
relative-datestamp: "24 hours"
created: '{{now}}'
deadline: '{{#from_now}}24 hours{{/from_now}}'
metadata:
source: '{{source}}'
owner: mozilla-taskcluster-maintenance@mozilla.com

Просмотреть файл

@ -50,8 +50,7 @@ task:
'public/android/checkstyle/checkstyle.xml':
type: file
path: '/home/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/app/reports/checkstyle/checkstyle.xml'
expires:
relative-datestamp: "1 year"
expires: '{{#from_now}}1 year{{/from_now}}'
extra:
treeherderEnv:

Просмотреть файл

@ -50,18 +50,15 @@ task:
'public/android/lint/lint-results-automationDebug.html':
type: file
path: '/home/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/app/outputs/lint-results-automationDebug.html'
expires:
relative-datestamp: "1 year"
expires: '{{#from_now}}1 year{{/from_now}}'
'public/android/lint/lint-results-automationDebug.xml':
type: file
path: '/home/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/app/outputs/lint-results-automationDebug.xml'
expires:
relative-datestamp: "1 year"
expires: '{{#from_now}}1 year{{/from_now}}'
'public/android/lint/lint-results-automationDebug_files':
type: directory
path: '/home/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/app/outputs/lint-results-automationDebug_files'
expires:
relative-datestamp: "1 year"
expires: '{{#from_now}}1 year{{/from_now}}'
extra:
treeherderEnv:

Просмотреть файл

@ -50,8 +50,7 @@ task:
'public/android/unittest':
type: directory
path: '/home/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/app/reports/tests'
expires:
relative-datestamp: "1 year"
expires: '{{#from_now}}1 year{{/from_now}}'
extra:
treeherderEnv:

Просмотреть файл

@ -34,8 +34,7 @@ task:
'public/clang.tar.xz':
type: file
path: '/home/worker/workspace/artifacts/clang.tar.xz'
expires:
relative-datestamp: "1 year"
expires: '{{#from_now}}1 year{{/from_now}}'
extra:
treeherderEnv:

Просмотреть файл

@ -27,8 +27,7 @@ task:
'public/gcc.tar.xz':
type: file
path: '/home/worker/workspace/artifacts/gcc.tar.xz'
expires:
relative-datestamp: "1 year"
expires: '{{#from_now}}1 year{{/from_now}}'
extra:
treeherderEnv:

Просмотреть файл

@ -19,10 +19,8 @@ tasks:
- index.gecko.v2.{{project}}.latest.firefox.decision
- tc-treeherder.v2.{{project}}.{{head_rev}}.{{pushlog_id}}
- tc-treeherder-stage.v2.{{project}}.{{head_rev}}.{{pushlog_id}}
created:
relative-datestamp: "0 seconds"
deadline:
relative-datestamp: "1 day"
created: '{{now}}'
deadline: '{{#from_now}}1 day{{/from_now}}'
metadata:
owner: mozilla-taskcluster-maintenance@mozilla.com
source: {{{source}}}
@ -92,8 +90,7 @@ tasks:
# Arbitrary value for keeping these artifacts around. They are just the
# graph.json and context directories for now, so nothing that needs
# to stay around for long.
expires:
relative-datestamp: "7 days"
expires: '{{#from_now}}7 days{{/from_now}}'
extra:
treeherder:
symbol: D

Просмотреть файл

@ -16,12 +16,9 @@ scopes:
tasks:
- taskId: '{{#as_slugid}}decision task{{/as_slugid}}'
task:
created:
relative-datestamp: "0 seconds"
deadline:
relative-datestamp: "1 day"
expires:
relative-datestamp: "14 days"
created: '{{now}}'
deadline: '{{#from_now}}1 day{{/from_now}}'
expires: '{{#from_now}}14 day{{/from_now}}'
metadata:
owner: mozilla-taskcluster-maintenance@mozilla.com
source: {{{source}}}
@ -98,8 +95,7 @@ tasks:
'public':
type: 'directory'
path: '/home/worker/artifacts'
expires:
relative-datestamp: '7 days'
expires: '{{#from_now}}7 days{{/from_now}}'
extra:
treeherder:

Просмотреть файл

@ -23,5 +23,4 @@ task:
'public/build':
type: directory
path: '/home/worker/artifacts/'
expires:
relative-datestamp: '1 year'
expires: '{{#from_now}}1 year{{/from_now}}'

Просмотреть файл

@ -20,10 +20,8 @@ task:
'public/logs/':
type: directory
path: '/home/worker/workspace/build/upload/logs/'
expires:
relative-datestamp: '1 year'
expires: '{{#from_now}}1 year{{/from_now}}'
'public/test_info/':
type: directory
path: '/home/worker/workspace/build/blobber_upload_dir/'
expires:
relative-datestamp: '1 year'
expires: '{{#from_now}}1 year{{/from_now}}'

Просмотреть файл

@ -6,10 +6,8 @@
taskId: {{build_slugid}}
docker-image: desktop-build
task:
created:
relative-datestamp: '0 seconds'
deadline:
relative-datestamp: '24 hours'
created: '{{now}}'
deadline: '{{#from_now}}24 hours{{/from_now}}'
metadata:
source: '{{source}}'
owner: mozilla-taskcluster-maintenance@mozilla.com
@ -42,8 +40,7 @@ task:
'public/logs/':
type: directory
path: '/home/worker/workspace/build/upload/logs/'
expires:
relative-datestamp: '1 year'
expires: '{{#from_now}}1 year{{/from_now}}'
command:
- bash
- /home/worker/bin/build.sh

Просмотреть файл

@ -4,10 +4,8 @@
taskId: '{{image_slugid}}'
task:
created:
relative-datestamp: '0 seconds'
deadline:
relative-datestamp: '24 hours'
created: '{{now}}'
deadline: '{{#from_now}}24 hours{{/from_now}}'
metadata:
name: 'Docker Artifact Image Builder'
description: 'Builder for docker images as artifacts'
@ -42,8 +40,7 @@ task:
'{{artifact_path}}':
type: 'file'
path: '/artifacts/image.tar'
expires:
relative-datestamp: '1 year'
expires: '{{#from_now}}1 year{{/from_now}}'
extra:
treeherderEnv:
- staging

Просмотреть файл

@ -4,10 +4,8 @@
taskId: {{build_slugid}}
task:
created:
relative-datestamp: '0 seconds'
deadline:
relative-datestamp: '24 hours'
created: '{{now}}'
deadline: '{{#from_now}}24 hours{{/from_now}}'
metadata:
source: '{{source}}'
owner: mozilla-taskcluster-maintenance@mozilla.com
@ -36,8 +34,7 @@ task:
'public/build':
type: directory
path: '/home/worker/workspace/build/upload/'
expires:
relative-datestamp: '1 year'
expires: '{{#from_now}}1 year{{/from_now}}'
env:

Просмотреть файл

@ -4,10 +4,8 @@
taskId: {{build_slugid}}
task:
created:
relative-datestamp: '0 seconds'
deadline:
relative-datestamp: '24 hours'
created: '{{now}}'
deadline: '{{#from_now}}24 hours{{/from_now}}'
metadata:
source: http://todo.com/soon
owner: mozilla-taskcluster-maintenance@mozilla.com

Просмотреть файл

@ -4,10 +4,8 @@
taskId: {{build_slugid}}
docker-image: phone-builder
task:
created:
relative-datestamp: '0 seconds'
deadline:
relative-datestamp: '24 hours'
created: '{{now}}'
deadline: '{{#from_now}}24 hours{{/from_now}}'
metadata:
source: '{{source}}'
owner: mozilla-taskcluster-maintenance@mozilla.com
@ -46,13 +44,11 @@ task:
'private/build':
type: directory
path: '/home/worker/artifacts/'
expires:
relative-datestamp: '1 year'
expires: '{{#from_now}}1 year{{/from_now}}'
'public/build':
type: directory
path: '/home/worker/artifacts-public/'
expires:
relative-datestamp: '1 year'
expires: '{{#from_now}}1 year{{/from_now}}'
env:

Просмотреть файл

@ -3,10 +3,8 @@
taskId: {{build_slugid}}
docker-image: builder
task:
created:
relative-datestamp: '0 seconds'
deadline:
relative-datestamp: '24 hours'
created: '{{now}}'
deadline: '{{#from_now}}24 hours{{/from_now}}'
metadata:
source: '{{source}}'
owner: apoirot@mozilla.com
@ -47,8 +45,7 @@ task:
'public/build':
type: directory
path: '/home/worker/artifacts/'
expires:
relative-datestamp: '1 year'
expires: '{{#from_now}}1 year{{/from_now}}'
env:
GECKO_BASE_REPOSITORY: '{{base_repository}}'

Просмотреть файл

@ -3,10 +3,8 @@
taskId: {{taskId}}
task:
created:
relative-datestamp: '0 seconds'
deadline:
relative-datestamp: '24 hours'
created: '{{now}}'
deadline: '{{#from_now}}24 hours{{/from_now}}'
metadata:
source: '{{source}}'
owner: amiyaguchi@mozilla.com

Просмотреть файл

@ -1,9 +1,7 @@
# This task is the base for most tests in gecko.
task:
created:
relative-datestamp: '0 seconds'
deadline:
relative-datestamp: '24 hours'
created: '{{now}}'
deadline: '{{#from_now}}24 hours{{/from_now}}'
metadata:
source: '{{source}}'
owner: mozilla-taskcluster-maintenance@mozilla.com

Просмотреть файл

@ -28,5 +28,4 @@ task:
'public/build':
type: directory
path: '/home/worker/artifacts/'
expires:
relative-datestamp: '1 year'
expires: '{{#from_now}}1 year{{/from_now}}'

Просмотреть файл

@ -29,8 +29,7 @@ task:
'public/test':
type: directory
path: '/home/worker/artifacts/'
expires:
relative-datestamp: '1 year'
expires: '{{#from_now}}1 year{{/from_now}}'
features:
allowPtrace: true
extra:

Просмотреть файл

@ -18,8 +18,7 @@ task:
-
type: 'directory'
path: 'public\test'
expires:
relative-datestamp: '1 year'
expires: '{{#from_now}}1 year{{/from_now}}'
extra:
treeherder:

Просмотреть файл

@ -24,8 +24,7 @@ task:
'public/logs/':
type: directory
path: '/home/worker/workspace/mozharness_workspace/upload/logs/'
expires:
relative-datestamp: '1 year'
expires: '{{#from_now}}1 year{{/from_now}}'
extra:
treeherder:
symbol: Mn-h

Просмотреть файл

@ -8,8 +8,7 @@ task:
-
type: 'directory'
path: 'public\build'
expires:
relative-datestamp: '1 year'
expires: '{{#from_now}}1 year{{/from_now}}'
extra:
treeherderEnv:
- production

Просмотреть файл

@ -7,10 +7,8 @@ task:
-
type: 'directory'
path: 'public\logs'
expires:
relative-datestamp: '1 year'
expires: '{{#from_now}}1 year{{/from_now}}'
-
type: 'directory'
path: 'public\test_info'
expires:
relative-datestamp: '1 year'
expires: '{{#from_now}}1 year{{/from_now}}'

Просмотреть файл

@ -188,12 +188,6 @@ Each task has the following properties:
``task``
The task's TaskCluster task definition.
The task definition may contain "relative datestamps" of the form
``{"relative-datestamp": "certain number of seconds/hours/days/years"}``.
These will be replaced in the last step, while creating tasks.
The UTC timestamp at that moment is noted, and all the relative datestamps
are replaced with respect to this timestamp.
The task definition may contain "task references" of the form
``{"task-reference": "string containing <task-label>"}``. These will be
replaced during the optimization step, with the appropriate taskId substituted

Просмотреть файл

@ -12,10 +12,6 @@ import os
import logging
from slugid import nice as slugid
from taskgraph.util.time import (
current_json_time,
json_time_from_now
)
logger = logging.getLogger(__name__)
@ -43,6 +39,7 @@ def create_tasks(taskgraph, label_to_taskid):
# that.
for task_id in taskgraph.graph.visit_postorder():
task_def = taskgraph.tasks[task_id].task
# if this task has no dependencies, make it depend on this decision
# task so that it does not start immediately; and so that if this loop
# fails halfway through, none of the already-created tasks run.
@ -68,11 +65,6 @@ def create_tasks(taskgraph, label_to_taskid):
def _create_task(session, task_id, label, task_def):
# create the task using 'http://taskcluster/queue', which is proxied to the queue service
# with credentials appropriate to this job.
# Resolve timestamps
now = current_json_time(datetime_format=True)
task_def = resolve_timestamps(now, task_def)
logger.debug("Creating task with taskId {} for {}".format(task_id, label))
res = session.put('http://taskcluster/queue/v1/task/{}'.format(task_id),
data=json.dumps(task_def))
@ -82,17 +74,3 @@ def _create_task(session, task_id, label, task_def):
except:
logger.error(res.text)
res.raise_for_status()
def resolve_timestamps(now, task_def):
def recurse(val):
if isinstance(val, list):
return [recurse(v) for v in val]
elif isinstance(val, dict):
if val.keys() == ['relative-datestamp']:
return json_time_from_now(val['relative-datestamp'], now)
else:
return {k: recurse(v) for k, v in val.iteritems()}
else:
return val
return recurse(task_def)

Просмотреть файл

@ -17,6 +17,10 @@ from taskgraph.util.docker import (
generate_context_hash
)
from taskgraph.util.templates import Templates
from taskgraph.util.time import (
json_time_from_now,
current_json_time,
)
logger = logging.getLogger(__name__)
GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
@ -50,6 +54,8 @@ class DockerImageTask(base.Task):
'head_rev': params['head_rev'],
'owner': params['owner'],
'level': params['level'],
'from_now': json_time_from_now,
'now': current_json_time(),
'source': '{repo}file/{rev}/testing/taskcluster/tasks/image.yml'
.format(repo=params['head_repository'], rev=params['head_rev']),
}

Просмотреть файл

@ -197,19 +197,11 @@ def query_vcs_info(repository, revision):
return None
def set_expiration(task, relative_datestamp):
def set_expiration(task, timestamp):
task_def = task['task']
task_def['expires'] = {'relative-datestamp': relative_datestamp}
if 'deadline' in task_def:
now = current_json_time(datetime_format=True)
timestamp = json_time_from_now(input_str=TRY_EXPIRATION,
now=now,
datetime_format=True)
deadline = json_time_from_now(input_str=task_def['deadline']['relative-datestamp'],
now=now,
datetime_format=True)
if deadline > timestamp:
task_def['deadline']['relative-datestamp'] = relative_datestamp
task_def['expires'] = timestamp
if task_def.get('deadline', timestamp) > timestamp:
task_def['deadline'] = timestamp
try:
artifacts = task_def['payload']['artifacts']
@ -220,7 +212,7 @@ def set_expiration(task, relative_datestamp):
# for generic-worker, artifacts is a list
# for taskcluster-worker, it will depend on what we do in artifacts plugin
for artifact in artifacts.values() if hasattr(artifacts, "values") else artifacts:
artifact['expires']['relative-datestamp'] = relative_datestamp
artifact['expires'] = timestamp
def format_treeherder_route(destination, project, revision, pushlog_id):
@ -370,6 +362,8 @@ class LegacyTask(base.Task):
'rank': push_epoch,
'owner': params['owner'],
'level': params['level'],
'from_now': json_time_from_now,
'now': current_json_time(),
}.items())
routes_file = os.path.join(root, 'routes.json')
@ -467,7 +461,7 @@ class LegacyTask(base.Task):
# try builds don't use cache
if project == "try":
remove_caches_from_task(build_task)
set_expiration(build_task, TRY_EXPIRATION)
set_expiration(build_task, json_time_from_now(TRY_EXPIRATION))
decorate_task_treeherder_routes(build_task['task'],
build_parameters['project'],
@ -550,7 +544,7 @@ class LegacyTask(base.Task):
set_interactive_task(post_task, interactive)
if project == "try":
set_expiration(post_task, TRY_EXPIRATION)
set_expiration(post_task, json_time_from_now(TRY_EXPIRATION))
post_task['attributes'] = attributes.copy()
post_task['attributes']['legacy_kind'] = 'post_build'
@ -601,7 +595,7 @@ class LegacyTask(base.Task):
test_parameters['pushlog_id'])
if project == "try":
set_expiration(test_task, TRY_EXPIRATION)
set_expiration(test_task, json_time_from_now(TRY_EXPIRATION))
test_task['attributes'] = attributes.copy()
test_task['attributes']['legacy_kind'] = 'unittest'

Просмотреть файл

@ -82,11 +82,10 @@ def value_of(input_str):
return ALIASES[unit](value)
def json_time_from_now(input_str, now=None, datetime_format=False):
def json_time_from_now(input_str, now=None):
'''
:param str input_str: Input string (see value of)
:param datetime now: Optionally set the definition of `now`
:param boolean datetime_format: Set `True` to get a `datetime` output
:returns: JSON string representation of time in future.
'''
@ -95,20 +94,14 @@ def json_time_from_now(input_str, now=None, datetime_format=False):
time = now + value_of(input_str)
if datetime_format is True:
return time
else:
# Sorta a big hack but the json schema validator for date does not like the
# ISO dates until 'Z' (for timezone) is added...
return time.isoformat() + 'Z'
# Sorta a big hack but the json schema validator for date does not like the
# ISO dates until 'Z' (for timezone) is added...
return time.isoformat() + 'Z'
def current_json_time(datetime_format=False):
def current_json_time():
'''
:param boolean datetime_format: Set `True` to get a `datetime` output
:returns: JSON string representation of the current time.
'''
if datetime_format is True:
return datetime.datetime.utcnow()
else:
return datetime.datetime.utcnow().isoformat() + 'Z'
return datetime.datetime.utcnow().isoformat() + 'Z'

Просмотреть файл

@ -35,8 +35,7 @@ task:
'private/build':
type: directory
path: '/home/worker/artifacts/'
expires:
relative-datestamp: '1 year'
expires: '{{#from_now}}1 year{{/from_now}}'
extra:
# Rather then enforcing particular conventions we require that all build