Backed out changeset cc4009e9cecc (bug 1284005) for causing merge conflicts when merging to m-c

--HG--
extra : rebase_source : 1076f58b4aa748ddc35920d1591bb99b8c1a7358

taskcluster/ci/docker-image/image.yml

@@ -1,9 +1,7 @@
 ---
 task:
-  created:
+  created: '{{now}}'
-    relative-datestamp: "0 seconds"
+  deadline: '{{#from_now}}24 hours{{/from_now}}'
-  deadline:
-    relative-datestamp: "24 hours"
   metadata:
     name: 'Docker Image Build: {{image_name}}'
     description: 'Build the docker image {{image_name}} for use by dependent tasks'
@@ -45,8 +43,7 @@ task:
       '{{artifact_path}}':
         type: 'file'
         path: '/artifacts/image.tar'
-        expires:
+        expires: '{{#from_now}}1 year{{/from_now}}'
-          relative-datestamp: "1 year"
   extra:
     treeherderEnv:
       - staging

taskcluster/ci/legacy/tasks/build.yml

@@ -4,10 +4,8 @@
 taskId: {{build_slugid}}
 
 task:
-  created:
+  created: '{{now}}'
-    relative-datestamp: "0 seconds"
+  deadline: '{{#from_now}}24 hours{{/from_now}}'
-  deadline:
-    relative-datestamp: "24 hours"
   metadata:
     source: '{{source}}'
     owner: mozilla-taskcluster-maintenance@mozilla.com

taskcluster/ci/legacy/tasks/builds/android_checkstyle.yml

@@ -50,8 +50,7 @@ task:
       'public/android/checkstyle/checkstyle.xml':
         type: file
         path: '/home/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/app/reports/checkstyle/checkstyle.xml'
-        expires:
+        expires: '{{#from_now}}1 year{{/from_now}}'
-          relative-datestamp: "1 year"
 
   extra:
     treeherderEnv:

taskcluster/ci/legacy/tasks/builds/android_lint.yml

@@ -50,18 +50,15 @@ task:
       'public/android/lint/lint-results-automationDebug.html':
         type: file
         path: '/home/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/app/outputs/lint-results-automationDebug.html'
-        expires:
+        expires: '{{#from_now}}1 year{{/from_now}}'
-          relative-datestamp: "1 year"
       'public/android/lint/lint-results-automationDebug.xml':
         type: file
         path: '/home/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/app/outputs/lint-results-automationDebug.xml'
-        expires:
+        expires: '{{#from_now}}1 year{{/from_now}}'
-          relative-datestamp: "1 year"
       'public/android/lint/lint-results-automationDebug_files':
         type: directory
         path: '/home/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/app/outputs/lint-results-automationDebug_files'
-        expires:
+        expires: '{{#from_now}}1 year{{/from_now}}'
-          relative-datestamp: "1 year"
 
   extra:
     treeherderEnv:

taskcluster/ci/legacy/tasks/builds/android_test.yml

@@ -50,8 +50,7 @@ task:
       'public/android/unittest':
         type: directory
         path: '/home/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/app/reports/tests'
-        expires:
+        expires: '{{#from_now}}1 year{{/from_now}}'
-          relative-datestamp: "1 year"
 
   extra:
     treeherderEnv:

taskcluster/ci/legacy/tasks/builds/linux64_clang.yml

@@ -34,8 +34,7 @@ task:
       'public/clang.tar.xz':
         type: file
         path: '/home/worker/workspace/artifacts/clang.tar.xz'
-        expires:
+        expires: '{{#from_now}}1 year{{/from_now}}'
-          relative-datestamp: "1 year"
 
   extra:
     treeherderEnv:

taskcluster/ci/legacy/tasks/builds/linux64_gcc.yml

@@ -27,8 +27,7 @@ task:
       'public/gcc.tar.xz':
         type: file
         path: '/home/worker/workspace/artifacts/gcc.tar.xz'
-        expires:
+        expires: '{{#from_now}}1 year{{/from_now}}'
-          relative-datestamp: "1 year"
 
   extra:
     treeherderEnv:

taskcluster/ci/legacy/tasks/decision/branch.yml

@@ -19,10 +19,8 @@ tasks:
         - index.gecko.v2.{{project}}.latest.firefox.decision
         - tc-treeherder.v2.{{project}}.{{head_rev}}.{{pushlog_id}}
         - tc-treeherder-stage.v2.{{project}}.{{head_rev}}.{{pushlog_id}}
-      created:
+      created: '{{now}}'
-        relative-datestamp: "0 seconds"
+      deadline: '{{#from_now}}1 day{{/from_now}}'
-      deadline:
-        relative-datestamp: "1 day"
       metadata:
         owner: mozilla-taskcluster-maintenance@mozilla.com
         source: {{{source}}}
@@ -92,8 +90,7 @@ tasks:
             # Arbitrary value for keeping these artifacts around.  They are just the
             # graph.json and context directories for now, so nothing that needs
             # to stay around for long.
-            expires:
+            expires: '{{#from_now}}7 days{{/from_now}}'
-              relative-datestamp: "7 days"
       extra:
         treeherder:
           symbol: D

taskcluster/ci/legacy/tasks/decision/try.yml

@@ -16,12 +16,9 @@ scopes:
 tasks:
   - taskId: '{{#as_slugid}}decision task{{/as_slugid}}'
     task:
-      created:
+      created: '{{now}}'
-        relative-datestamp: "0 seconds"
+      deadline: '{{#from_now}}1 day{{/from_now}}'
-      deadline:
+      expires: '{{#from_now}}14 day{{/from_now}}'
-        relative-datestamp: "1 day"
-      expires:
-        relative-datestamp: "14 days"
       metadata:
         owner: mozilla-taskcluster-maintenance@mozilla.com
         source: {{{source}}}
@@ -98,8 +95,7 @@ tasks:
           'public':
             type: 'directory'
             path: '/home/worker/artifacts'
-            expires:
+            expires: '{{#from_now}}7 days{{/from_now}}'
-              relative-datestamp: '7 days'
 
       extra:
         treeherder:

taskcluster/ci/legacy/tasks/docker_build.yml

@@ -23,5 +23,4 @@ task:
       'public/build':
         type: directory
         path: '/home/worker/artifacts/'
-        expires:
+        expires: '{{#from_now}}1 year{{/from_now}}'
-          relative-datestamp: '1 year'

taskcluster/ci/legacy/tasks/docker_test.yml

@@ -20,10 +20,8 @@ task:
       'public/logs/':
         type: directory
         path: '/home/worker/workspace/build/upload/logs/'
-        expires:
+        expires: '{{#from_now}}1 year{{/from_now}}'
-          relative-datestamp: '1 year'
       'public/test_info/':
         type: directory
         path: '/home/worker/workspace/build/blobber_upload_dir/'
-        expires:
+        expires: '{{#from_now}}1 year{{/from_now}}'
-          relative-datestamp: '1 year'

taskcluster/ci/legacy/tasks/harness_test.yml

@@ -6,10 +6,8 @@
 taskId: {{build_slugid}}
 docker-image: desktop-build
 task:
-  created:
+  created: '{{now}}'
-    relative-datestamp: '0 seconds'
+  deadline: '{{#from_now}}24 hours{{/from_now}}'
-  deadline:
-    relative-datestamp: '24 hours'
   metadata:
     source: '{{source}}'
     owner: mozilla-taskcluster-maintenance@mozilla.com
@@ -42,8 +40,7 @@ task:
       'public/logs/':
         type: directory
         path: '/home/worker/workspace/build/upload/logs/'
-        expires:
+        expires: '{{#from_now}}1 year{{/from_now}}'
-          relative-datestamp: '1 year'
     command:
       - bash
       - /home/worker/bin/build.sh

taskcluster/ci/legacy/tasks/image.yml

@@ -4,10 +4,8 @@
 taskId: '{{image_slugid}}'
 
 task:
-  created:
+  created: '{{now}}'
-    relative-datestamp: '0 seconds'
+  deadline: '{{#from_now}}24 hours{{/from_now}}'
-  deadline:
-    relative-datestamp: '24 hours'
   metadata:
     name: 'Docker Artifact Image Builder'
     description: 'Builder for docker images as artifacts'
@@ -42,8 +40,7 @@ task:
       '{{artifact_path}}':
         type: 'file'
         path: '/artifacts/image.tar'
-        expires:
+        expires: '{{#from_now}}1 year{{/from_now}}'
-          relative-datestamp: '1 year'
   extra:
     treeherderEnv:
       - staging

taskcluster/ci/legacy/tasks/l10n.yml

@@ -4,10 +4,8 @@
 taskId: {{build_slugid}}
 
 task:
-  created:
+  created: '{{now}}'
-    relative-datestamp: '0 seconds'
+  deadline: '{{#from_now}}24 hours{{/from_now}}'
-  deadline:
-    relative-datestamp: '24 hours'
   metadata:
     source: '{{source}}'
     owner: mozilla-taskcluster-maintenance@mozilla.com
@@ -36,8 +34,7 @@ task:
       'public/build':
         type: directory
         path: '/home/worker/workspace/build/upload/'
-        expires:
+        expires: '{{#from_now}}1 year{{/from_now}}'
-          relative-datestamp: '1 year'
 
 
     env:

taskcluster/ci/legacy/tasks/lint.yml

@@ -4,10 +4,8 @@
 taskId: {{build_slugid}}
 
 task:
-  created:
+  created: '{{now}}'
-    relative-datestamp: '0 seconds'
+  deadline: '{{#from_now}}24 hours{{/from_now}}'
-  deadline:
-    relative-datestamp: '24 hours'
   metadata:
     source: http://todo.com/soon
     owner: mozilla-taskcluster-maintenance@mozilla.com

taskcluster/ci/legacy/tasks/phone_build.yml

@@ -4,10 +4,8 @@
 taskId: {{build_slugid}}
 docker-image: phone-builder
 task:
-  created:
+  created: '{{now}}'
-    relative-datestamp: '0 seconds'
+  deadline: '{{#from_now}}24 hours{{/from_now}}'
-  deadline:
-    relative-datestamp: '24 hours'
   metadata:
     source: '{{source}}'
     owner: mozilla-taskcluster-maintenance@mozilla.com
@@ -46,13 +44,11 @@ task:
       'private/build':
         type: directory
         path: '/home/worker/artifacts/'
-        expires:
+        expires: '{{#from_now}}1 year{{/from_now}}'
-          relative-datestamp: '1 year'
       'public/build':
         type: directory
         path: '/home/worker/artifacts-public/'
-        expires:
+        expires: '{{#from_now}}1 year{{/from_now}}'
-          relative-datestamp: '1 year'
 
 
     env:

taskcluster/ci/legacy/tasks/post-builds/mulet_simulator.yml

@@ -3,10 +3,8 @@
 taskId: {{build_slugid}}
 docker-image: builder
 task:
-  created:
+  created: '{{now}}'
-    relative-datestamp: '0 seconds'
+  deadline: '{{#from_now}}24 hours{{/from_now}}'
-  deadline:
-    relative-datestamp: '24 hours'
   metadata:
     source: '{{source}}'
     owner: apoirot@mozilla.com
@@ -47,8 +45,7 @@ task:
       'public/build':
         type: directory
         path: '/home/worker/artifacts/'
-        expires:
+        expires: '{{#from_now}}1 year{{/from_now}}'
-          relative-datestamp: '1 year'
 
     env:
       GECKO_BASE_REPOSITORY: '{{base_repository}}'

taskcluster/ci/legacy/tasks/post-builds/upload_symbols.yml

@@ -3,10 +3,8 @@
 taskId: {{taskId}}
 
 task:
-  created:
+  created: '{{now}}'
-    relative-datestamp: '0 seconds'
+  deadline: '{{#from_now}}24 hours{{/from_now}}'
-  deadline:
-    relative-datestamp: '24 hours'
   metadata:
     source: '{{source}}'
     owner: amiyaguchi@mozilla.com

taskcluster/ci/legacy/tasks/test.yml

@@ -1,9 +1,7 @@
 # This task is the base for most tests in gecko.
 task:
-  created:
+  created: '{{now}}'
-    relative-datestamp: '0 seconds'
+  deadline: '{{#from_now}}24 hours{{/from_now}}'
-  deadline:
-    relative-datestamp: '24 hours'
   metadata:
     source: '{{source}}'
     owner: mozilla-taskcluster-maintenance@mozilla.com

taskcluster/ci/legacy/tasks/tests/b2g_unittest_base.yml

@@ -28,5 +28,4 @@ task:
       'public/build':
         type: directory
         path: '/home/worker/artifacts/'
-        expires:
+        expires: '{{#from_now}}1 year{{/from_now}}'
-          relative-datestamp: '1 year'

taskcluster/ci/legacy/tasks/tests/fx_docker_test_base.yml

@@ -29,8 +29,7 @@ task:
       'public/test':
         type: directory
         path: '/home/worker/artifacts/'
-        expires:
+        expires: '{{#from_now}}1 year{{/from_now}}'
-          relative-datestamp: '1 year'
     features:
       allowPtrace: true
   extra:

taskcluster/ci/legacy/tasks/tests/fx_windows_test_base.yml

@@ -18,8 +18,7 @@ task:
       -
         type: 'directory'
         path: 'public\test'
-        expires:
+        expires: '{{#from_now}}1 year{{/from_now}}'
-          relative-datestamp: '1 year'
 
   extra:
     treeherder:

taskcluster/ci/legacy/tasks/tests/harness_marionette.yml

@@ -24,8 +24,7 @@ task:
       'public/logs/':
         type: directory
         path: '/home/worker/workspace/mozharness_workspace/upload/logs/'
-        expires:
+        expires: '{{#from_now}}1 year{{/from_now}}'
-          relative-datestamp: '1 year'
   extra:
     treeherder:
       symbol: Mn-h

taskcluster/ci/legacy/tasks/windows_build.yml

@@ -8,8 +8,7 @@ task:
       -
         type: 'directory'
         path: 'public\build'
-        expires:
+        expires: '{{#from_now}}1 year{{/from_now}}'
-          relative-datestamp: '1 year'
   extra:
     treeherderEnv:
       - production

taskcluster/ci/legacy/tasks/windows_test.yml

@@ -7,10 +7,8 @@ task:
       -
         type: 'directory'
         path: 'public\logs'
-        expires:
+        expires: '{{#from_now}}1 year{{/from_now}}'
-          relative-datestamp: '1 year'
       -
         type: 'directory'
         path: 'public\test_info'
-        expires:
+        expires: '{{#from_now}}1 year{{/from_now}}'
-          relative-datestamp: '1 year'

taskcluster/docs/taskgraph.rst

@@ -188,12 +188,6 @@ Each task has the following properties:
 ``task``
    The task's TaskCluster task definition.
 
-The task definition may contain "relative datestamps" of the form
-``{"relative-datestamp": "certain number of seconds/hours/days/years"}``.
-These will be replaced in the last step, while creating tasks.
-The UTC timestamp at that moment is noted, and all the relative datestamps
-are replaced with respect to this timestamp.
-
 The task definition may contain "task references" of the form
 ``{"task-reference": "string containing <task-label>"}``.  These will be
 replaced during the optimization step, with the appropriate taskId substituted

taskcluster/taskgraph/create.py

@@ -12,10 +12,6 @@ import os
 import logging
 
 from slugid import nice as slugid
-from taskgraph.util.time import (
-    current_json_time,
-    json_time_from_now
-)
 
 logger = logging.getLogger(__name__)
 
@@ -43,6 +39,7 @@ def create_tasks(taskgraph, label_to_taskid):
         # that.
         for task_id in taskgraph.graph.visit_postorder():
             task_def = taskgraph.tasks[task_id].task
+
             # if this task has no dependencies, make it depend on this decision
             # task so that it does not start immediately; and so that if this loop
             # fails halfway through, none of the already-created tasks run.
@@ -68,11 +65,6 @@ def create_tasks(taskgraph, label_to_taskid):
 def _create_task(session, task_id, label, task_def):
     # create the task using 'http://taskcluster/queue', which is proxied to the queue service
     # with credentials appropriate to this job.
-
-    # Resolve timestamps
-    now = current_json_time(datetime_format=True)
-    task_def = resolve_timestamps(now, task_def)
-
     logger.debug("Creating task with taskId {} for {}".format(task_id, label))
     res = session.put('http://taskcluster/queue/v1/task/{}'.format(task_id),
                       data=json.dumps(task_def))
@@ -82,17 +74,3 @@ def _create_task(session, task_id, label, task_def):
         except:
             logger.error(res.text)
         res.raise_for_status()
-
-
-def resolve_timestamps(now, task_def):
-    def recurse(val):
-        if isinstance(val, list):
-            return [recurse(v) for v in val]
-        elif isinstance(val, dict):
-            if val.keys() == ['relative-datestamp']:
-                return json_time_from_now(val['relative-datestamp'], now)
-            else:
-                return {k: recurse(v) for k, v in val.iteritems()}
-        else:
-            return val
-    return recurse(task_def)

taskcluster/taskgraph/kind/docker_image.py

@@ -17,6 +17,10 @@ from taskgraph.util.docker import (
     generate_context_hash
 )
 from taskgraph.util.templates import Templates
+from taskgraph.util.time import (
+    json_time_from_now,
+    current_json_time,
+)
 
 logger = logging.getLogger(__name__)
 GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
@@ -50,6 +54,8 @@ class DockerImageTask(base.Task):
             'head_rev': params['head_rev'],
             'owner': params['owner'],
             'level': params['level'],
+            'from_now': json_time_from_now,
+            'now': current_json_time(),
             'source': '{repo}file/{rev}/testing/taskcluster/tasks/image.yml'
                       .format(repo=params['head_repository'], rev=params['head_rev']),
         }

taskcluster/taskgraph/kind/legacy.py

@@ -197,19 +197,11 @@ def query_vcs_info(repository, revision):
         return None
 
 
-def set_expiration(task, relative_datestamp):
+def set_expiration(task, timestamp):
     task_def = task['task']
-    task_def['expires'] = {'relative-datestamp': relative_datestamp}
+    task_def['expires'] = timestamp
-    if 'deadline' in task_def:
+    if task_def.get('deadline', timestamp) > timestamp:
-        now = current_json_time(datetime_format=True)
+        task_def['deadline'] = timestamp
-        timestamp = json_time_from_now(input_str=TRY_EXPIRATION,
-                                       now=now,
-                                       datetime_format=True)
-        deadline = json_time_from_now(input_str=task_def['deadline']['relative-datestamp'],
-                                      now=now,
-                                      datetime_format=True)
-        if deadline > timestamp:
-            task_def['deadline']['relative-datestamp'] = relative_datestamp
 
     try:
         artifacts = task_def['payload']['artifacts']
@@ -220,7 +212,7 @@ def set_expiration(task, relative_datestamp):
     # for generic-worker, artifacts is a list
     # for taskcluster-worker, it will depend on what we do in artifacts plugin
     for artifact in artifacts.values() if hasattr(artifacts, "values") else artifacts:
-        artifact['expires']['relative-datestamp'] = relative_datestamp
+        artifact['expires'] = timestamp
 
 
 def format_treeherder_route(destination, project, revision, pushlog_id):
@@ -370,6 +362,8 @@ class LegacyTask(base.Task):
             'rank': push_epoch,
             'owner': params['owner'],
             'level': params['level'],
+            'from_now': json_time_from_now,
+            'now': current_json_time(),
         }.items())
 
         routes_file = os.path.join(root, 'routes.json')
@@ -467,7 +461,7 @@ class LegacyTask(base.Task):
             # try builds don't use cache
             if project == "try":
                 remove_caches_from_task(build_task)
-                set_expiration(build_task, TRY_EXPIRATION)
+                set_expiration(build_task, json_time_from_now(TRY_EXPIRATION))
 
             decorate_task_treeherder_routes(build_task['task'],
                                             build_parameters['project'],
@@ -550,7 +544,7 @@ class LegacyTask(base.Task):
                 set_interactive_task(post_task, interactive)
 
                 if project == "try":
-                    set_expiration(post_task, TRY_EXPIRATION)
+                    set_expiration(post_task, json_time_from_now(TRY_EXPIRATION))
 
                 post_task['attributes'] = attributes.copy()
                 post_task['attributes']['legacy_kind'] = 'post_build'
@@ -601,7 +595,7 @@ class LegacyTask(base.Task):
                                                     test_parameters['pushlog_id'])
 
                     if project == "try":
-                        set_expiration(test_task, TRY_EXPIRATION)
+                        set_expiration(test_task, json_time_from_now(TRY_EXPIRATION))
 
                     test_task['attributes'] = attributes.copy()
                     test_task['attributes']['legacy_kind'] = 'unittest'

taskcluster/taskgraph/util/time.py

@@ -82,11 +82,10 @@ def value_of(input_str):
     return ALIASES[unit](value)
 
 
-def json_time_from_now(input_str, now=None, datetime_format=False):
+def json_time_from_now(input_str, now=None):
     '''
     :param str input_str: Input string (see value of)
     :param datetime now: Optionally set the definition of `now`
-    :param boolean datetime_format: Set `True` to get a `datetime` output
     :returns: JSON string representation of time in future.
     '''
 
@@ -95,20 +94,14 @@ def json_time_from_now(input_str, now=None, datetime_format=False):
 
     time = now + value_of(input_str)
 
-    if datetime_format is True:
+    # Sorta a big hack but the json schema validator for date does not like the
-        return time
+    # ISO dates until 'Z' (for timezone) is added...
-    else:
+    return time.isoformat() + 'Z'
-        # Sorta a big hack but the json schema validator for date does not like the
-        # ISO dates until 'Z' (for timezone) is added...
-        return time.isoformat() + 'Z'
 
 
-def current_json_time(datetime_format=False):
+def current_json_time():
     '''
-    :param boolean datetime_format: Set `True` to get a `datetime` output
     :returns: JSON string representation of the current time.
     '''
-    if datetime_format is True:
+
-        return datetime.datetime.utcnow()
+    return datetime.datetime.utcnow().isoformat() + 'Z'
-    else:
-        return datetime.datetime.utcnow().isoformat() + 'Z'

testing/docker/phone-builder/tests/valid.yml

@@ -35,8 +35,7 @@ task:
       'private/build':
         type: directory
         path: '/home/worker/artifacts/'
-        expires:
+        expires: '{{#from_now}}1 year{{/from_now}}'
-          relative-datestamp: '1 year'
 
   extra:
     # Rather then enforcing particular conventions we require that all build