Bug 1097246 - Part1 - Added branched config support r=lightsofapollo

--HG--
extra : rebase_source : 821d2d92889970c04dce161c5a06f6a61d7f7369
extra : source : 21aa76d8d003732ebc46449163a9bf4b52bc3a69
extra : histedit_source : da47f152d5003d072903869ce3d24300df708e0f

testing/taskcluster/mach_commands.py

@@ -36,6 +36,9 @@ REGISTRY = open(os.path.join(DOCKER_ROOT, 'REGISTRY')).read().strip()
 
 DEFINE_TASK = 'queue:define-task:aws-provisioner/{}'
 
+DEFAULT_TRY = 'try: -b do -p all -u all'
+DEFAULT_JOB_PATH = os.path.join(ROOT, 'tasks', 'job_flags.yml')
+
 def get_hg_url():
     ''' Determine the url for the mercurial repository'''
     try:
@@ -84,6 +87,7 @@ def docker_image(name):
 def get_task(task_id):
     return json.load(urllib2.urlopen("https://queue.taskcluster.net/v1/task/" + task_id))
 
+
 @CommandProvider
 class DecisionTask(object):
     @Command('taskcluster-decision', category="ci",
@@ -91,6 +95,9 @@ class DecisionTask(object):
     @CommandArgument('--project',
         required=True,
         help='Treeherder project name')
+    @CommandArgument('--repository',
+        required=True,
+        help='Gecko repository to use as head repository.')
     @CommandArgument('--revision',
         required=True,
         help='Revision for this project')
@@ -108,6 +115,7 @@ class DecisionTask(object):
             'source': 'http://todo.com/soon',
             'project': params['project'],
             'comment': params['comment'],
+            'repository_url': params['repository'],
             'revision': params['revision'],
             'owner': params['owner'],
             'as_slugid': SlugidJar(),
@@ -118,9 +126,9 @@ class DecisionTask(object):
         print(json.dumps(task, indent=4))
 
 @CommandProvider
-class TryGraph(object):
-    @Command('taskcluster-trygraph', category="ci",
-        description="Create taskcluster try server graph")
+class Graph(object):
+    @Command('taskcluster-graph', category="ci",
+        description="Create taskcluster task graph")
     @CommandArgument('--base-repository',
         default=os.environ.get('GECKO_BASE_REPOSITORY'),
         help='URL for "base" repository to clone')
@@ -134,17 +142,34 @@ class TryGraph(object):
         default=os.environ.get('GECKO_HEAD_REV'),
         help='Commit revision to use from head repository')
     @CommandArgument('--message',
+        help='Commit message to be parsed. Example: "try: -b do -p all -u all"')
+    @CommandArgument('--project',
         required=True,
-        help='Commit message to be parsed')
+        help='Project to use for creating task graph. Example: --project=try')
     @CommandArgument('--owner',
         required=True,
         help='email address of who owns this graph')
     @CommandArgument('--extend-graph',
         action="store_true", dest="ci", help='Omit create graph arguments')
     def create_graph(self, **params):
+        project = params['project']
+        message = params.get('message', '') if project == 'try' else DEFAULT_TRY
+
+        # Message would only be blank when not created from decision task
+        if project == 'try' and not message:
+            sys.stderr.write(
+                    "Must supply commit message when creating try graph. " \
+                    "Example: --message='try: -b do -p all -u all'"
+            )
+            sys.exit(1)
+
         templates = Templates(ROOT)
-        jobs = templates.load('job_flags.yml', {})
-        job_graph = parse_commit(params['message'], jobs)
+        job_path = os.path.join(ROOT, 'tasks', 'branches', project, 'job_flags.yml')
+        job_path = job_path if os.path.exists(job_path) else DEFAULT_JOB_PATH
+
+        jobs = templates.load(job_path, {})
+
+        job_graph = parse_commit(message, jobs)
         # Template parameters used when expanding the graph
         parameters = {
             'docker_image': docker_image,
@@ -168,8 +193,8 @@ class TryGraph(object):
             'source': 'http://todo.com/what/goes/here',
             'owner': params['owner'],
             # TODO: Add full mach commands to this example?
-            'description': 'Try task graph generated via ./mach trygraph',
-            'name': 'trygraph local'
+            'description': 'Task graph generated via ./mach taskcluster-graph',
+            'name': 'task graph local'
         }
 
         for build in job_graph:

testing/taskcluster/tasks/branches/alder/job_flags.yml

@@ -0,0 +1,75 @@
+---
+# For complete sample of all build and test jobs,
+# see <gecko>/testing/taskcluster/tasks/job_flags.yml
+
+$inherits:
+  from: tasks/branches/base_job_flags.yml
+
+builds:
+  linux64_gecko:
+    platforms:
+      - b2g
+    types:
+      opt:
+        task: tasks/builds/b2g_desktop_opt.yml
+      debug:
+        task: tasks/builds/b2g_desktop_debug.yml
+  linux64-mulet:
+    platforms:
+      - b2g
+    types:
+      opt:
+        task: tasks/builds/mulet_linux.yml
+
+tests:
+  gaia-build:
+    allowed_build_tasks:
+      tasks/builds/b2g_desktop_opt.yml:
+        task: tasks/tests/b2g_build_test.yml
+  gaia-build-unit:
+    allowed_build_tasks:
+      tasks/builds/b2g_desktop_opt.yml:
+        task: tasks/tests/b2g_build_unit.yml
+  gaia-js-integration:
+    allowed_build_tasks:
+      tasks/builds/b2g_desktop_opt.yml:
+        task: tasks/tests/b2g_gaia_js_integration_tests.yml
+        chunks: 4
+  gaia-linter:
+    allowed_build_tasks:
+      tasks/builds/b2g_desktop_opt.yml:
+        task: tasks/tests/b2g_linter.yml
+  gaia-ui-test-accessibility:
+    allowed_build_tasks:
+      tasks/builds/b2g_desktop_opt.yml:
+        task: tasks/tests/b2g_gaia_ui_test_accessibility.yml
+      tasks/builds/b2g_emulator.yml:
+        task: tasks/tests/b2g_emulator_gaia_ui_test_accessibility.yml
+  gaia-ui-test-functional:
+    allowed_build_tasks:
+      tasks/builds/b2g_desktop_opt.yml:
+        task: tasks/tests/b2g_gaia_ui_test_functional.yml
+        chunks: 3
+  gaia-ui-test-unit:
+    allowed_build_tasks:
+      tasks/builds/b2g_desktop_opt.yml:
+        task: tasks/tests/b2g_gaia_ui_test_unit.yml
+  gaia-unit:
+    allowed_build_tasks:
+      tasks/builds/b2g_desktop_opt.yml:
+        task: tasks/tests/b2g_gaia_unit.yml
+  mochitest:
+    allowed_build_tasks:
+      tasks/builds/mulet_linux.yml:
+        task: tasks/tests/mulet_mochitests.yml
+        chunks: 5
+  mochitest-oop:
+    allowed_build_tasks:
+      tasks/builds/b2g_desktop_opt.yml:
+        task: tasks/tests/b2g_mochitest_oop.yml
+        chunks: 1
+  reftest-sanity-oop:
+    allowed_build_tasks:
+      tasks/builds/b2g_desktop_opt.yml:
+        task: tasks/tests/b2g_reftests_sanity_oop.yml
+        chunks: 1

testing/taskcluster/tasks/branches/base_job_flags.yml

@@ -0,0 +1,44 @@
+---
+# List of all possible flags for each category of tests used in the case where
+# "all" is specified.
+flags:
+  builds:
+    - emulator
+    - emulator-jb
+    - emulator-kk
+    - linux32_gecko  # b2g desktop linux 32 bit
+    - linux64_gecko  # b2g desktop linux 64 bit
+    - linux64-mulet  # Firefox desktop - b2g gecko linux 64 bit
+    - macosx64_gecko # b2g desktop osx 64 bit
+    - win32_gecko    # b2g desktop win 32 bit
+
+  tests:
+    - cppunit
+    - crashtest
+    - crashtest-ipc
+    - gaia-build
+    - gaia-build-unit
+    - gaia-js-integration
+    - gaia-linter
+    - gaia-unit
+    - gaia-unit-oop
+    - gaia-ui-test-oop
+    - gaia-ui-test-accessibility
+    - gaia-ui-test-functional
+    - gaia-ui-test-unit
+    - jetpack
+    - jittests
+    - jsreftest
+    - marionette
+    - marionette-webapi
+    - mochitest
+    - mochitest-media
+    - mochitest-oop
+    - mozmill
+    - reftest
+    - reftest-ipc
+    - reftest-no-accel
+    - reftest-sanity-oop
+    - web-platform-tests
+    - xpcshell
+

testing/taskcluster/tasks/builds/b2g_desktop_base.yml

@@ -2,8 +2,8 @@ $inherits:
   from: 'tasks/build.yml'
 task:
   metadata:
-    name: B2G Desktop Opt
-    description: B2G Desktop Opt
+    name: B2G Desktop {{build_type}}
+    description: B2G Desktop {{build_type}}
 
   scopes:
     - 'docker-worker:cache:sources-gecko'

testing/taskcluster/tasks/builds/b2g_desktop_debug.yml

@@ -1,5 +1,7 @@
 $inherits:
   from: 'tasks/builds/b2g_desktop_base.yml'
+  variables:
+    build_type: 'debug'
 
 task:
   payload:

testing/taskcluster/tasks/builds/b2g_desktop_opt.yml

@@ -1,5 +1,7 @@
 $inherits:
   from: 'tasks/builds/b2g_desktop_base.yml'
+  variables:
+    build_type: 'opt'
 
 task:
   payload:

testing/taskcluster/tasks/decision/branch.yml

@@ -0,0 +1,90 @@
+---
+metadata:
+  name: 'Task graph used to build {{project}} specific jobs'
+  description: |
+    {{project}} push for {{owner}} with comment: {{comment}}.
+  owner: "{{owner}}"
+  source: "{{source}}"
+
+scopes:
+  - "docker-worker:image:quay.io/mozilla/decision:*"
+  - "queue:define-task:aws-provisioner/gecko-decision"
+  - "queue:create-task:aws-provisioner/gecko-decision"
+  - "docker-worker:cache:tc-vcs-public-sources"
+  - "docker-worker:cache:build-emulator-jb-opt"
+  - "docker-worker:cache:build-mulet-linux-objects"
+  - "docker-worker:cache:build-emulator-ics-opt"
+  - "queue:define-task:aws-provisioner/b2gtest"
+  - "queue:create-task:aws-provisioner/b2gtest"
+  - "docker-worker:image:quay.io/mozilla/builder:*"
+  - "docker-worker:cache:tooltool-cache"
+  - "queue:define-task:aws-provisioner/b2gbuild"
+  - "queue:create-task:aws-provisioner/b2gbuild"
+  - "docker-worker:cache:build-emulator-kk-debug"
+  - "docker-worker:cache:build-b2g-desktop-objects"
+  - "docker-worker:cache:build-emulator-kk-opt"
+  - "docker-worker:cache:build-emulator-jb-debug"
+  - "docker-worker:cache:tc-vcs"
+  - "docker-worker:cache:sources-gecko"
+  - "docker-worker:cache:sources-gaia"
+  - "docker-worker:cache:build-emulator-ics-debug"
+tasks:
+  - taskId: '{{#as_slugid}}decision task{{/as_slugid}}'
+    task:
+      created: '{{now}}'
+      deadline: '{{#from_now}}1 day{{/from_now}}'
+      metadata:
+        source: {{source}}
+        owner: {{owner}}
+        name: Initial decision task for {{project}} specific graphs
+        description: |
+          This is the single most important task as it decides how all other tasks
+          get built.
+
+      workerType: "gecko-decision"
+      provisionerId: "aws-provisioner"
+
+      scopes:
+        - "docker-worker:cache:tc-vcs-public-sources"
+        - "docker-worker:image:quay.io/mozilla/decision:0.0.3"
+
+      payload:
+        env:
+          GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-central'
+          GECKO_HEAD_REPOSITORY: '{{repository_url}}'
+          GECKO_HEAD_REF: '{{revision}}'
+          GECKO_HEAD_REV: '{{revision}}'
+
+        cache:
+          # The taskcluster-vcs tooling stores the large clone caches in this
+          # directory and will reuse them for new requests this saves about 20s~ and
+          # is the most generic cache possible.
+          tc-vcs-public-sources: '/home/worker/.tc-vcs/'
+
+        # Note: This task is built server side without the context or tooling that
+        # exist in tree so we must hard code the version
+        image: 'quay.io/mozilla/decision:0.0.3'
+
+        # Virtually no network or other potentially risky operations happen as part
+        # of the task timeout aside from the initial clone. We intentionally have
+        # set this to a lower value _all_ decision tasks should use a root
+        # repository which is cached.
+        maxRunTime: 1800
+
+        command:
+          - /bin/bash
+          - -cx
+          - >
+            source $(which entrypoint) &&
+            ./mach taskcluster-graph
+            --message='{{comment}}'
+            --project='{{project}}'
+            --owner='{{owner}}'
+            --extend-graph > /home/worker/graph.json
+        graphs:
+          - /home/worker/graph.json
+
+      extra:
+        treeherder:
+          symbol: D
+

testing/taskcluster/tasks/decision/try.yml

@@ -51,7 +51,7 @@ tasks:
       payload:
         env:
           GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-central'
-          GECKO_HEAD_REPOSITORY: 'https://hg.mozilla.org/try'
+          GECKO_HEAD_REPOSITORY: '{{repository_url}}'
           GECKO_HEAD_REF: '{{revision}}'
           GECKO_HEAD_REV: '{{revision}}'
 
@@ -76,7 +76,8 @@ tasks:
           - -cx
           - >
             source $(which entrypoint) &&
-            ./mach taskcluster-trygraph
+            ./mach taskcluster-graph
+            --project='{{project}}'
             --message='{{comment}}'
             --owner='{{owner}}'
             --extend-graph > /home/worker/graph.json

testing/taskcluster/tasks/job_flags.yml

@@ -153,6 +153,8 @@ tests:
     allowed_build_tasks:
       tasks/builds/b2g_desktop_opt.yml:
         task: tasks/tests/b2g_gaia_ui_test_accessibility.yml
+      tasks/builds/b2g_emulator.yml:
+        task: tasks/tests/b2g_emulator_gaia_ui_test_accessibility.yml
   gaia-ui-test-functional:
     allowed_build_tasks:
       tasks/builds/b2g_desktop_opt.yml: