Bug 1371038: Add configs for doing DMD builds r=Callek

This adds the mozconfigs, mozharness configs and taskcluster changes required
to create optimized DMD builds for linux64, win32, win64 and macosx64.

These builds will happen nightly on mozilla-central

We also add support for custom build variants on Windows (or other generic
worker environments).

MozReview-Commit-ID: HrVT9PLSWVx

--HG--
extra : rebase_source : 39ac752a312afe04187728da82a4a7f722634811

.cron.yml

@@ -67,3 +67,15 @@ jobs:
       when:
           - {hour: 16, minute: 0}
           - {hour: 4, minute: 0}
+
+    - name: nightly-dmd
+      job:
+          type: decision-task
+          treeherder-symbol: Ndmd
+          target-tasks-method: nightly_dmd
+      run-on-projects:
+          - mozilla-central
+      when:
+          by-project:
+            mozilla-central: [{hour: 10, minute: 0}]
+            # No default

browser/config/mozconfigs/linux32/opt-dmd

@@ -0,0 +1,3 @@
+ac_add_options --enable-dmd
+
+. "$topsrcdir/browser/config/mozconfigs/linux32/nightly"

browser/config/mozconfigs/linux64/opt-dmd

@@ -0,0 +1,3 @@
+ac_add_options --enable-dmd
+
+. "$topsrcdir/browser/config/mozconfigs/linux64/nightly"

browser/config/mozconfigs/macosx64/opt-dmd

@@ -0,0 +1,3 @@
+ac_add_options --enable-dmd
+
+. "$topsrcdir/browser/config/mozconfigs/macosx64/nightly"

browser/config/mozconfigs/win32/opt-dmd

@@ -0,0 +1,3 @@
+ac_add_options --enable-dmd
+
+. "$topsrcdir/browser/config/mozconfigs/win32/nightly"

browser/config/mozconfigs/win64/opt-dmd

@@ -0,0 +1,3 @@
+ac_add_options --enable-dmd
+
+. "$topsrcdir/browser/config/mozconfigs/win64/nightly"

browser/installer/package-manifest.in

@@ -825,6 +825,14 @@ bin/libfreebl_32int64_3.so
 @RESPATH@/components/SanityTest.js
 #endif
 
-#ifdef MOZ_MULET
-#include ../../b2g/installer/package-manifest.in
+#ifdef MOZ_DMD
+; DMD
+@RESPATH@/dmd.py
+@RESPATH@/fix_stack_using_bpsyms.py
+#ifdef XP_MACOSX
+@RESPATH@/fix_macosx_stack.py
+#endif
+#ifdef XP_LINUX
+@RESPATH@/fix_linux_stack.py
+#endif
 #endif

taskcluster/ci/build/linux.yml

@@ -22,6 +22,32 @@ linux64/opt:
         tooltool-downloads: public
         need-xvfb: true
 
+linux64-dmd/opt:
+    description: "Linux64 DMD Opt"
+    index:
+        product: firefox
+        job-name: linux64-dmd-opt
+    treeherder:
+        platform: linux64-dmd/opt
+        symbol: tc(Bdmd)
+        tier: 2
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        max-run-time: 36000
+        env:
+            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
+    run:
+        using: mozharness
+        actions: [get-secrets build check-test update]
+        config:
+            - builds/releng_base_linux_64_builds.py
+        script: "mozharness/scripts/fx_desktop_build.py"
+        custom-build-variant-cfg: dmd
+        secrets: true
+        tooltool-downloads: public
+        need-xvfb: true
+    run-on-projects: [ ]
+
 linux64/pgo:
     description: "Linux64 PGO"
     index:

taskcluster/ci/build/macosx.yml

@@ -45,6 +45,31 @@ macosx64/opt:
         secrets: true
         tooltool-downloads: internal
 
+macosx64-dmd/opt:
+    description: "MacOS X x64 DMD Cross-compile"
+    index:
+        product: firefox
+        job-name: macosx64-dmd-opt
+    treeherder:
+        platform: osx-10-10-dmd/opt
+        symbol: tc(Bdmd)
+        tier: 2
+    worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64
+    worker:
+        max-run-time: 36000
+        env:
+            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-releng.manifest"
+    run:
+        using: mozharness
+        actions: [get-secrets build update]
+        config:
+            - builds/releng_base_mac_64_cross_builds.py
+        script: "mozharness/scripts/fx_desktop_build.py"
+        custom-build-variant-cfg: dmd
+        secrets: true
+        tooltool-downloads: internal
+    run-on-projects: [ ]
+
 macosx64-devedition/opt:
     description: "MacOS X Dev Edition x64"
     index:

taskcluster/ci/build/windows.yml

@@ -38,6 +38,28 @@ win32/opt:
         config:
             - builds/taskcluster_firefox_windows_32_opt.py
 
+win32-dmd/opt:
+    description: "Win32 DMD Opt"
+    index:
+        product: firefox
+        job-name: win32-dmd-opt
+    treeherder:
+        platform: windows2012-32-dmd/opt
+        symbol: tc(Bdmd)
+        tier: 2
+    worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
+    worker:
+        max-run-time: 7200
+        env:
+            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/releng.manifest"
+    run:
+        using: mozharness
+        script: mozharness/scripts/fx_desktop_build.py
+        config:
+            - builds/taskcluster_firefox_windows_32_opt.py
+        custom-build-variant-cfg: dmd
+    run-on-projects: [ ]
+
 win32/pgo:
     description: "Win32 Opt PGO"
     index:
@@ -99,6 +121,28 @@ win64/opt:
         config:
             - builds/taskcluster_firefox_windows_64_opt.py
 
+win64-dmd/opt:
+    description: "Win64 DMD Opt"
+    index:
+        product: firefox
+        job-name: win64-dmd-opt
+    treeherder:
+        platform: windows2012-64-dmd/opt
+        symbol: tc(Bdmd)
+        tier: 2
+    worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
+    worker:
+        max-run-time: 7200
+        env:
+            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/releng.manifest"
+    run:
+        using: mozharness
+        script: mozharness/scripts/fx_desktop_build.py
+        config:
+            - builds/taskcluster_firefox_windows_64_opt.py
+        custom-build-variant-cfg: dmd
+    run-on-projects: [ ]
+
 win64-nightly/opt:
     description: "Win64 Nightly"
     index:

taskcluster/taskgraph/target_tasks.py

@@ -319,3 +319,13 @@ def target_tasks_nightly_win64(full_task_graph, parameters):
         if platform in ('win64-nightly', ):
             return task.attributes.get('nightly', False)
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
+
+
+# Opt DMD builds should only run nightly
+@_target_task('nightly_dmd')
+def target_tasks_dmd(full_task_graph, parameters):
+    """Target DMD that run nightly on the m-c branch."""
+    def filter(task):
+        platform = task.attributes.get('build_platform', '')
+        return platform.endswith('-dmd')
+    return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]

taskcluster/taskgraph/transforms/gecko_v2_whitelist.py

@@ -49,6 +49,7 @@ JOB_NAME_WHITELIST = set([
     'linux64-stylo-debug',
     'linux64-stylo-opt',
     'linux64-valgrind-opt',
+    'linux64-dmd-opt',
     'macosx64-add-on-devel',
     'macosx64-clang-tidy',
     'macosx64-debug',
@@ -59,6 +60,7 @@ JOB_NAME_WHITELIST = set([
     'macosx64-devedition-opt',
     'macosx64-st-an-debug',
     'macosx64-st-an-opt',
+    'macosx64-dmd-opt',
     'shell-haz-debug',
     'sm-arm-sim-debug',
     'sm-arm64-sim-debug',
@@ -81,6 +83,7 @@ JOB_NAME_WHITELIST = set([
     'win32-pgo',
     'win32-st-an-debug',
     'win32-st-an-opt',
+    'win32-dmd-opt',
     'win64-add-on-devel',
     'win64-clang-tidy',
     'win64-debug',
@@ -93,6 +96,7 @@ JOB_NAME_WHITELIST = set([
     'win64-st-an-opt',
     'win64-asan-debug',
     'win64-asan-opt',
+    'win64-dmd-opt',
 ])
 
 JOB_NAME_WHITELIST_ERROR = """\

taskcluster/taskgraph/transforms/job/mozharness.py

@@ -185,7 +185,7 @@ def mozharness_on_generic_worker(config, job, taskdesc):
 
     # fail if invalid run options are included
     invalid = []
-    for prop in ['actions', 'custom-build-variant-cfg',
+    for prop in ['actions',
                  'tooltool-downloads', 'secrets', 'taskcluster-proxy',
                  'need-xvfb']:
         if prop in run and run[prop]:
@@ -226,6 +226,9 @@ def mozharness_on_generic_worker(config, job, taskdesc):
     mh_command.append(r'--skip-buildbot-actions --work-dir %cd:Z:=z:%\build')
     for option in run.get('options', []):
         mh_command.append('--' + option)
+    if run.get('custom-build-variant-cfg'):
+        mh_command.append('--custom-build-variant')
+        mh_command.append(run['custom-build-variant-cfg'])
 
     hg_command = ['"c:\\Program Files\\Mercurial\\hg.exe"']
     hg_command.append('robustcheckout')

testing/mozharness/configs/builds/releng_sub_linux_configs/32_dmd.py

@@ -0,0 +1,3 @@
+config = {
+    'src_mozconfig': 'browser/config/mozconfigs/linux32/opt-dmd',
+}

testing/mozharness/configs/builds/releng_sub_linux_configs/64_dmd.py

@@ -0,0 +1,3 @@
+config = {
+    'src_mozconfig': 'browser/config/mozconfigs/linux64/opt-dmd',
+}

testing/mozharness/configs/builds/releng_sub_mac_configs/64_dmd.py

@@ -0,0 +1,3 @@
+config = {
+    'src_mozconfig': 'browser/config/mozconfigs/macosx64/opt-dmd',
+}

testing/mozharness/configs/builds/releng_sub_windows_configs/32_dmd.py

@@ -0,0 +1,3 @@
+config = {
+    'src_mozconfig': 'browser/config/mozconfigs/win32/opt-dmd',
+}

testing/mozharness/configs/builds/releng_sub_windows_configs/64_dmd.py

@@ -0,0 +1,3 @@
+config = {
+    'src_mozconfig': 'browser/config/mozconfigs/win64/opt-dmd',
+}

testing/mozharness/mozharness/mozilla/building/buildbase.py

@@ -384,6 +384,7 @@ class BuildOptionParser(object):
         'artifact': 'builds/releng_sub_%s_configs/%s_artifact.py',
         'debug-artifact': 'builds/releng_sub_%s_configs/%s_debug_artifact.py',
         'devedition': 'builds/releng_sub_%s_configs/%s_devedition.py',
+        'dmd': 'builds/releng_sub_%s_configs/%s_dmd.py',
     }
     build_pool_cfg_file = 'builds/build_pool_specifics.py'
     branch_cfg_file = 'builds/branch_specifics.py'