Merge mozilla-central to autoland

2017-07-26 17:10:11 +02:00 · 2017-07-26 17:10:11 +02:00 · 2bc71dbfe3
--- a/.cron.yml
+++ b/.cron.yml
@ -39,8 +39,14 @@ jobs:
          treeherder-symbol: Nd-Win
          target-tasks-method: nightly_win
      run-on-projects:
+          - mozilla-central
          - date
-      when: [] # never (hook only)
+      when:
+          by-project:
+            # Match buildbot starts for now
+            date: [{hour: 15, minute: 0}]
+            mozilla-central: [{hour: 10, minute: 0}]
+            # No default

    - name: nightly-android
      job:
--- a/taskcluster/ci/build/windows.yml
+++ b/taskcluster/ci/build/windows.yml
@ -26,7 +26,7 @@ win32/opt:
    treeherder:
        platform: windows2012-32/opt
        symbol: tc(B)
-        tier: 2
+        tier: 1
    worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
    worker:
        max-run-time: 7200
@ -68,7 +68,7 @@ win32/pgo:
    treeherder:
        platform: windows2012-32/pgo
        symbol: tc(B)
-        tier: 2
+        tier: 1
    worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
    worker:
        max-run-time: 9000
@ -109,7 +109,7 @@ win64/opt:
    treeherder:
        platform: windows2012-64/opt
        symbol: tc(B)
-        tier: 2
+        tier: 1
    worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
    worker:
        max-run-time: 7200
@ -143,6 +143,48 @@ win64-dmd/opt:
        custom-build-variant-cfg: dmd
    run-on-projects: [ ]

+win32-devedition/opt:
+    description: "Win32 Dev Edition Opt"
+    index:
+        product: devedition
+        job-name: win32-devedition-opt
+    treeherder:
+        platform: windows2012-32-devedition/opt
+        symbol: tc(B)
+        tier: 1
+    worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
+    worker:
+        max-run-time: 7200
+        env:
+            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/releng.manifest"
+    run:
+        using: mozharness
+        script: mozharness/scripts/fx_desktop_build.py
+        config:
+            - builds/taskcluster_firefox_windows_32_opt.py
+    run-on-projects: ['mozilla-beta', ]
+
+win64-devedition/opt:
+    description: "Win64 Dev Edition Opt"
+    index:
+        product: devedition
+        job-name: win64-devedition-opt
+    treeherder:
+        platform: windows2012-64-devedition/opt
+        symbol: tc(B)
+        tier: 1
+    worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
+    worker:
+        max-run-time: 7200
+        env:
+            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/releng.manifest"
+    run:
+        using: mozharness
+        script: mozharness/scripts/fx_desktop_build.py
+        config:
+            - builds/taskcluster_firefox_windows_64_opt.py
+    run-on-projects: ['mozilla-beta', ]
+
 win32-nightly/opt:
    description: "Win32 Nightly"
    index:
@ -154,7 +196,7 @@ win32-nightly/opt:
    treeherder:
        platform: windows2012-32/opt
        symbol: tc(N)
-        tier: 2
+        tier: 1
    worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
    worker:
        max-run-time: 7200
@ -180,7 +222,7 @@ win64-nightly/opt:
    treeherder:
        platform: windows2012-64/opt
        symbol: tc(N)
-        tier: 2
+        tier: 1
    worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
    worker:
        max-run-time: 7200
@ -203,7 +245,7 @@ win64/pgo:
    treeherder:
        platform: windows2012-64/pgo
        symbol: tc(B)
-        tier: 2
+        tier: 1
    worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
    worker:
        max-run-time: 10800
--- a/taskcluster/ci/test/test-platforms.yml
+++ b/taskcluster/ci/test/test-platforms.yml
@ -144,7 +144,7 @@ windows7-32/opt:
        - awsy
        - desktop-screenshot-capture
        - headless
-        #- windows-talos
+        - windows-talos
        - windows-tests

 windows7-32-pgo/opt:
@ -154,7 +154,7 @@ windows7-32-pgo/opt:
        - desktop-screenshot-capture
        - headless
        - windows-tests
-        #- windows-talos
+        - windows-talos

 windows7-32-nightly/opt:
    build-platform: win32-nightly/opt
@ -164,19 +164,28 @@ windows7-32-nightly/opt:
        - headless
        - windows-tests

+windows7-32-devedition/opt:
+    build-platform: win32-devedition/opt
+    test-sets:
+        - awsy
+        - desktop-screenshot-capture
+        - headless
+        - windows-tests
+
 # win64
 windows10-64/debug:
    build-platform: win64/debug
    test-sets:
        - headless
        - windows-tests
+
 windows10-64/opt:
    build-platform: win64/opt
    test-sets:
        - awsy
        - desktop-screenshot-capture
        - headless
-        #- windows-talos
+        - windows-talos
        - windows-tests

 windows10-64-pgo/opt:
@ -185,7 +194,7 @@ windows10-64-pgo/opt:
        - awsy
        - desktop-screenshot-capture
        - headless
-        #- windows-talos
+        - windows-talos
        - windows-tests

 windows10-64-nightly/opt:
@ -196,29 +205,40 @@ windows10-64-nightly/opt:
        - headless
        - windows-tests

+windows10-64-devedition/opt:
+    build-platform: win64-devedition/opt
+    test-sets:
+        - awsy
+        - desktop-screenshot-capture
+        - headless
+        - windows-tests
+
 windows10-64-asan/opt:
    build-platform: win64-asan/opt
    test-sets:
        - common-tests

 # Windows8 tests; all via BBB
-# Disabled until flag day
 windows8-64/debug:
    build-platform: win64/debug
-    test-sets: []
-        #- windows8-tests
+    test-sets:
+        - windows8-tests
 windows8-64/opt:
    build-platform: win64/opt
-    test-sets: []
-        #- windows8-tests
+    test-sets:
+        - windows8-tests
 windows8-64-pgo/opt:
    build-platform: win64-pgo/opt
-    test-sets: []
-        #- windows8-tests
+    test-sets:
+        - windows8-tests
 windows8-64-nightly/opt:
    build-platform: win64-nightly/opt
-    test-sets: []
-        #- windows8-tests
+    test-sets:
+        - windows8-tests
+windows8-64-devedition/opt:
+    build-platform: win64-devedition/opt
+    test-sets:
+        - windows8-tests

 ##
 # MacOS X platforms (matching /macosx.*/)
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@ -255,3 +255,4 @@ windows8-tests:
    - mochitest-webgl
    - mochitest-clipboard
    - mochitest-chrome
+    - xpcshell
--- a/taskcluster/ci/test/tests.yml
+++ b/taskcluster/ci/test/tests.yml
@ -9,7 +9,6 @@ awsy:
    description: "Are we slim yet"
    suite: awsy
    treeherder-symbol: tc-SY(sy)
-    run-on-projects: ['release', 'trunk', 'try']
    docker-image: {"in-tree": "desktop1604-test"}
    max-run-time: 7200
    e10s: true
@ -184,6 +183,7 @@ gtest:
    run-on-projects:
        by-test-platform:
            linux64-qr/.*: ['mozilla-central', 'try']
+            windows.*-pgo/.*: [] # permafails on pgo
            default: built-projects
    mozharness:
        script: desktop_unittest.py
@ -302,7 +302,6 @@ jsreftest:
    tier:
        by-test-platform:
            linux64-qr/.*: 1
-            windows10-64/.*: 3
            default: default

 marionette:
@ -321,7 +320,6 @@ marionette:
    tier:
        by-test-platform:
            android.*: 2
-            windows10.*: 3
            default: default
    chunks:
        by-test-platform:
@ -373,9 +371,6 @@ marionette-headless:
    e10s:
        by-test-platform:
            default: both
-    run-on-projects:
-        by-test-platform:
-            default: ['all']
    mozharness:
        by-test-platform:
            default:
@ -569,6 +564,8 @@ mochitest-browser-screenshots:
            windows.*/opt: ['mozilla-central', 'try']
            linux64/opt: ['mozilla-central', 'try']
            macosx.*/opt: ['mozilla-central', 'try']
+            windows7-32-devedition/opt: []
+            windows10-64-devedition/opt: []
            default: []
    e10s: both
    max-run-time: 3600
@ -610,7 +607,7 @@ mochitest-chrome:
    e10s: false
    run-on-projects:
        by-test-platform:
-            windows.*: ['try']
+            windows7-32.*: ['try']
            default: built-projects
    worker-type:
        by-test-platform:
@ -1125,6 +1122,7 @@ reftest-gpu:
            default: built-projects
    worker-type:
        by-test-platform:
+            windows7-32.*/debug: buildbot-bridge/buildbot-bridge
            default: null
    instance-size:
        by-test-platform:
@ -1273,7 +1271,6 @@ talos-chrome:
        by-test-platform:
            linux64-stylo-sequential/.*: ['mozilla-central','try']
            linux64-stylo/.*: ['mozilla-central', 'try']
-            windows.*: ['try']
            default: ['mozilla-beta', 'mozilla-central', 'mozilla-inbound', 'autoland', 'try']
    max-run-time: 3600
    e10s: true
@ -1303,7 +1300,6 @@ talos-dromaeojs:
        by-test-platform:
            linux64-stylo-sequential/.*: ['mozilla-central','try']
            linux64-stylo/.*: ['mozilla-central', 'try']
-            windows.*: ['try']
            default: ['mozilla-beta', 'mozilla-central', 'mozilla-inbound', 'autoland', 'try']
    max-run-time: 3600
    e10s: true
@ -1333,7 +1329,6 @@ talos-g1:
        by-test-platform:
            linux64-stylo-sequential/.*: ['mozilla-central','try']
            linux64-stylo/.*: ['mozilla-central', 'try']
-            windows.*: ['try']
            default: ['mozilla-beta', 'mozilla-central', 'mozilla-inbound', 'autoland', 'try']
    max-run-time: 7200
    e10s: true
@ -1364,7 +1359,6 @@ talos-g2:
        by-test-platform:
            linux64-stylo-sequential/.*: ['mozilla-central','try']
            linux64-stylo/.*: ['mozilla-central', 'try']
-            windows.*: ['try']
            default: ['mozilla-beta', 'mozilla-central', 'mozilla-inbound', 'autoland', 'try']
    e10s: true
    mozharness:
@ -1422,7 +1416,6 @@ talos-g4:
        by-test-platform:
            linux64-stylo-sequential/.*: ['mozilla-central','try']
            linux64-stylo/.*: ['mozilla-central', 'try']
-            windows.*: ['try']
            default: ['mozilla-beta','mozilla-central', 'mozilla-inbound', 'autoland', 'try']
    max-run-time: 3600
    e10s: true
@ -1452,7 +1445,6 @@ talos-g5:
        by-test-platform:
            linux64-stylo-sequential/.*: ['mozilla-central','try']
            linux64-stylo/.*: ['mozilla-central', 'try']
-            windows.*: ['try']
            default: ['mozilla-beta', 'mozilla-central', 'mozilla-inbound', 'autoland', 'try']
    max-run-time: 3600
    e10s: true
@ -1483,7 +1475,6 @@ talos-other:
        by-test-platform:
            linux64-stylo-sequential/.*: ['mozilla-central','try']
            linux64-stylo/.*: ['mozilla-central', 'try']
-            windows.*: ['try']
            default: ['mozilla-beta','mozilla-central', 'mozilla-inbound', 'autoland', 'try']
    max-run-time: 3600
    e10s: true
@ -1513,7 +1504,6 @@ talos-perf-reftest:
        by-test-platform:
            linux64-stylo-sequential/.*: ['mozilla-central', 'try']
            linux64-stylo/.*: ['mozilla-central', 'try']
-            windows.*: ['try']
            default: ['mozilla-central', 'mozilla-inbound', 'autoland', 'try']
    max-run-time: 3600
    e10s: true
@ -1535,7 +1525,7 @@ talos-perf-reftest-singletons:
    try-name: perf-reftest-singletons
    treeherder-symbol: tc-T(ps)
    virtualization: hardware
-    run-on-projects: ['mozilla-central', 'try']
+    run-on-projects: ['mozilla-beta', 'mozilla-central', 'mozilla-inbound', 'autoland', 'try']
    max-run-time: 3600
    e10s: true
    mozharness:
@ -1546,7 +1536,7 @@ talos-perf-reftest-singletons:
                macosx.*:
                    - talos/mac_config.py
                windows.*:
-                    - talos/windows_taskcluster_config.py
+                    - talos/windows_config.py
                default:
                    - talos/linux_config.py
        extra-options:
@ -1599,7 +1589,7 @@ talos-quantum-pageload-stylo:
                macosx.*:
                    - talos/mac_config.py
                windows.*:
-                    - talos/windows_taskcluster_config.py
+                    - talos/windows_config.py
                default:
                    - talos/linux_config.py
        extra-options:
@ -1615,7 +1605,6 @@ talos-svgr:
        by-test-platform:
            linux64-stylo-sequential/.*: ['mozilla-central','try']
            linux64-stylo/.*: ['mozilla-central', 'try']
-            windows.*: ['try']
            default: ['mozilla-beta', 'mozilla-central', 'mozilla-inbound', 'autoland', 'try']
    max-run-time: 3600
    e10s: true
@ -1645,7 +1634,6 @@ talos-tp5o:
        by-test-platform:
            linux64-stylo-sequential/.*: ['mozilla-central','try']
            linux64-stylo/.*: ['mozilla-central', 'try']
-            windows.*: ['try']
            default: ['mozilla-beta', 'mozilla-central', 'mozilla-inbound', 'autoland', 'try']
    max-run-time: 3600
    e10s: true
@ -1807,7 +1795,15 @@ xpcshell:
            linux64-qr/.*: ['mozilla-central', 'try']
            windows10-64-asan/opt: []  # No XPCShell on ASAN yet
            default: built-projects
+    worker-type:
+        by-test-platform:
+            windows8-64.*: buildbot-bridge/buildbot-bridge
+            default: null
    docker-image: {"in-tree": "desktop1604-test"}
+    tier:
+        by-test-platform:
+            windows10-64.*: 2
+            default: default
    chunks:
        by-test-platform:
            linux64/debug: 10
@ -1820,10 +1816,6 @@ xpcshell:
            android.*: xlarge
            default: legacy # Bug 1281241: migrating to m3.large instances
    max-run-time: 5400
-    tier:
-        by-test-platform:
-            windows.*: 3
-            default: default
    e10s: false
    allow-software-gl-layers: false
    mozharness:
--- a/taskcluster/taskgraph/loader/build_signing.py
+++ b/taskcluster/taskgraph/loader/build_signing.py
@ -11,6 +11,7 @@ from taskgraph.loader.single_dep import loader as base_loader
 LABELS_WHICH_SHOULD_SIGN_CI_BUILDS = (
    'build-win32/debug', 'build-win32/opt', 'build-win32/pgo',
    'build-win64/debug', 'build-win64/opt', 'build-win64/pgo',
+    'build-win32-devedition/opt', 'build-win64-devedition/opt',
 )


--- a/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
+++ b/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
@ -78,6 +78,7 @@ JOB_NAME_WHITELIST = set([
    'win32-add-on-devel',
    'win32-clang-tidy',
    'win32-debug',
+    'win32-devedition-opt',
    'win32-nightly-repackage',
    'win32-nightly-repackage-signing',
    'win32-noopt-debug',
@ -89,6 +90,7 @@ JOB_NAME_WHITELIST = set([
    'win64-add-on-devel',
    'win64-clang-tidy',
    'win64-debug',
+    'win64-devedition-opt',
    'win64-nightly-repackage',
    'win64-nightly-repackage-signing',
    'win64-noopt-debug',
--- a/taskcluster/taskgraph/transforms/job/mozharness_test.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness_test.py
@ -38,10 +38,13 @@ BUILDER_NAME_PREFIX = {
    'windows10-64-pgo': 'Windows 10 64-bit',
    'windows10-64-asan': 'Windows 10 64-bit',
    'windows7-32': 'Windows 7 32-bit',
+    ('windows7-32', 'virtual-with-gpu'): 'Windows 7 VM-GFX 32-bit',
    'windows7-32-nightly': 'Windows 7 32-bit',
+    'windows7-32-devedition': 'Windows 7 32-bit DevEdition',
    'windows7-32-pgo': 'Windows 7 32-bit',
    'windows8-64': 'Windows 8 64-bit',
    'windows8-64-nightly': 'Windows 8 64-bit',
+    'windows8-64-devedition': 'Windows 8 64-bit DevEdition',
    'windows8-64-pgo': 'Windows 8 64-bit',
 }

@ -487,11 +490,19 @@ def mozharness_test_buildbot_bridge(config, job, taskdesc):
        if buildername.startswith('Ubuntu'):
            buildername = buildername.replace('VM', 'HW')
    else:
-        buildername = '{} {} {} test {}'.format(
-            BUILDER_NAME_PREFIX[test_platform],
-            branch,
-            build_type,
-            test_name
+        variant = get_variant(test['test-platform'])
+        # If we are a pgo type, munge the build_type for the
+        # Unittest builder name generation
+        if 'pgo' in variant:
+            build_type = variant
+        prefix = BUILDER_NAME_PREFIX.get(
+            (test_platform, test.get('virtualization')),
+            BUILDER_NAME_PREFIX[test_platform])
+        buildername = '{prefix} {branch} {build_type} test {test_name}'.format(
+            prefix=prefix,
+            branch=branch,
+            build_type=build_type,
+            test_name=test_name
        )

    worker.update({
--- a/taskcluster/taskgraph/transforms/signing.py
+++ b/taskcluster/taskgraph/transforms/signing.py
@ -96,9 +96,7 @@ def make_task_description(config, jobs):
            dep_th_platform, build_platform, build_type
        ))

-        # TODO: Make non-nightly (i.e. windows CI builds) Tier 1 once green on
-        # central, inbound, autoland and try
-        treeherder.setdefault('tier', 1 if is_nightly else 3)
+        treeherder.setdefault('tier', 1)
        treeherder.setdefault('kind', 'build')

        label = job.get('label', "{}-signing".format(dep_job.label))
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@ -61,6 +61,11 @@ WINDOWS_WORKER_TYPES = {
      'virtual-with-gpu': 'aws-provisioner-v1/gecko-t-win7-32-gpu',
      'hardware': 'releng-hardware/gecko-t-win7-32-hw',
    },
+    'windows7-32-devedition': {
+      'virtual': 'aws-provisioner-v1/gecko-t-win7-32',
+      'virtual-with-gpu': 'aws-provisioner-v1/gecko-t-win7-32-gpu',
+      'hardware': 'releng-hardware/gecko-t-win7-32-hw',
+    },
    'windows10-64': {
      'virtual': 'aws-provisioner-v1/gecko-t-win10-64',
      'virtual-with-gpu': 'aws-provisioner-v1/gecko-t-win10-64-gpu',
@ -71,6 +76,11 @@ WINDOWS_WORKER_TYPES = {
      'virtual-with-gpu': 'aws-provisioner-v1/gecko-t-win10-64-gpu',
      'hardware': 'releng-hardware/gecko-t-win10-64-hw',
    },
+    'windows10-64-devedition': {
+      'virtual': 'aws-provisioner-v1/gecko-t-win10-64',
+      'virtual-with-gpu': 'aws-provisioner-v1/gecko-t-win10-64-gpu',
+      'hardware': 'releng-hardware/gecko-t-win10-64-hw',
+    },
    'windows10-64-nightly': {
      'virtual': 'aws-provisioner-v1/gecko-t-win10-64',
      'virtual-with-gpu': 'aws-provisioner-v1/gecko-t-win10-64-gpu',
@ -503,7 +513,15 @@ def set_tier(config, tests):
                                         'linux64-devedition/opt',
                                         'linux64-asan/opt',
                                         'windows7-32/debug',
-                                         'windows7-32-vm/debug',
+                                         'windows7-32/opt',
+                                         'windows7-32-pgo/opt',
+                                         'windows7-32-devedition/opt',
+                                         'windows7-32-nightly/opt',
+                                         'windows10-64/debug',
+                                         'windows10-64/opt',
+                                         'windows10-64-pgo/opt',
+                                         'windows10-64-devedition/opt',
+                                         'windows10-64-nightly/opt',
                                         'macosx64/opt',
                                         'macosx64/debug',
                                         'android-4.3-arm7-api-15/opt',
@ -808,6 +826,18 @@ def set_worker_type(config, tests):
        yield test


+@transforms.add
+def skip_win10_hardware(config, tests):
+    """Windows 10 hardware isn't ready yet, don't even bother scheduling
+    unless we're on try"""
+    for test in tests:
+        if 'releng-hardware/gecko-t-win10-64-hw' not in test['worker-type']:
+            yield test
+        if config.params == 'try':
+            yield test
+        # Silently drop the test on the floor if its win10 hardware and we're not try
+
+
@transforms.add
 def make_job_description(config, tests):
    """Convert *test* descriptions to *job* descriptions (input to
--- a/testing/mozharness/configs/builds/releng_base_windows_32_builds.py
+++ b/testing/mozharness/configs/builds/releng_base_windows_32_builds.py
@ -40,8 +40,8 @@ config = {
                        'C:/mozilla-build/tooltool.py'],
    'tooltool_bootstrap': "setup.sh",
    'enable_count_ctors': False,
-    'enable_talos_sendchange': True,
-    'enable_unittest_sendchange': True,
+    'enable_talos_sendchange': False,
+    'enable_unittest_sendchange': False,
    'max_build_output_timeout': 60 * 80,
    #########################################################################

--- a/testing/mozharness/configs/builds/releng_base_windows_64_builds.py
+++ b/testing/mozharness/configs/builds/releng_base_windows_64_builds.py
@ -40,8 +40,8 @@ config = {
                        'C:/mozilla-build/tooltool.py'],
    'tooltool_bootstrap': "setup.sh",
    'enable_count_ctors': False,
-    'enable_talos_sendchange': True,
-    'enable_unittest_sendchange': True,
+    'enable_talos_sendchange': False,
+    'enable_unittest_sendchange': False,
    'max_build_output_timeout': 60 * 80,
    #########################################################################