Bug 1403548: Enable nightly asan reportor builds and updates r=Callek

MozReview-Commit-ID: 2X03vU5bO32 --HG-- extra : rebase_source : d4b6672eae5b378ac5a5d80d7f14d193d201f517
2018-04-19 09:28:59 -04:00 · 2018-04-19 09:28:59 -04:00 · a8def10cc3
--- a/browser/config/mozconfigs/linux64/nightly-asan-reporter
+++ b/browser/config/mozconfigs/linux64/nightly-asan-reporter
@ -1,6 +1,8 @@
 # We still need to build with debug symbols
 ac_add_options --disable-debug
 ac_add_options --enable-optimize="-O2 -gline-tables-only"
+ac_add_options --enable-update-channel=${MOZ_UPDATE_CHANNEL}
+MOZ_AUTOMATION_UPDATE_PACKAGING=${MOZ_AUTOMATION_UPDATE_PACKAGING-1}

 . $topsrcdir/build/mozconfig.stylo

@ -15,4 +17,7 @@ ac_add_options --enable-address-sanitizer-reporter
 # Need this to prevent name conflicts with the normal nightly build packages
 export MOZ_PKG_SPECIAL=asan-reporter

+# Need this to add source information into platform.ini
+export MOZILLA_OFFICIAL=1
+
 . "$topsrcdir/build/mozconfig.common.override"
--- a/browser/confvars.sh
+++ b/browser/confvars.sh
@ -53,6 +53,12 @@ else
  ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-central
  MAR_CHANNEL_ID=firefox-mozilla-central
 fi
+# ASan reporter builds should have different channel ids
+if [ "${MOZ_ASAN_REPORTER}" = "1" ]; then
+    ACCEPTED_MAR_CHANNEL_IDS="${ACCEPTED_MAR_CHANNEL_IDS}-asan"
+    MAR_CHANNEL_ID="${MAR_CHANNEL_ID}-asan"
+fi
+
 MOZ_PROFILE_MIGRATOR=1

 # Enable checking that add-ons are signed by the trusted root
--- a/taskcluster/ci/beetmover-repackage/kind.yml
+++ b/taskcluster/ci/beetmover-repackage/kind.yml
@ -26,6 +26,7 @@ only-for-build-platforms:
   - macosx64-devedition-nightly/opt
   - win32-devedition-nightly/opt
   - win64-devedition-nightly/opt
+   - linux64-asan-reporter-nightly/opt

 job-template:
   shipping-phase: promote
--- a/taskcluster/ci/beetmover/kind.yml
+++ b/taskcluster/ci/beetmover/kind.yml
@ -27,3 +27,4 @@ not-for-build-platforms:
   - macosx64-devedition-nightly/opt
   - win32-devedition-nightly/opt
   - win64-devedition-nightly/opt
+   - linux64-asan-reporter-nightly/opt
--- a/taskcluster/ci/build/linux.yml
+++ b/taskcluster/ci/build/linux.yml
@ -526,13 +526,16 @@ linux64-asan-fuzzing/opt:
        - linux64-rust
        - linux64-sccache

-linux64-asan-reporter/opt:
+linux64-asan-reporter-nightly/opt:
    description: "Linux64 Opt ASAN Reporter Nightly"
+    attributes:
+        nightly: true
    index:
        product: firefox
        job-name: linux64-asan-reporter-opt
+        type: nightly
    treeherder:
-        platform: linux64/asan
+        platform: linux64-asan-reporter/opt
        symbol: BoR
    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
    worker:
@ -545,6 +548,7 @@ linux64-asan-reporter/opt:
        config:
            - builds/releng_base_firefox.py
            - builds/releng_base_linux_64_builds.py
+            - taskcluster_nightly.py
        script: "mozharness/scripts/fx_desktop_build.py"
        secrets: true
        custom-build-variant-cfg: asan-reporter-tc
--- a/taskcluster/ci/partials/kind.yml
+++ b/taskcluster/ci/partials/kind.yml
@ -26,6 +26,7 @@ only-for-build-platforms:
   - linux-devedition-nightly/opt
   - linux64-nightly/opt
   - linux64-devedition-nightly/opt
+   - linux64-asan-reporter-nightly/opt

 job-template:
   shipping-phase: promote
--- a/taskcluster/ci/repackage-signing/kind.yml
+++ b/taskcluster/ci/repackage-signing/kind.yml
@ -20,6 +20,7 @@ only-for-build-platforms:
   - linux-devedition-nightly/opt
   - linux64-nightly/opt
   - linux64-devedition-nightly/opt
+   - linux64-asan-reporter-nightly/opt
   - macosx64-nightly/opt
   - macosx64-devedition-nightly/opt
   - win32-nightly/opt
--- a/taskcluster/ci/repackage/kind.yml
+++ b/taskcluster/ci/repackage/kind.yml
@ -21,6 +21,7 @@ only-for-build-platforms:
    - linux-devedition-nightly/opt
    - linux64-nightly/opt
    - linux64-devedition-nightly/opt
+    - linux64-asan-reporter-nightly/opt
    - macosx64-nightly/opt
    - macosx64-devedition-nightly/opt
    - win32-nightly/opt
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@ -603,6 +603,15 @@ def target_tasks_nightly_win64(full_task_graph, parameters, graph_config):
    return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]


+@_target_task('nightly_asan')
+def target_tasks_nightly_asan(full_task_graph, parameters, graph_config):
+    """Select the set of tasks required for a nightly build of asan. The
+    nightly build process involves a pipeline of builds, signing,
+    and, eventually, uploading the tasks to balrog."""
+    filter = make_nightly_filter({'linux64-asan-reporter-nightly'})
+    return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]
+
+
@_target_task('nightly_desktop')
 def target_tasks_nightly_desktop(full_task_graph, parameters, graph_config):
    """Select the set of tasks required for a nightly build of linux, mac,
@ -613,6 +622,7 @@ def target_tasks_nightly_desktop(full_task_graph, parameters, graph_config):
        | set(target_tasks_nightly_win64(full_task_graph, parameters, graph_config))
        | set(target_tasks_nightly_macosx(full_task_graph, parameters, graph_config))
        | set(target_tasks_nightly_linux(full_task_graph, parameters, graph_config))
+        | set(target_tasks_nightly_asan(full_task_graph, parameters, graph_config))
    )


--- a/taskcluster/taskgraph/transforms/beetmover.py
+++ b/taskcluster/taskgraph/transforms/beetmover.py
@ -136,6 +136,14 @@ UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
        "host/bin/mar",
        "host/bin/mbsdiff",
    ],
+    'linux64-asan-reporter-nightly':
+        # ASan reporter builds don't generate the regular crashreporter symbol
+        # packages, so we shouldn't try to beetmove them
+        filter(lambda a: a != 'target.crashreporter-symbols.zip',
+               _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
+                    "host/bin/mar",
+                    "host/bin/mbsdiff",
+                ]),
    'linux64-source': [
    ],
    'linux64-devedition-source': [
@ -205,6 +213,10 @@ UPSTREAM_ARTIFACT_SIGNED_PATHS = {
        "target.tar.bz2",
        "target.tar.bz2.asc",
    ],
+    'linux64-asan-reporter-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
+        "target.tar.bz2",
+        "target.tar.bz2.asc",
+    ],
    'linux64-source': [
        "source.tar.xz",
        "source.tar.xz.asc",
--- a/taskcluster/taskgraph/transforms/beetmover_repackage.py
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage.py
@ -75,6 +75,12 @@ UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
            # TODO Bug 1453033: Sign devedition langpacks
            'target.langpack.xpi',
        ],
+    r'^linux64-asan-reporter-nightly$':
+        filter(lambda a: a not in ('target.crashreporter-symbols.zip', 'target.jsshell.zip'),
+               _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
+                    "host/bin/mar",
+                    "host/bin/mbsdiff",
+                ]),
    r'^win(32|64)-nightly$':
        _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
            'host/bin/mar.exe',
@ -100,7 +106,8 @@ UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
 UPSTREAM_ARTIFACT_SIGNED_PATHS = {
-    r'^linux(|64)(|-devedition)-nightly(|-l10n)$': ['target.tar.bz2', 'target.tar.bz2.asc'],
+    r'^linux(|64)(|-devedition|-asan-reporter)-nightly(|-l10n)$':
+        ['target.tar.bz2', 'target.tar.bz2.asc'],
    r'^win(32|64)(|-devedition)-nightly(|-l10n)$': ['target.zip'],
 }

@ -116,7 +123,8 @@ UPSTREAM_ARTIFACT_REPACKAGE_PATHS = {
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
 UPSTREAM_ARTIFACT_SIGNED_REPACKAGE_PATHS = {
-    r'^(linux(|64)|macosx64)(|-devedition)-nightly(|-l10n)$': ['target.complete.mar'],
+    r'^(linux(|64)|macosx64)(|-devedition|-asan-reporter)-nightly(|-l10n)$':
+        ['target.complete.mar'],
    r'^win64(|-devedition)-nightly(|-l10n)$': ['target.complete.mar', 'target.installer.exe'],
    r'^win32(|-devedition)-nightly(|-l10n)$': [
        'target.complete.mar',
@ -291,11 +299,11 @@ def generate_upstream_artifacts(job, build_task_ref, build_signing_task_ref,
    ]

    for ref, tasktype, mapping in zip(task_refs, tasktypes, mapping):
-        plarform_was_previously_matched_by_regex = None
+        platform_was_previously_matched_by_regex = None
        for platform_regex, paths in mapping.iteritems():
            if platform_regex.match(platform) is not None:
                _check_platform_matched_only_one_regex(
-                    tasktype, platform, plarform_was_previously_matched_by_regex, platform_regex
+                    tasktype, platform, platform_was_previously_matched_by_regex, platform_regex
                )
                if paths:
                    upstream_artifacts.append({
@ -304,7 +312,7 @@ def generate_upstream_artifacts(job, build_task_ref, build_signing_task_ref,
                        "paths": ["{}/{}".format(artifact_prefix, path) for path in paths],
                        "locale": locale or "en-US",
                    })
-                plarform_was_previously_matched_by_regex = platform_regex
+                platform_was_previously_matched_by_regex = platform_regex

    return upstream_artifacts

@ -326,14 +334,14 @@ def generate_partials_upstream_artifacts(job, artifacts, platform, locale=None):


 def _check_platform_matched_only_one_regex(
-    task_type, platform, plarform_was_previously_matched_by_regex, platform_regex
+    task_type, platform, platform_was_previously_matched_by_regex, platform_regex
 ):
-    if plarform_was_previously_matched_by_regex is not None:
+    if platform_was_previously_matched_by_regex is not None:
        raise Exception('In task type "{task_type}", platform "{platform}" matches at \
 least 2 regular expressions. First matched: "{first_matched}". Second matched: \
 "{second_matched}"'.format(
            task_type=task_type, platform=platform,
-            first_matched=plarform_was_previously_matched_by_regex.pattern,
+            first_matched=platform_was_previously_matched_by_regex.pattern,
            second_matched=platform_regex.pattern
        ))

--- a/taskcluster/taskgraph/util/partials.py
+++ b/taskcluster/taskgraph/util/partials.py
@ -28,6 +28,9 @@ BALROG_PLATFORM_MAP = {
    "linux64": [
        "Linux_x86_64-gcc3"
    ],
+    "linux64-asan-reporter": [
+        "Linux_x86_64-gcc3-asan"
+    ],
    "macosx64": [
        "Darwin_x86_64-gcc3-u-i386-x86_64",
        "Darwin_x86-gcc3-u-i386-x86_64",
@ -52,6 +55,7 @@ FTP_PLATFORM_MAP = {
    "Darwin_x86_64-gcc3-u-i386-x86_64": "mac",
    "Linux_x86-gcc3": "linux-i686",
    "Linux_x86_64-gcc3": "linux-x86_64",
+    "Linux_x86_64-gcc3-asan": "linux-x86_64-asan-reporter",
    "WINNT_x86-msvc": "win32",
    "WINNT_x86-msvc-x64": "win32",
    "WINNT_x86-msvc-x86": "win32",
--- a/toolkit/moz.configure
+++ b/toolkit/moz.configure
@ -1159,6 +1159,7 @@ def enable_asan_reporter(value):

 set_config('MOZ_ASAN_REPORTER', enable_asan_reporter)
 set_define('MOZ_ASAN_REPORTER', enable_asan_reporter)
+add_old_configure_assignment('MOZ_ASAN_REPORTER', enable_asan_reporter)

 # Elfhack
 # ==============================================================