Bug 1391408 - Build minidump_stackwalk as a toolchain. r=froydnj

This produces the same executables (produced for the same platforms) as
those currently pulled from tooltool (modulo timestamps, maybe changes
since last manifest change, etc.). Unfortunately, as of currently, the
Windows variant needs to be cross-built with mingw because it doesn't
compile without some POSIX APIs that MSVC/Windows SDK don't provide.

One thing that is left out of this change is whether to be completely
accurate with the toolchain cache hash (requiring a large list of files
as resources, and making those built very frequently), whether we'd
rely on manual updates, or if we should go with completely uncached
tasks. This can be left for a followup, the tasks not being hooked up
to be actually used by other tasks yet.

Differential Revision: https://phabricator.services.mozilla.com/D16302

--HG--
extra : moz-landing-system : lando

taskcluster/ci/toolchain/linux.yml

@@ -55,7 +55,7 @@ linux64-clang-trunk-mingw-x86:
         kind: build
         platform: toolchains/opt
         symbol: TMW(clang-x86)
-        tier: 2
+        tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux-xlarge
     worker:
         max-run-time: 7200
@@ -905,3 +905,25 @@ wrench-deps:
         toolchain-artifact: public/build/wrench-deps.tar.bz2
     toolchains:
         - linux64-rust-1.31  # whatever m-c is built with
+
+linux64-minidump-stackwalk:
+    description: "minidump_stackwalk toolchain build"
+    treeherder:
+        kind: build
+        platform: toolchains/opt
+        symbol: TL(stackwalk)
+        tier: 1
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        max-run-time: 1800
+    run:
+        using: toolchain-script
+        script: build-minidump-stackwalk.sh
+        sparse-profile: null
+        resources:
+            - 'taskcluster/scripts/misc/tooltool-download.sh'
+        toolchain-artifact: public/build/minidump_stackwalk.tar.xz
+    toolchains:
+        - linux64-clang-7
+        - linux64-binutils
+        - linux64-rust-1.31

taskcluster/ci/toolchain/macosx.yml

@@ -183,3 +183,29 @@ macosx64-cbindgen:
         - linux64-clang-7
         - linux64-llvm-dsymutil
         - linux64-rust-macos-1.31
+
+macosx64-minidump-stackwalk:
+    description: "minidump_stackwalk toolchain build"
+    treeherder:
+        kind: build
+        platform: toolchains/opt
+        symbol: TM(stackwalk)
+        tier: 1
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        max-run-time: 1800
+        env:
+            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
+    run:
+        using: toolchain-script
+        script: build-minidump-stackwalk.sh
+        sparse-profile: null
+        arguments: ['macosx64']
+        tooltool-downloads: internal
+        resources:
+            - 'taskcluster/scripts/misc/tooltool-download.sh'
+        toolchain-artifact: public/build/minidump_stackwalk.tar.xz
+    toolchains:
+        - linux64-cctools-port
+        - linux64-clang-7
+        - linux64-rust-macos-1.31

taskcluster/ci/toolchain/windows.yml

@@ -336,3 +336,26 @@ win32-gn:
     run-on-projects:
         - trunk
         - try
+
+win32-minidump-stackwalk:
+    description: "minidump_stackwalk toolchain build"
+    treeherder:
+        kind: build
+        platform: toolchains/opt
+        symbol: TW32(stackwalk)
+        tier: 1
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        docker-image: {in-tree: mingw32-build}
+        max-run-time: 1800
+    run:
+        using: toolchain-script
+        script: build-minidump-stackwalk.sh
+        sparse-profile: null
+        arguments: ['mingw32']
+        resources:
+            - 'taskcluster/scripts/misc/tooltool-download.sh'
+        toolchain-artifact: public/build/minidump_stackwalk.tar.xz
+    toolchains:
+        - linux64-clang-trunk-mingw-x86
+        - mingw32-rust-1.31

taskcluster/docker/toolchain-build/Dockerfile

@@ -15,6 +15,7 @@ RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAG
 RUN apt-get update && \
     apt-get install \
       autoconf \
+      autoconf2.13 \
       automake \
       bison \
       build-essential \
@@ -25,6 +26,7 @@ RUN apt-get update && \
       gcc-multilib \
       gnupg \
       libbz2-dev \
+      libcurl4-openssl-dev \
       libssl-dev \
       libtool \
       ninja-build \

taskcluster/scripts/misc/build-minidump-stackwalk.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+set -x -e -v
+
+# This script is for building minidump_stackwalk
+
+WORKSPACE=$HOME/workspace
+UPLOAD_DIR=$HOME/artifacts
+COMPRESS_EXT=xz
+
+cd $WORKSPACE/build/src
+
+. taskcluster/scripts/misc/tooltool-download.sh
+
+export MOZ_OBJDIR=obj-minidump
+
+echo ac_add_options --enable-project=tools/crashreporter > .mozconfig
+
+MINIDUMP_STACKWALK=minidump_stackwalk
+
+case "$1" in
+macosx64)
+    TOOLCHAINS="cctools rustc clang"
+    echo ac_add_options --target=x86_64-apple-darwin11 >> .mozconfig
+    echo ac_add_options --with-macos-sdk=$WORKSPACE/build/src/MacOSX10.11.sdk >> .mozconfig
+    ;;
+mingw32)
+    TOOLCHAINS="binutils rustc clang"
+    echo ac_add_options --target=i686-w64-mingw32 >> .mozconfig
+    echo export CC=i686-w64-mingw32-clang >> .mozconfig
+    echo export HOST_CC=clang >> .mozconfig
+    MINIDUMP_STACKWALK=minidump_stackwalk.exe
+    ;;
+*)
+    TOOLCHAINS="binutils rustc clang"
+    ;;
+esac
+
+for t in $TOOLCHAINS; do
+    PATH="$WORKSPACE/build/src/$t/bin:$PATH"
+done
+
+./mach build -v
+
+mkdir minidump_stackwalk
+cp $MOZ_OBJDIR/dist/bin/$MINIDUMP_STACKWALK minidump_stackwalk/
+
+tar -acf minidump_stackwalk.tar.$COMPRESS_EXT minidump_stackwalk/
+mkdir -p $UPLOAD_DIR
+cp minidump_stackwalk.tar.$COMPRESS_EXT $UPLOAD_DIR

taskcluster/scripts/misc/minidump_stackwalk.sh

@@ -1,125 +0,0 @@
-#!/bin/bash
-#
-# This script builds minidump_stackwalk binaries from the Google Breakpad
-# source for all of the operating systems that we run Firefox tests on:
-# Linux x86, Linux x86-64, Windows x86, OS X x86-64.
-#
-# It expects to be run in the luser/breakpad-builder:0.7 Docker image and
-# needs access to the relengapiproxy to download internal tooltool files.
-
-set -v -e -x
-
-# This is a pain to support properly with gclient.
-#: BREAKPAD_REPO        ${BREAKPAD_REPO:=https://google-breakpad.googlecode.com/svn/trunk/}
-: BREAKPAD_REV         "${BREAKPAD_REV:=master}"
-: STACKWALK_HTTP_REPO  "${STACKWALK_HTTP_REPO:=https://hg.mozilla.org/users/tmielczarek_mozilla.com/stackwalk-http}"
-: STACKWALK_HTTP_REV   "${STACKWALK_HTTP_REV:=default}"
-
-ncpu=$(getconf _NPROCESSORS_ONLN)
-
-function build()
-{
-    cd /tmp
-    local platform=$1
-    local strip_prefix=$2
-    local configure_args=$3
-    local make_args=$4
-    local objdir=/tmp/obj-breakpad-$platform
-    local ext=
-    if test "$platform" = "win32"; then
-        ext=.exe
-    fi
-    rm -rf "$objdir"
-    mkdir "$objdir"
-    # First, build Breakpad
-    cd "$objdir"
-    # shellcheck disable=SC2086
-    CFLAGS="-O2 $CFLAGS" CXXFLAGS="-O2 $CXXFLAGS" /tmp/breakpad/src/configure --disable-tools $configure_args
-    # shellcheck disable=SC2086
-    make -j$ncpu $make_args src/libbreakpad.a src/third_party/libdisasm/libdisasm.a src/processor/stackwalk_common.o
-    # Second, build stackwalk-http
-    make -f /tmp/stackwalk-http/Makefile BREAKPAD_SRCDIR=/tmp/breakpad/src "BREAKPAD_OBJDIR=$(pwd)" "OS=$platform" "-j$ncpu"
-    "${strip_prefix}strip" "stackwalk${ext}"
-    cp "stackwalk${ext}" "/tmp/stackwalker/${platform}-minidump_stackwalk${ext}"
-}
-
-function linux64()
-{
-    export LDFLAGS="-static-libgcc -static-libstdc++"
-    build linux64
-    unset LDFLAGS
-}
-
-function linux32()
-{
-    export LDFLAGS="-static-libgcc -static-libstdc++ -L/tmp/libcurl-i386/lib"
-    export CFLAGS="-m32 -I/tmp/libcurl-i386/include"
-    export CXXFLAGS="-m32 -I/tmp/libcurl-i386/include"
-    build linux32 "" "--enable-m32"
-    unset LDFLAGS CFLAGS CXXFLAGS
-}
-
-function macosx64()
-{
-    cd /tmp
-    if ! test -d MacOSX10.7.sdk; then
-      python tooltool.py -v --manifest=macosx-sdk.manifest --url=http://relengapi/tooltool/ fetch
-    fi
-    export MACOSX_SDK=/tmp/MacOSX10.7.sdk
-    export CCTOOLS=/tmp/cctools
-    local FLAGS="-stdlib=libc++ -target x86_64-apple-darwin10 -mlinker-version=136 -B /tmp/cctools/bin -isysroot ${MACOSX_SDK} -mmacosx-version-min=10.7"
-    export CC="clang $FLAGS"
-    export CXX="clang++ $FLAGS -std=c++11"
-    local old_path="$PATH"
-    export PATH="/tmp/clang/bin:/tmp/cctools/bin/:$PATH"
-    export LD_LIBRARY_PATH=/usr/lib/llvm-3.6/lib/
-
-    build macosx64 "/tmp/cctools/bin/x86_64-apple-darwin10-" "--host=x86_64-apple-darwin10" "AR=/tmp/cctools/bin/x86_64-apple-darwin10-ar"
-
-    unset CC CXX LD_LIBRARY_PATH MACOSX_SDK CCTOOLS
-    export PATH="$old_path"
-}
-
-function win32()
-{
-    export LDFLAGS="-static-libgcc -static-libstdc++"
-    export CFLAGS="-D__USE_MINGW_ANSI_STDIO=1"
-    export CXXFLAGS="-D__USE_MINGW_ANSI_STDIO=1"
-    export ZLIB_DIR=/tmp/zlib-mingw
-    build win32 "i686-w64-mingw32-" "--host=i686-w64-mingw32"
-    unset LDFLAGS CFLAGS CXXFLAGS ZLIB_DIR
-}
-
-cd /tmp
-if ! test -d depot_tools; then
-  git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git
-else
-  (cd depot_tools; git pull origin master)
-fi
-export PATH=$(pwd)/depot_tools:"$PATH"
-if ! test -d breakpad; then
-    mkdir breakpad
-    pushd breakpad
-    fetch breakpad
-    popd
-else
-    pushd breakpad/src
-    git pull origin master
-    popd
-fi
-pushd breakpad/src
-git checkout "${BREAKPAD_REV}"
-gclient sync
-popd
-
-(cd breakpad/src; git rev-parse master)
-if ! test -d stackwalk-http; then
-  hg clone -u "$STACKWALK_HTTP_REV" "$STACKWALK_HTTP_REPO"
-else
-  (cd stackwalk-http && hg pull "$STACKWALK_HTTP_REPO" && hg up "$STACKWALK_HTTP_REV")
-fi
-mkdir -p stackwalker
-linux64
-linux32
-macosx64
-win32