Bug 1331957 - Part 8: Build cctools-port in the infrastructure; r=froydnj,dustin

taskcluster/ci/toolchain/linux.yml

@@ -91,3 +91,23 @@ linux64-binutils/opt:
             - 'build/unix/build-binutils/**'
             - 'taskcluster/scripts/misc/build-binutils-linux.sh'
             - 'taskcluster/taskgraph/transforms/job/toolchain.py'
+
+linux64-cctools-port/opt:
+    description: "cctools-port toolchain build"
+    treeherder:
+        kind: build
+        platform: linux64/opt
+        symbol: Cc(cctools-port)
+        tier: 1
+    run:
+        using: toolchain-script
+        script: build-cctools-port.sh
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        implementation: docker-worker
+        docker-image: {in-tree: desktop-build}
+        max-run-time: 36000
+    when:
+        files-changed:
+            - 'taskcluster/scripts/misc/build-cctools-port.sh'
+            - 'taskcluster/taskgraph/transforms/job/toolchain.py'

taskcluster/ci/toolchain/macosx.yml

@@ -50,3 +50,23 @@ macosx64-clang-tidy/opt:
             - 'taskcluster/scripts/misc/build-clang-tidy-macosx.sh'
             - 'taskcluster/taskgraph/transforms/job/toolchain.py'
 
+macosx64-cctools-port/opt:
+    description: "cctools-port toolchain build"
+    treeherder:
+        kind: build
+        platform: osx-10-10/opt
+        symbol: Cc(cctools-port)
+        tier: 1
+    run:
+        using: toolchain-script
+        script: build-cctools-port-macosx.sh
+        tooltool-downloads: internal
+    worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64
+    worker:
+        implementation: docker-worker
+        docker-image: {in-tree: desktop-build}
+        max-run-time: 36000
+    when:
+        files-changed:
+            - 'taskcluster/scripts/misc/build-cctools-port-macosx.sh'
+            - 'taskcluster/taskgraph/transforms/job/toolchain.py'

taskcluster/docker/desktop-build/Dockerfile

@@ -67,5 +67,8 @@ ADD           buildprops.json /home/worker/
 # Install bison to build binutils.
 RUN yum install -y bison screen
 
+# Install libtool.
+RUN yum install -y libtool
+
 # Set a default command useful for debugging
 CMD ["/bin/bash", "--login"]

taskcluster/scripts/misc/build-cctools-port-macosx.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+set -x -e -v
+
+# This script is for building cctools (Apple's binutils) for Mac OS X on
+# Linux using ctools-port (https://github.com/tpoechtrager/cctools-port).
+
+WORKSPACE=$HOME/workspace
+UPLOAD_DIR=$WORKSPACE/artifacts
+
+# Repository info
+: CROSSTOOL_PORT_REPOSITORY    ${CROSSTOOL_PORT_REPOSITORY:=https://github.com/tpoechtrager/cctools-port}
+: CROSSTOOL_PORT_REV           ${CROSSTOOL_PORT_REV:=master}
+
+# Set some crosstools-port directories
+CROSSTOOLS_SOURCE_DIR=$WORKSPACE/crosstools-port
+CROSSTOOLS_CCTOOLS_DIR=$CROSSTOOLS_SOURCE_DIR/cctools
+CROSSTOOLS_BUILD_DIR=/tmp/cctools
+CLANG_DIR=$WORKSPACE/clang
+CCTOOLS_DIR=$WORKSPACE/cctools
+MACOSX_SDK_DIR=$WORKSPACE/MacOSX10.10.sdk
+
+TARGET_TRIPLE=x86_64-apple-darwin11
+
+# Create our directories
+mkdir -p $CROSSTOOLS_BUILD_DIR
+
+tc-vcs checkout --force-clone $CROSSTOOLS_SOURCE_DIR $CROSSTOOL_PORT_REPOSITORY $CROSSTOOL_PORT_REPOSITORY $CROSSTOOL_PORT_REV
+cd $CROSSTOOLS_SOURCE_DIR
+echo "Building from commit hash `git rev-parse $CROSSTOOL_PORT_REV`..."
+
+# Fetch clang from tooltool
+cd $WORKSPACE
+wget -O tooltool.py https://raw.githubusercontent.com/mozilla/build-tooltool/master/tooltool.py
+chmod +x tooltool.py
+: TOOLTOOL_CACHE                ${TOOLTOOL_CACHE:=/home/worker/tooltool-cache}
+export TOOLTOOL_CACHE
+
+wget ${GECKO_HEAD_REPOSITORY}/raw-file/${GECKO_HEAD_REV}/browser/config/tooltool-manifests/macosx64/cross-clang.manifest
+
+python tooltool.py -v --manifest=cross-clang.manifest --url=http://relengapi/tooltool/ fetch
+
+# Configure crosstools-port
+cd $CROSSTOOLS_CCTOOLS_DIR
+export CC=$CLANG_DIR/bin/clang
+export CXX=$CLANG_DIR/bin/clang++
+export CFLAGS="-mcpu=generic -mtune=generic -O3 -target $TARGET_TRIPLE -isysroot $MACOSX_SDK_DIR"
+export CXXFLAGS="-mcpu=generic -mtune=generic -O3 -target $TARGET_TRIPLE -isysroot $MACOSX_SDK_DIR"
+export LDFLAGS="-Wl,-syslibroot,$MACOSX_SDK_DIR -Wl,-dead_strip"
+export PATH="$CCTOOLS_DIR/bin:$PATH"
+./autogen.sh
+./configure --prefix=$CROSSTOOLS_BUILD_DIR --build=$MACHTYPE --host=$TARGET_TRIPLE --with-llvm-config=$CLANG_DIR/bin/llvm-config
+
+# Build cctools
+make -j `nproc --all` install
+$CCTOOLS_DIR/bin/$TARGET_TRIPLE-strip $CROSSTOOLS_BUILD_DIR/bin/*
+
+# Put a tarball in the artifacts dir
+mkdir -p $UPLOAD_DIR
+tar cjf $UPLOAD_DIR/cctools.tar.bz2 -C $CROSSTOOLS_BUILD_DIR/.. `basename $CROSSTOOLS_BUILD_DIR`
+

taskcluster/scripts/misc/build-cctools-port.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+set -x -e -v
+
+# This script is for building cctools (Apple's binutils) for Linux using
+# cctools-port (https://github.com/tpoechtrager/cctools-port).
+
+WORKSPACE=$HOME/workspace
+UPLOAD_DIR=$WORKSPACE/artifacts
+
+# Repository info
+: CROSSTOOL_PORT_REPOSITORY    ${CROSSTOOL_PORT_REPOSITORY:=https://github.com/tpoechtrager/cctools-port}
+: CROSSTOOL_PORT_REV           ${CROSSTOOL_PORT_REV:=master}
+
+# Set some crosstools-port directories
+CROSSTOOLS_SOURCE_DIR=$WORKSPACE/crosstools-port
+CROSSTOOLS_CCTOOLS_DIR=$CROSSTOOLS_SOURCE_DIR/cctools
+CROSSTOOLS_BUILD_DIR=$WORKSPACE/cctools
+CLANG_DIR=$WORKSPACE/clang
+
+# Create our directories
+mkdir -p $CROSSTOOLS_BUILD_DIR
+
+tc-vcs checkout --force-clone $CROSSTOOLS_SOURCE_DIR $CROSSTOOL_PORT_REPOSITORY $CROSSTOOL_PORT_REPOSITORY $CROSSTOOL_PORT_REV
+cd $CROSSTOOLS_SOURCE_DIR
+echo "Building from commit hash `git rev-parse $CROSSTOOL_PORT_REV`..."
+
+# Fetch clang from tooltool
+cd $WORKSPACE
+wget -O tooltool.py https://raw.githubusercontent.com/mozilla/build-tooltool/master/tooltool.py
+chmod +x tooltool.py
+: TOOLTOOL_CACHE                ${TOOLTOOL_CACHE:=/home/worker/tooltool-cache}
+export TOOLTOOL_CACHE
+
+wget ${GECKO_HEAD_REPOSITORY}/raw-file/${GECKO_HEAD_REV}/browser/config/tooltool-manifests/linux64/clang.manifest
+
+python tooltool.py -v --manifest=clang.manifest fetch
+
+# Configure crosstools-port
+cd $CROSSTOOLS_CCTOOLS_DIR
+export CC=$CLANG_DIR/bin/clang
+export CXX=$CLANG_DIR/bin/clang++
+export LDFLAGS=/lib64/libpthread.so.0
+./autogen.sh
+./configure --prefix=$CROSSTOOLS_BUILD_DIR --target=x86_64-apple-darwin11 --with-llvm-config=$CLANG_DIR/bin/llvm-config
+
+# Build cctools
+make -j `nproc --all` install
+strip $CROSSTOOLS_BUILD_DIR/bin/*
+
+# Put a tarball in the artifacts dir
+mkdir -p $UPLOAD_DIR
+tar cJf $UPLOAD_DIR/cctools.tar.xz -C $CROSSTOOLS_BUILD_DIR/.. `basename $CROSSTOOLS_BUILD_DIR`