From 53ed8c41811a50da6b03811875a15cdcf4e98b61 Mon Sep 17 00:00:00 2001 From: Steve Fink Date: Mon, 15 Feb 2016 21:06:42 -0800 Subject: [PATCH] Bug 1250709 - Taskcluster-based browser rooting hazard analysis, r=garndt --HG-- extra : rebase_source : 3223926ea521639f910093e54297eab38e3a60e1 --- browser/config/mozconfigs/linux64/hazards | 7 ++- .../builder/build-browser-haz-linux.sh | 27 +++++++++ .../taskcluster/scripts/builder/get-objdir.py | 2 +- .../scripts/builder/hazard-analysis.sh | 33 +++++++++-- .../tasks/branches/try/job_flags.yml | 6 ++ .../taskcluster/tasks/builds/haz_linux.yml | 59 +++++++++++++++++++ 6 files changed, 125 insertions(+), 9 deletions(-) create mode 100755 testing/taskcluster/scripts/builder/build-browser-haz-linux.sh create mode 100644 testing/taskcluster/tasks/builds/haz_linux.yml diff --git a/browser/config/mozconfigs/linux64/hazards b/browser/config/mozconfigs/linux64/hazards index ebe46ccb82ef..593dd88c733e 100644 --- a/browser/config/mozconfigs/linux64/hazards +++ b/browser/config/mozconfigs/linux64/hazards @@ -1,5 +1,5 @@ -# This mozconfig is used when compiling the browser for the SM(Hf) rooting -# hazard analysis build, see +# This mozconfig is used when compiling the browser for the rooting hazard +# analysis build (labeled H on treeherder). See # https://wiki.mozilla.org/Javascript:SpiderMonkey:ExactStackRooting # Do NOT include build/unix/mozconfig.linux because it points directly at the @@ -21,12 +21,13 @@ mk_add_options MOZ_OBJDIR=obj-analyzed ac_add_options --enable-debug ac_add_options --enable-tests ac_add_options --enable-optimize +ac_add_options --with-compiler-wrapper=$(cd $topsrcdir/sixgill/usr/libexec/sixgill/scripts/wrap_gcc && echo $PWD)/basecc +ac_add_options --without-ccache CFLAGS="$CFLAGS -Wno-attributes" CPPFLAGS="$CPPFLAGS -Wno-attributes" CXXFLAGS="$CXXFLAGS -Wno-attributes" -TOOLTOOL_DIR="$(dirname $topsrcdir)" export PKG_CONFIG_LIBDIR=/usr/lib64/pkgconfig:/usr/share/pkgconfig . $topsrcdir/build/unix/mozconfig.gtk diff --git a/testing/taskcluster/scripts/builder/build-browser-haz-linux.sh b/testing/taskcluster/scripts/builder/build-browser-haz-linux.sh new file mode 100755 index 000000000000..1b95c29df929 --- /dev/null +++ b/testing/taskcluster/scripts/builder/build-browser-haz-linux.sh @@ -0,0 +1,27 @@ +#!/bin/bash -ex + +################################### build-mulet-haz-linux.sh ################################### +# Ensure all the scripts in this dir are on the path.... +DIRNAME=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +PATH=$DIRNAME:$PATH + +. desktop-setup.sh +. hazard-analysis.sh + +build_js_shell + +# Artifacts folder is outside of the cache. +mkdir -p $HOME/artifacts/ || true + +function onexit () { + grab_artifacts "$WORKSPACE/analysis" "$HOME/artifacts" +} + +trap onexit EXIT + +configure_analysis "$WORKSPACE/analysis" +run_analysis "$WORKSPACE/analysis" browser + +check_hazards "$WORKSPACE/analysis" + +################################### script end ################################### diff --git a/testing/taskcluster/scripts/builder/get-objdir.py b/testing/taskcluster/scripts/builder/get-objdir.py index ec17e56ecd34..132e20d4f60a 100755 --- a/testing/taskcluster/scripts/builder/get-objdir.py +++ b/testing/taskcluster/scripts/builder/get-objdir.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2.7 from __future__ import print_function import sys diff --git a/testing/taskcluster/scripts/builder/hazard-analysis.sh b/testing/taskcluster/scripts/builder/hazard-analysis.sh index 66c7effbe4b1..2a8a9b22b518 100755 --- a/testing/taskcluster/scripts/builder/hazard-analysis.sh +++ b/testing/taskcluster/scripts/builder/hazard-analysis.sh @@ -1,6 +1,10 @@ #!/bin/bash -ex -HAZARD_SHELL_OBJDIR=$MOZ_OBJDIR/obj-haz-shell +[ -n "$WORKSPACE" ] +[ -n "$MOZ_OBJDIR" ] +[ -n "$GECKO_DIR" ] + +HAZARD_SHELL_OBJDIR=$WORKSPACE/obj-haz-shell JS_SRCDIR=$GECKO_DIR/js/src ANALYSIS_SRCDIR=$JS_SRCDIR/devtools/rootAnalysis @@ -8,6 +12,11 @@ ANALYSIS_SRCDIR=$JS_SRCDIR/devtools/rootAnalysis TOOLTOOL_MANIFEST=js/src/devtools/rootAnalysis/build/sixgill.manifest . install-packages.sh "$GECKO_DIR" +PYTHON=python2.7 +if ! which $PYTHON; then + PYTHON=python +fi + export CC="$GECKO_DIR/gcc/bin/gcc" export CXX="$GECKO_DIR/gcc/bin/g++" @@ -45,7 +54,7 @@ function run_analysis () { ( cd "$analysis_dir" - python "$ANALYSIS_SRCDIR/analyze.py" --buildcommand="$GECKO_DIR/testing/mozharness/scripts/spidermonkey/build.${build_type}" + $PYTHON "$ANALYSIS_SRCDIR/analyze.py" --buildcommand="$GECKO_DIR/testing/mozharness/scripts/spidermonkey/build.${build_type}" ) } @@ -59,13 +68,16 @@ function grab_artifacts () { cd "$analysis_dir" ls -lah + # Do not error out if no files found + shopt -s nullglob + set +e for f in *.txt *.lst; do gzip -9 -c "$f" > "${artifacts}/$f.gz" done # Check whether the user requested .xdb file upload in the top commit comment - if hg --cwd "$GECKO_DIR" log -l1 --template '{desc}\n' | grep -q 'haz: --upload-xdbs'; then + if hg --cwd "$GECKO_DIR" log -l1 --template '{desc}\n' | grep -q -- '--upload-xdbs'; then for f in *.xdb; do bzip2 -c "$f" > "${artifacts}/$f.bz2" done @@ -74,8 +86,19 @@ function grab_artifacts () { } function check_hazards () { - if grep 'Function.*has unrooted.*live across GC call' "$1"/rootingHazards.txt; then - echo "TEST-UNEXPECTED-FAIL hazards detected" >&2 + ( + set +e + NUM_HAZARDS=$(grep -c 'Function.*has unrooted.*live across GC call' "$1"/rootingHazards.txt) + NUM_UNSAFE=$(grep -c '^Function.*takes unsafe address of unrooted' "$1"/refs.txt) + NUM_UNNECESSARY=$(grep -c '^Function.* has unnecessary root' "$1"/unnecessary.txt) + + echo "TinderboxPrint: $NUM_HAZARDS rooting hazards" + echo "TinderboxPrint: $NUM_UNSAFE unsafe references to unrooted GC pointers" + echo "TinderboxPrint: $NUM_UNSAFE unnecessary roots" + + if [ $NUM_HAZARDS -gt 0 ]; then + echo "TEST-UNEXPECTED-FAIL $NUM_HAZARDS hazards detected" >&2 exit 1 fi + ) } diff --git a/testing/taskcluster/tasks/branches/try/job_flags.yml b/testing/taskcluster/tasks/branches/try/job_flags.yml index 337558f58503..1ff5da773986 100644 --- a/testing/taskcluster/tasks/branches/try/job_flags.yml +++ b/testing/taskcluster/tasks/branches/try/job_flags.yml @@ -75,6 +75,12 @@ builds: types: opt: task: tasks/builds/opt_linux64_st-an_clobber.yml + linux64-haz: + platforms: + - Linux64 + types: + debug: + task: tasks/builds/haz_linux.yml macosx64: platforms: - MacOSX64 diff --git a/testing/taskcluster/tasks/builds/haz_linux.yml b/testing/taskcluster/tasks/builds/haz_linux.yml new file mode 100644 index 000000000000..004a871f11a3 --- /dev/null +++ b/testing/taskcluster/tasks/builds/haz_linux.yml @@ -0,0 +1,59 @@ +$inherits: + from: 'tasks/builds/firefox_base.yml' + variables: + build_name: 'browser-haz' + build_type: 'debug' +task: + metadata: + name: '[TC] Browser Hazard Linux' + description: Browser Hazard Analysis Linux + + workerType: dbg-linux64 + + scopes: + - 'docker-worker:cache:tooltool-cache' + - 'docker-worker:relengapi-proxy:tooltool.download.public' + - 'docker-worker:cache:level-{{level}}-{{project}}-build-linux64-haz-workspace' + + payload: + cache: + level-{{level}}-{{project}}-build-linux64-haz-workspace: '/home/worker/workspace' + tooltool-cache: '/home/worker/tools/tooltool-cache' + + features: + relengAPIProxy: true + + env: + MOZCONFIG: 'browser/config/mozconfigs/linux64/hazards' + TOOLTOOL_CACHE: '/home/worker/tools/tooltool-cache' + TOOLTOOL_MANIFEST: 'browser/config/tooltool-manifests/linux64/releng.manifest' + + maxRunTime: 36000 + + command: + - /bin/bash + - -c + - > + tc-vcs checkout workspace/gecko "$GECKO_BASE_REPOSITORY" "$GECKO_HEAD_REPOSITORY" "$GECKO_HEAD_REV" "$GECKO_HEAD_REF" && + cd ./workspace/gecko/testing/taskcluster/scripts/builder && + ./build-browser-haz-linux.sh $HOME/workspace + + extra: + treeherderEnv: + - production + - staging + treeherder: + machine: + # see https://github.com/mozilla/treeherder/blob/master/ui/js/values.js + platform: linux64 + groupSymbol: tc + groupName: Submitted by taskcluster for your pleasure + symbol: H + collection: + debug: true + # Rather then enforcing particular conventions we require that all build + # tasks provide the "build" extra field to specify where the build and tests + # files are located. + locations: + build: null + tests: null