From 2eb2535e9758eafac33309f9916c085708fdd37a Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Tue, 28 Jun 2016 19:46:40 +0200 Subject: [PATCH] Bug 1282226 - Kill secrets from phone-builder image and fix aries/hammerhead r=wcosta MozReview-Commit-ID: LOZHgih2TrM --HG-- extra : rebase_source : e17d08683891ca5c74bb8b3930d0aa5e62121c4f --- b2g/config/aries/config.json | 2 +- b2g/config/aries/releng-aries.manifest | 17 +++++++++ b2g/config/aries/sources.xml | 10 ++--- b2g/config/nexus-5-l/config.json | 2 +- b2g/config/nexus-5-l/releng-nexus5.manifest | 15 +++++++- b2g/config/nexus-5-l/sources.xml | 10 ++--- taskcluster/ci/docker-image/kind.yml | 1 + .../ci/legacy/tasks/builds/b2g_aries_eng.yml | 5 ++- .../legacy/tasks/builds/b2g_nexus_5l_eng.yml | 10 ++--- .../tasks/builds/b2g_phone_eng_base.yml | 1 + taskcluster/ci/legacy/tasks/phone_build.yml | 9 +++-- .../scripts/phone-builder/pre-build.sh | 13 ------- testing/docker/phone-builder/Dockerfile | 11 ++---- testing/docker/phone-builder/REGISTRY | 1 - testing/docker/phone-builder/VERSION | 1 - testing/docker/phone-builder/build.sh | 37 ------------------- testing/docker/phone-builder/hgrc | 4 ++ .../configs/b2g/taskcluster-phone-eng.py | 10 ++++- .../mozilla/building/buildb2gbase.py | 2 + testing/mozharness/scripts/b2g_build.py | 2 +- 20 files changed, 76 insertions(+), 87 deletions(-) delete mode 100644 testing/docker/phone-builder/REGISTRY delete mode 100644 testing/docker/phone-builder/VERSION delete mode 100755 testing/docker/phone-builder/build.sh create mode 100644 testing/docker/phone-builder/hgrc diff --git a/b2g/config/aries/config.json b/b2g/config/aries/config.json index a963a62a97dd..1902a3000145 100644 --- a/b2g/config/aries/config.json +++ b/b2g/config/aries/config.json @@ -45,7 +45,7 @@ "gecko_l10n_root": "https://hg.mozilla.org/l10n-central", "gaia": { "l10n": { - "vcs": "hgtool", + "vcs": "hg", "root": "https://hg.mozilla.org/gaia-l10n" } } diff --git a/b2g/config/aries/releng-aries.manifest b/b2g/config/aries/releng-aries.manifest index 7179957e1181..a1276ffe34a6 100644 --- a/b2g/config/aries/releng-aries.manifest +++ b/b2g/config/aries/releng-aries.manifest @@ -1,5 +1,22 @@ [ { +"version": "Android NDK r11b for B2G", +"algorithm": "sha512", +"visibility": "internal", +"filename": "android-ndk-b2g.tar.xz", +"unpack": true, +"digest": "bc37c6b2e38f4ff19e3326786312d8f893600e155d35dfba45163bd909e022db852b9c6920863cb498bbe7da8b86a6a387fa024bc9444ce3a8d1715cf2c24b21", +"size": 292442020 +}, +{ +"algorithm": "sha512", +"visibility": "internal", +"filename": "backup-aries_23.0.1.A.5.77.tar.xz", +"unpack": true, +"digest": "79c8e390e88cc4765ff7f5f29f3d5337c9037b7eb9414006947d38d34acefdbcf7090c18a366948c682b1c2c9d9ef51012e7be44daa28fdde7b837ade647c257", +"size": 227555180 +}, +{ "version": "gcc 4.8.5 + PR64905", "size": 80160264, "digest": "c1a9dc9da289b8528874d16300b9d13a997cec99195bb0bc46ff665216d8535d6d6cb5af6b4b1f2749af6815dab12e703fdb3849014e5c23a70eff351a0baf4e", diff --git a/b2g/config/aries/sources.xml b/b2g/config/aries/sources.xml index e0f83fc2406d..c003106bc369 100644 --- a/b2g/config/aries/sources.xml +++ b/b2g/config/aries/sources.xml @@ -6,23 +6,21 @@ - - + - - + - - + + diff --git a/b2g/config/nexus-5-l/config.json b/b2g/config/nexus-5-l/config.json index 21fc9361f67a..7b8bada40221 100644 --- a/b2g/config/nexus-5-l/config.json +++ b/b2g/config/nexus-5-l/config.json @@ -48,7 +48,7 @@ "gecko_l10n_root": "https://hg.mozilla.org/l10n-central", "gaia": { "l10n": { - "vcs": "hgtool", + "vcs": "hg", "root": "https://hg.mozilla.org/gaia-l10n" } } diff --git a/b2g/config/nexus-5-l/releng-nexus5.manifest b/b2g/config/nexus-5-l/releng-nexus5.manifest index 40720bbe9227..f7851b4e3724 100644 --- a/b2g/config/nexus-5-l/releng-nexus5.manifest +++ b/b2g/config/nexus-5-l/releng-nexus5.manifest @@ -1,8 +1,19 @@ -[{ +[ +{ +"version": "Android NDK r11b for B2G", +"algorithm": "sha512", +"visibility": "internal", +"filename": "android-ndk-b2g.tar.xz", +"unpack": true, +"digest": "bc37c6b2e38f4ff19e3326786312d8f893600e155d35dfba45163bd909e022db852b9c6920863cb498bbe7da8b86a6a387fa024bc9444ce3a8d1715cf2c24b21", +"size": 292442020 +}, +{ "version": "gcc 4.8.5 + PR64905", "size": 80160264, "digest": "c1a9dc9da289b8528874d16300b9d13a997cec99195bb0bc46ff665216d8535d6d6cb5af6b4b1f2749af6815dab12e703fdb3849014e5c23a70eff351a0baf4e", "algorithm": "sha512", "filename": "gcc.tar.xz", "unpack": "True" -}] +} +] diff --git a/b2g/config/nexus-5-l/sources.xml b/b2g/config/nexus-5-l/sources.xml index c381b7cfb235..03a11da9df38 100644 --- a/b2g/config/nexus-5-l/sources.xml +++ b/b2g/config/nexus-5-l/sources.xml @@ -6,23 +6,21 @@ - - + - - + - - + + diff --git a/taskcluster/ci/docker-image/kind.yml b/taskcluster/ci/docker-image/kind.yml index 2c05054bb3fb..7ca6ab5a1ba7 100644 --- a/taskcluster/ci/docker-image/kind.yml +++ b/taskcluster/ci/docker-image/kind.yml @@ -17,3 +17,4 @@ images: - tester - lint - android-gradle-build + - phone-builder diff --git a/taskcluster/ci/legacy/tasks/builds/b2g_aries_eng.yml b/taskcluster/ci/legacy/tasks/builds/b2g_aries_eng.yml index 05a6ec28f94d..c1b66b5dd0bc 100644 --- a/taskcluster/ci/legacy/tasks/builds/b2g_aries_eng.yml +++ b/taskcluster/ci/legacy/tasks/builds/b2g_aries_eng.yml @@ -7,6 +7,8 @@ task: scopes: - 'docker-worker:cache:level-{{level}}-{{project}}-build-aries-eng' - 'docker-worker:cache:level-{{level}}-{{project}}-build-aries-eng-objdir-gecko' + - 'docker-worker:relengapi-proxy:tooltool.download.internal' + - 'docker-worker:relengapi-proxy:tooltool.download.public' metadata: name: '[TC] B2G Aries Eng' @@ -14,9 +16,10 @@ task: cache: level-{{level}}-{{project}}-build-aries-eng: /home/worker/workspace level-{{level}}-{{project}}-build-aries-eng-objdir-gecko: /home/worker/objdir-gecko + features: + relengAPIProxy: true env: TARGET: 'aries' - MOZHARNESS_CONFIG: b2g/taskcluster-phone-eng.py extra: treeherderEnv: - production diff --git a/taskcluster/ci/legacy/tasks/builds/b2g_nexus_5l_eng.yml b/taskcluster/ci/legacy/tasks/builds/b2g_nexus_5l_eng.yml index cf5e4e2d24d5..2d5b50cc3208 100644 --- a/taskcluster/ci/legacy/tasks/builds/b2g_nexus_5l_eng.yml +++ b/taskcluster/ci/legacy/tasks/builds/b2g_nexus_5l_eng.yml @@ -8,20 +8,18 @@ task: scopes: - 'docker-worker:cache:level-{{level}}-{{project}}-build-nexus-5-l-eng' - 'docker-worker:cache:level-{{level}}-{{project}}-build-nexus-5-l-eng-objdir-gecko' + - 'docker-worker:relengapi-proxy:tooltool.download.internal' + - 'docker-worker:relengapi-proxy:tooltool.download.public' metadata: name: '[TC] B2G Nexus 5-L Eng' payload: cache: level-{{level}}-{{project}}-build-nexus-5-l-eng: /home/worker/object-folder level-{{level}}-{{project}}-build-nexus-5-l-eng-objdir-gecko: /home/worker/objdir-gecko + features: + relengAPIProxy: true env: TARGET: 'nexus-5-l' - DEBUG: 0 - command: - - > - checkout-gecko workspace && - cd ./workspace/gecko/taskcluster/scripts/phone-builder && - buildbot_step 'Build' ./build-phone.sh $HOME/workspace extra: treeherderEnv: - production diff --git a/taskcluster/ci/legacy/tasks/builds/b2g_phone_eng_base.yml b/taskcluster/ci/legacy/tasks/builds/b2g_phone_eng_base.yml index 310a14f8019d..d9a2f5f6aaa1 100644 --- a/taskcluster/ci/legacy/tasks/builds/b2g_phone_eng_base.yml +++ b/taskcluster/ci/legacy/tasks/builds/b2g_phone_eng_base.yml @@ -13,6 +13,7 @@ task: MOZ_CRASHREPORTER_NO_REPORT: '1' GAIA_KEYBOARD_LAYOUTS: 'en,pt-BR,es,de,fr,pl,zh-Hans-Pinyin,zh-Hant-Zhuyin,en-Dvorak' B2G_UPDATE_CHANNEL: 'default' + MOZHARNESS_CONFIG: b2g/taskcluster-phone-eng.py command: - > checkout-gecko workspace && diff --git a/taskcluster/ci/legacy/tasks/phone_build.yml b/taskcluster/ci/legacy/tasks/phone_build.yml index 1f6cb95e325c..0d0c83cb1b63 100644 --- a/taskcluster/ci/legacy/tasks/phone_build.yml +++ b/taskcluster/ci/legacy/tasks/phone_build.yml @@ -2,7 +2,7 @@ # provide. --- taskId: {{build_slugid}} - +docker-image: phone-builder task: created: '{{now}}' deadline: '{{#from_now}}24 hours{{/from_now}}' @@ -25,10 +25,13 @@ task: # Nearly all of our build tasks use tc-vcs so just include the scope across # the board. - 'docker-worker:cache:level-{{level}}-{{project}}-tc-vcs' - - 'docker-worker:image:{{#docker_image}}phone-builder{{/docker_image}}' payload: - image: '{{#docker_image}}phone-builder{{/docker_image}}' + image: + type: 'task-image' + path: 'public/image.tar' + taskId: + task-reference: "" # One hour is a good value for most builds maxRunTime: 3600 diff --git a/taskcluster/scripts/phone-builder/pre-build.sh b/taskcluster/scripts/phone-builder/pre-build.sh index 000e7ef444b6..fea9eae2a859 100755 --- a/taskcluster/scripts/phone-builder/pre-build.sh +++ b/taskcluster/scripts/phone-builder/pre-build.sh @@ -31,16 +31,3 @@ if [ 0$B2G_DEBUG -ne 0 ]; then debug_flag='--debug' fi -if ! aws --output=text s3 ls s3://b2g-phone-backups/; then - echo "[aws:error] Failed to connect to AWS! Are the AWS credentials ok?" - exit 1 -fi - -backup_file=$(aws --output=text s3 ls s3://b2g-phone-backups/$TARGET/ | tail -1 | awk '{print $NF}') - -if echo $backup_file | grep '\.tar\.bz2'; then - aws s3 cp s3://b2g-phone-backups/$TARGET/$backup_file . - tar -xjf $backup_file -C $WORKSPACE/B2G - rm -f $backup_file -fi - diff --git a/testing/docker/phone-builder/Dockerfile b/testing/docker/phone-builder/Dockerfile index 82f3bfc10e63..c45c3dbcc049 100644 --- a/testing/docker/phone-builder/Dockerfile +++ b/testing/docker/phone-builder/Dockerfile @@ -1,17 +1,14 @@ FROM taskcluster/builder:0.5.13 MAINTAINER Wander Lairson Costa -ENV SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE /home/worker/socorro.token - # Add utilities and configuration ADD bin /home/worker/bin -ADD config /home/worker/.aws/config -ADD socorro.token /home/worker/socorro.token + +# Builds need the share module enabled +ADD hgrc /home/worker/.hgrc +RUN chown -R worker:worker /home/worker/.hgrc RUN yum install -y bc lzop java-1.7.0-openjdk RUN pip install awscli RUN npm install -g bower gulp apm grunt-cli -# Set a default command useful for debugging -ENTRYPOINT ["validate_task.py"] - diff --git a/testing/docker/phone-builder/REGISTRY b/testing/docker/phone-builder/REGISTRY deleted file mode 100644 index fc410bd6571b..000000000000 --- a/testing/docker/phone-builder/REGISTRY +++ /dev/null @@ -1 +0,0 @@ -taskclusterprivate diff --git a/testing/docker/phone-builder/VERSION b/testing/docker/phone-builder/VERSION deleted file mode 100644 index c4475d3bb782..000000000000 --- a/testing/docker/phone-builder/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.0.26 diff --git a/testing/docker/phone-builder/build.sh b/testing/docker/phone-builder/build.sh deleted file mode 100755 index 8a5e598a887f..000000000000 --- a/testing/docker/phone-builder/build.sh +++ /dev/null @@ -1,37 +0,0 @@ -#! /bin/bash -e - -while getopts "t:i:k:s:" arg; do - case $arg in - t) - TAG=$OPTARG - ;; - i) - AWS_ACCESS_KEY_ID=$OPTARG - ;; - k) - AWS_SECRET_ACCESS_KEY=$OPTARG - ;; - s) - SOCORRO_TOKEN=$OPTARG - ;; - esac -done - -pushd $(dirname $0) - -test $TAG -test $AWS_ACCESS_KEY_ID -test $AWS_SECRET_ACCESS_KEY -test $SOCORRO_TOKEN - -(echo '[default]' -echo "aws_access_key_id = $AWS_ACCESS_KEY_ID" -echo "aws_secret_access_key = $AWS_SECRET_ACCESS_KEY") > config - -echo $SOCORRO_TOKEN > socorro.token - -docker build -t $TAG . -rm -f config -rm -f socorro.token - -popd diff --git a/testing/docker/phone-builder/hgrc b/testing/docker/phone-builder/hgrc new file mode 100644 index 000000000000..8ba4cf6680a2 --- /dev/null +++ b/testing/docker/phone-builder/hgrc @@ -0,0 +1,4 @@ +[extensions] +share = +[ui] +username = TaskCluster diff --git a/testing/mozharness/configs/b2g/taskcluster-phone-eng.py b/testing/mozharness/configs/b2g/taskcluster-phone-eng.py index 578e8b50e1f9..1829f358831c 100644 --- a/testing/mozharness/configs/b2g/taskcluster-phone-eng.py +++ b/testing/mozharness/configs/b2g/taskcluster-phone-eng.py @@ -1,11 +1,15 @@ #!/usr/bin/env python + +HG_SHARE_BASE_DIR = "/builds/hg-shared" + config = { + # mozharness script options + "vcs_share_base": HG_SHARE_BASE_DIR, "default_vcs": "tc-vcs", "default_actions": [ 'checkout-sources', 'get-blobs', 'build', - 'build-symbols', 'prep-upload' ], "upload": { @@ -19,6 +23,9 @@ config = { "B2G_PATH": "%(work_dir)s", "BOWER_FLAGS": "--allow-root", "WGET_OPTS": "-c -q", + "HG_SHARE_BASE_DIR": HG_SHARE_BASE_DIR, + "B2G_ANDROID_NDK_PATH": "%(b2g_repo)s/android-ndk", + "ANDROIDFS_DIR": "%(b2g_repo)s/backup-%(b2g_target)s", }, "is_automation": True, "repo_remote_mappings": { @@ -32,4 +39,5 @@ config = { 'git://github.com/apitrace/': 'https://git.mozilla.org/external/apitrace', }, "download_tooltool": True, + "tooltool_servers": ['http://relengapi/tooltool/'], } diff --git a/testing/mozharness/mozharness/mozilla/building/buildb2gbase.py b/testing/mozharness/mozharness/mozilla/building/buildb2gbase.py index c6adc57b5513..468edb5fabe3 100644 --- a/testing/mozharness/mozharness/mozilla/building/buildb2gbase.py +++ b/testing/mozharness/mozharness/mozilla/building/buildb2gbase.py @@ -161,6 +161,8 @@ class B2GBuildBaseScript(BuildbotMixin, MockMixin, 'work_dir': abs_dirs['abs_work_dir'], 'b2g_src': abs_dirs['abs_work_dir'], 'abs_tools_dir': os.path.join(abs_dirs['abs_work_dir'], 'build-tools'), + 'b2g_repo': self.config['repo'], + 'b2g_target': self.config['target'], } abs_dirs.update(dirs) diff --git a/testing/mozharness/scripts/b2g_build.py b/testing/mozharness/scripts/b2g_build.py index 01c144df400a..fcd3364d6f30 100755 --- a/testing/mozharness/scripts/b2g_build.py +++ b/testing/mozharness/scripts/b2g_build.py @@ -438,7 +438,7 @@ class B2GBuild(LocalesMixin, PurgeMixin, repo = self.config['compare_locales_repo'] rev = self.config['compare_locales_rev'] vcs = self.config['compare_locales_vcs'] - abs_rev = self.vcs_checkout(repo=repo, dest=dest, revision=rev, vcs=vcs) + abs_rev = self.vcs_checkout(repo=repo, dest=dest, branch=rev, vcs=vcs) self.set_buildbot_property('compare_locales_revision', abs_rev, write_to_file=True) def query_do_translate_hg_to_git(self, gecko_config_key=None):