Bug 1282226 - Kill secrets from phone-builder image and fix aries/hammerhead r=wcosta

MozReview-Commit-ID: LOZHgih2TrM --HG-- extra : rebase_source : e17d08683891ca5c74bb8b3930d0aa5e62121c4f
2016-06-28 19:46:40 +02:00 · 2016-06-28 19:46:40 +02:00 · 2eb2535e97
--- a/b2g/config/aries/config.json
+++ b/b2g/config/aries/config.json
@ -45,7 +45,7 @@
    "gecko_l10n_root": "https://hg.mozilla.org/l10n-central",
    "gaia": {
        "l10n": {
-            "vcs": "hgtool",
+            "vcs": "hg",
            "root": "https://hg.mozilla.org/gaia-l10n"
        }
    }
--- a/b2g/config/aries/releng-aries.manifest
+++ b/b2g/config/aries/releng-aries.manifest
@ -1,5 +1,22 @@
 [
 {
+"version": "Android NDK r11b for B2G",
+"algorithm": "sha512",
+"visibility": "internal",
+"filename": "android-ndk-b2g.tar.xz",
+"unpack": true,
+"digest": "bc37c6b2e38f4ff19e3326786312d8f893600e155d35dfba45163bd909e022db852b9c6920863cb498bbe7da8b86a6a387fa024bc9444ce3a8d1715cf2c24b21",
+"size": 292442020
+},
+{
+"algorithm": "sha512",
+"visibility": "internal",
+"filename": "backup-aries_23.0.1.A.5.77.tar.xz",
+"unpack": true,
+"digest": "79c8e390e88cc4765ff7f5f29f3d5337c9037b7eb9414006947d38d34acefdbcf7090c18a366948c682b1c2c9d9ef51012e7be44daa28fdde7b837ade647c257",
+"size": 227555180
+},
+{
 "version": "gcc 4.8.5 + PR64905",
 "size": 80160264,
 "digest": "c1a9dc9da289b8528874d16300b9d13a997cec99195bb0bc46ff665216d8535d6d6cb5af6b4b1f2749af6815dab12e703fdb3849014e5c23a70eff351a0baf4e",
--- a/b2g/config/aries/sources.xml
+++ b/b2g/config/aries/sources.xml
@ -6,23 +6,21 @@
  <remote fetch="https://git.mozilla.org/external/aosp" name="aosp"/>
  <!--original fetch url was git://github.com/apitrace/-->
  <remote fetch="https://git.mozilla.org/external/apitrace" name="apitrace"/>
-  <!--original fetch url was git://github.com/mozilla-b2g/-->
-  <remote fetch="https://git.mozilla.org/b2g" name="b2g"/>
+  <remote fetch="git://github.com/mozilla-b2g/" name="b2g"/>
  <!--original fetch url was https://git.mozilla.org/b2g-->
  <remote fetch="https://git.mozilla.org/b2g" name="b2gmozilla"/>
  <!--original fetch url was git://codeaurora.org/-->
  <remote fetch="https://git.mozilla.org/external/caf" name="caf"/>
  <!--original fetch url was http://android.git.linaro.org/git-ro/-->
  <remote fetch="https://git.mozilla.org/external/linaro" name="linaro"/>
-  <!--original fetch url was git://github.com/mozilla/-->
-  <remote fetch="https://git.mozilla.org/b2g" name="mozilla"/>
+  <remote fetch="git://github.com/mozilla/" name="mozilla"/>
  <!--original fetch url was https://git.mozilla.org/releases-->
  <remote fetch="https://git.mozilla.org/releases" name="mozillaorg"/>
  <!--
    B2G repositories for all targets
    -->
-  <project name="gaia" path="gaia" remote="mozillaorg" revision="a954bd2954c422b7d24d92cfd73000cb455dce44"/>
-  <project name="gonk-misc" path="gonk-misc" remote="b2g" revision="19150d320e6802ec211ccc5e74c254ae9992312d"/>
+  <project name="gaia" path="gaia" remote="b2g" revision="99c01f5646b2d8aa5ebf1968114ab2f5db5ac6a8"/>
+  <project name="gonk-misc" path="gonk-misc" remote="b2g" revision="31f2fbb02035c18b84f3387317aab75adf65da87"/>
  <project name="moztt" path="external/moztt" remote="b2g" revision="99c333dab00ed79baff9e1cf76b320aee8e1c123"/>
  <project name="platform_hardware_libhardware_moz" path="hardware/libhardware_moz" remote="b2g" revision="fdf3a143dc777e5f9d33a88373af7ea161d3b440"/>
  <project name="platform_system_libfdio" path="system/libfdio" remote="b2g" revision="34adfb400e031f3dd3353d92413572db5e3a7376"/>
--- a/b2g/config/nexus-5-l/config.json
+++ b/b2g/config/nexus-5-l/config.json
@ -48,7 +48,7 @@
    "gecko_l10n_root": "https://hg.mozilla.org/l10n-central",
    "gaia": {
        "l10n": {
-            "vcs": "hgtool",
+            "vcs": "hg",
            "root": "https://hg.mozilla.org/gaia-l10n"
        }
    }
--- a/b2g/config/nexus-5-l/releng-nexus5.manifest
+++ b/b2g/config/nexus-5-l/releng-nexus5.manifest
@ -1,8 +1,19 @@
-[{
+[
+{
+"version": "Android NDK r11b for B2G",
+"algorithm": "sha512",
+"visibility": "internal",
+"filename": "android-ndk-b2g.tar.xz",
+"unpack": true,
+"digest": "bc37c6b2e38f4ff19e3326786312d8f893600e155d35dfba45163bd909e022db852b9c6920863cb498bbe7da8b86a6a387fa024bc9444ce3a8d1715cf2c24b21",
+"size": 292442020
+},
+{
 "version": "gcc 4.8.5 + PR64905",
 "size": 80160264,
 "digest": "c1a9dc9da289b8528874d16300b9d13a997cec99195bb0bc46ff665216d8535d6d6cb5af6b4b1f2749af6815dab12e703fdb3849014e5c23a70eff351a0baf4e",
 "algorithm": "sha512",
 "filename": "gcc.tar.xz",
 "unpack": "True"
-}]
+}
+]
--- a/b2g/config/nexus-5-l/sources.xml
+++ b/b2g/config/nexus-5-l/sources.xml
@ -6,23 +6,21 @@
  <remote fetch="https://git.mozilla.org/external/aosp" name="aosp"/>
  <!--original fetch url was git://github.com/apitrace/-->
  <remote fetch="https://git.mozilla.org/external/apitrace" name="apitrace"/>
-  <!--original fetch url was git://github.com/mozilla-b2g/-->
-  <remote fetch="https://git.mozilla.org/b2g" name="b2g"/>
+  <remote fetch="git://github.com/mozilla-b2g/" name="b2g"/>
  <!--original fetch url was https://git.mozilla.org/b2g-->
  <remote fetch="https://git.mozilla.org/b2g" name="b2gmozilla"/>
  <!--original fetch url was git://codeaurora.org/-->
  <remote fetch="https://git.mozilla.org/external/caf" name="caf"/>
  <!--original fetch url was http://android.git.linaro.org/git-ro/-->
  <remote fetch="https://git.mozilla.org/external/linaro" name="linaro"/>
-  <!--original fetch url was git://github.com/mozilla/-->
-  <remote fetch="https://git.mozilla.org/b2g" name="mozilla"/>
+  <remote fetch="git://github.com/mozilla/" name="mozilla"/>
  <!--original fetch url was https://git.mozilla.org/releases-->
  <remote fetch="https://git.mozilla.org/releases" name="mozillaorg"/>
  <!--
    B2G repositories for all targets
    -->
-  <project name="gaia" path="gaia" remote="mozillaorg" revision="a954bd2954c422b7d24d92cfd73000cb455dce44"/>
-  <project name="gonk-misc" path="gonk-misc" remote="b2g" revision="19150d320e6802ec211ccc5e74c254ae9992312d"/>
+  <project name="gaia" path="gaia" remote="b2g" revision="99c01f5646b2d8aa5ebf1968114ab2f5db5ac6a8"/>
+  <project name="gonk-misc" path="gonk-misc" remote="b2g" revision="31f2fbb02035c18b84f3387317aab75adf65da87"/>
  <project name="moztt" path="external/moztt" remote="b2g" revision="99c333dab00ed79baff9e1cf76b320aee8e1c123"/>
  <project name="platform_hardware_libhardware_moz" path="hardware/libhardware_moz" remote="b2g" revision="fdf3a143dc777e5f9d33a88373af7ea161d3b440"/>
  <project name="platform_system_libfdio" path="system/libfdio" remote="b2g" revision="34adfb400e031f3dd3353d92413572db5e3a7376"/>
--- a/taskcluster/ci/docker-image/kind.yml
+++ b/taskcluster/ci/docker-image/kind.yml
@ -17,3 +17,4 @@ images:
  - tester
  - lint
  - android-gradle-build
+  - phone-builder
--- a/taskcluster/ci/legacy/tasks/builds/b2g_aries_eng.yml
+++ b/taskcluster/ci/legacy/tasks/builds/b2g_aries_eng.yml
@ -7,6 +7,8 @@ task:
  scopes:
    - 'docker-worker:cache:level-{{level}}-{{project}}-build-aries-eng'
    - 'docker-worker:cache:level-{{level}}-{{project}}-build-aries-eng-objdir-gecko'
+    - 'docker-worker:relengapi-proxy:tooltool.download.internal'
+    - 'docker-worker:relengapi-proxy:tooltool.download.public'
  metadata:
    name: '[TC] B2G Aries Eng'

@ -14,9 +16,10 @@ task:
    cache:
      level-{{level}}-{{project}}-build-aries-eng: /home/worker/workspace
      level-{{level}}-{{project}}-build-aries-eng-objdir-gecko: /home/worker/objdir-gecko
+    features:
+      relengAPIProxy: true
    env:
      TARGET: 'aries'
-      MOZHARNESS_CONFIG: b2g/taskcluster-phone-eng.py
  extra:
    treeherderEnv:
      - production
--- a/taskcluster/ci/legacy/tasks/builds/b2g_nexus_5l_eng.yml
+++ b/taskcluster/ci/legacy/tasks/builds/b2g_nexus_5l_eng.yml
@ -8,20 +8,18 @@ task:
  scopes:
    - 'docker-worker:cache:level-{{level}}-{{project}}-build-nexus-5-l-eng'
    - 'docker-worker:cache:level-{{level}}-{{project}}-build-nexus-5-l-eng-objdir-gecko'
+    - 'docker-worker:relengapi-proxy:tooltool.download.internal'
+    - 'docker-worker:relengapi-proxy:tooltool.download.public'
  metadata:
    name: '[TC] B2G Nexus 5-L Eng'
  payload:
    cache:
      level-{{level}}-{{project}}-build-nexus-5-l-eng: /home/worker/object-folder
      level-{{level}}-{{project}}-build-nexus-5-l-eng-objdir-gecko: /home/worker/objdir-gecko
+    features:
+      relengAPIProxy: true
    env:
      TARGET: 'nexus-5-l'
-      DEBUG: 0
-    command:
-      - >
-        checkout-gecko workspace &&
-        cd ./workspace/gecko/taskcluster/scripts/phone-builder &&
-        buildbot_step 'Build' ./build-phone.sh $HOME/workspace
  extra:
    treeherderEnv:
      - production
--- a/taskcluster/ci/legacy/tasks/builds/b2g_phone_eng_base.yml
+++ b/taskcluster/ci/legacy/tasks/builds/b2g_phone_eng_base.yml
@ -13,6 +13,7 @@ task:
      MOZ_CRASHREPORTER_NO_REPORT: '1'
      GAIA_KEYBOARD_LAYOUTS: 'en,pt-BR,es,de,fr,pl,zh-Hans-Pinyin,zh-Hant-Zhuyin,en-Dvorak'
      B2G_UPDATE_CHANNEL: 'default'
+      MOZHARNESS_CONFIG: b2g/taskcluster-phone-eng.py
    command:
      - >
        checkout-gecko workspace &&
--- a/taskcluster/ci/legacy/tasks/phone_build.yml
+++ b/taskcluster/ci/legacy/tasks/phone_build.yml
@ -2,7 +2,7 @@
 # provide.
 ---
 taskId: {{build_slugid}}
-
+docker-image: phone-builder
 task:
  created: '{{now}}'
  deadline: '{{#from_now}}24 hours{{/from_now}}'
@ -25,10 +25,13 @@ task:
    # Nearly all of our build tasks use tc-vcs so just include the scope across
    # the board.
    - 'docker-worker:cache:level-{{level}}-{{project}}-tc-vcs'
-    - 'docker-worker:image:{{#docker_image}}phone-builder{{/docker_image}}'

  payload:
-    image: '{{#docker_image}}phone-builder{{/docker_image}}'
+    image:
+      type: 'task-image'
+      path: 'public/image.tar'
+      taskId:
+        task-reference: "<docker-image>"

    # One hour is a good value for most builds
    maxRunTime: 3600
--- a/taskcluster/scripts/phone-builder/pre-build.sh
+++ b/taskcluster/scripts/phone-builder/pre-build.sh
@ -31,16 +31,3 @@ if [ 0$B2G_DEBUG -ne 0 ]; then
  debug_flag='--debug'
 fi

-if ! aws --output=text s3 ls s3://b2g-phone-backups/; then
-    echo "[aws:error] Failed to connect to AWS! Are the AWS credentials ok?"
-    exit 1
-fi
-
-backup_file=$(aws --output=text s3 ls s3://b2g-phone-backups/$TARGET/ | tail -1 | awk '{print $NF}')
-
-if echo $backup_file | grep '\.tar\.bz2'; then
-    aws s3 cp s3://b2g-phone-backups/$TARGET/$backup_file .
-    tar -xjf $backup_file -C $WORKSPACE/B2G
-    rm -f $backup_file
-fi
-
--- a/testing/docker/phone-builder/Dockerfile
+++ b/testing/docker/phone-builder/Dockerfile
@ -1,17 +1,14 @@
 FROM          taskcluster/builder:0.5.13
 MAINTAINER    Wander Lairson Costa <wcosta@mozilla.com>

-ENV           SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE /home/worker/socorro.token
-
 # Add utilities and configuration
 ADD           bin                   /home/worker/bin
-ADD           config                /home/worker/.aws/config
-ADD           socorro.token         /home/worker/socorro.token
+
+# Builds need the share module enabled
+ADD           hgrc /home/worker/.hgrc
+RUN chown -R worker:worker /home/worker/.hgrc

 RUN           yum install -y bc lzop java-1.7.0-openjdk
 RUN           pip install awscli
 RUN           npm install -g bower gulp apm grunt-cli

-# Set a default command useful for debugging
-ENTRYPOINT ["validate_task.py"]
-
--- a/testing/docker/phone-builder/REGISTRY
+++ b/testing/docker/phone-builder/REGISTRY
@ -1 +0,0 @@
-taskclusterprivate
--- a/testing/docker/phone-builder/VERSION
+++ b/testing/docker/phone-builder/VERSION
@ -1 +0,0 @@
-0.0.26
--- a/testing/docker/phone-builder/build.sh
+++ b/testing/docker/phone-builder/build.sh
@ -1,37 +0,0 @@
-#! /bin/bash -e
-
-while getopts "t:i:k:s:" arg; do
-  case $arg in
-    t)
-      TAG=$OPTARG
-      ;;
-    i)
-      AWS_ACCESS_KEY_ID=$OPTARG
-      ;;
-    k)
-      AWS_SECRET_ACCESS_KEY=$OPTARG
-      ;;
-    s)
-      SOCORRO_TOKEN=$OPTARG
-      ;;
-  esac
-done
-
-pushd $(dirname $0)
-
-test $TAG
-test $AWS_ACCESS_KEY_ID
-test $AWS_SECRET_ACCESS_KEY
-test $SOCORRO_TOKEN
-
-(echo '[default]'
-echo "aws_access_key_id = $AWS_ACCESS_KEY_ID"
-echo "aws_secret_access_key = $AWS_SECRET_ACCESS_KEY") > config
-
-echo $SOCORRO_TOKEN > socorro.token
-
-docker build -t $TAG .
-rm -f config
-rm -f socorro.token
-
-popd
--- a/testing/docker/phone-builder/hgrc
+++ b/testing/docker/phone-builder/hgrc
@ -0,0 +1,4 @@
+[extensions]
+share =
+[ui]
+username = TaskCluster <nobody@mozilla.org>
--- a/testing/mozharness/configs/b2g/taskcluster-phone-eng.py
+++ b/testing/mozharness/configs/b2g/taskcluster-phone-eng.py
@ -1,11 +1,15 @@
 #!/usr/bin/env python
+
+HG_SHARE_BASE_DIR = "/builds/hg-shared"
+
 config = {
+    # mozharness script options
+    "vcs_share_base": HG_SHARE_BASE_DIR,
    "default_vcs": "tc-vcs",
    "default_actions": [
        'checkout-sources',
        'get-blobs',
        'build',
-        'build-symbols',
        'prep-upload'
    ],
    "upload": {
@ -19,6 +23,9 @@ config = {
        "B2G_PATH": "%(work_dir)s",
        "BOWER_FLAGS": "--allow-root",
        "WGET_OPTS": "-c -q",
+        "HG_SHARE_BASE_DIR": HG_SHARE_BASE_DIR,
+        "B2G_ANDROID_NDK_PATH": "%(b2g_repo)s/android-ndk",
+        "ANDROIDFS_DIR": "%(b2g_repo)s/backup-%(b2g_target)s",
    },
    "is_automation": True,
    "repo_remote_mappings": {
@ -32,4 +39,5 @@ config = {
        'git://github.com/apitrace/': 'https://git.mozilla.org/external/apitrace',
    },
    "download_tooltool": True,
+    "tooltool_servers": ['http://relengapi/tooltool/'],
 }
--- a/testing/mozharness/mozharness/mozilla/building/buildb2gbase.py
+++ b/testing/mozharness/mozharness/mozilla/building/buildb2gbase.py
@ -161,6 +161,8 @@ class B2GBuildBaseScript(BuildbotMixin, MockMixin,
            'work_dir': abs_dirs['abs_work_dir'],
            'b2g_src': abs_dirs['abs_work_dir'],
            'abs_tools_dir': os.path.join(abs_dirs['abs_work_dir'], 'build-tools'),
+            'b2g_repo': self.config['repo'],
+            'b2g_target': self.config['target'],
        }

        abs_dirs.update(dirs)
--- a/testing/mozharness/scripts/b2g_build.py
+++ b/testing/mozharness/scripts/b2g_build.py
@ -438,7 +438,7 @@ class B2GBuild(LocalesMixin, PurgeMixin,
        repo = self.config['compare_locales_repo']
        rev = self.config['compare_locales_rev']
        vcs = self.config['compare_locales_vcs']
-        abs_rev = self.vcs_checkout(repo=repo, dest=dest, revision=rev, vcs=vcs)
+        abs_rev = self.vcs_checkout(repo=repo, dest=dest, branch=rev, vcs=vcs)
        self.set_buildbot_property('compare_locales_revision', abs_rev, write_to_file=True)

    def query_do_translate_hg_to_git(self, gecko_config_key=None):