From cd9b82449957d40e3f6d68302df5c9e1a72c8d31 Mon Sep 17 00:00:00 2001 From: Jordan Lund Date: Wed, 30 Sep 2015 13:56:39 -0700 Subject: [PATCH] Bug 1163084 - Releng work for producing dummy partner Android APK, r=dustin --HG-- rename : testing/mozharness/configs/builds/releng_sub_android_configs/64_api_11.py => testing/mozharness/configs/builds/releng_sub_android_configs/64_api_11_partner_sample1.py rename : testing/taskcluster/tasks/builds/android_api_11.yml => testing/taskcluster/tasks/builds/android_api_11_partner_sample1.yml --- .../distribution_sample/mozconfig1 | 21 +++++ testing/docker/desktop-build/REGISTRY | 2 +- testing/docker/desktop-build/VERSION | 2 +- .../desktop-build/bin/checkout-sources.sh | 12 +++ .../64_api_11_partner_sample1.py | 9 +++ .../mozharness/mozilla/building/buildbase.py | 42 ++++++---- .../tasks/branches/base_job_flags.yml | 1 + .../tasks/branches/try/job_flags.yml | 6 ++ .../builds/android_api_11_partner_sample1.yml | 78 +++++++++++++++++++ 9 files changed, 154 insertions(+), 19 deletions(-) create mode 100644 mobile/android/config/mozconfigs/public-partner/distribution_sample/mozconfig1 create mode 100644 testing/mozharness/configs/builds/releng_sub_android_configs/64_api_11_partner_sample1.py create mode 100644 testing/taskcluster/tasks/builds/android_api_11_partner_sample1.yml diff --git a/mobile/android/config/mozconfigs/public-partner/distribution_sample/mozconfig1 b/mobile/android/config/mozconfigs/public-partner/distribution_sample/mozconfig1 new file mode 100644 index 000000000000..e8a6ae43376a --- /dev/null +++ b/mobile/android/config/mozconfigs/public-partner/distribution_sample/mozconfig1 @@ -0,0 +1,21 @@ +# currently a copy of mobile/android/config/mozconfigs/android-api-11/nightly +. "$topsrcdir/mobile/android/config/mozconfigs/common" + +ac_add_options --enable-profiling + +# Android +ac_add_options --with-android-min-sdk=11 +ac_add_options --target=arm-linux-androideabi + +ac_add_options --with-branding=mobile/android/branding/nightly + +# This will overwrite the default of stripping everything and keep the symbol table. +# This is useful for profiling with eideticker. See bug 788680 +STRIP_FLAGS="--strip-debug" + +export MOZILLA_OFFICIAL=1 +export MOZ_TELEMETRY_REPORTING=1 + +MOZ_ANDROID_GECKOLIBS_AAR=1 + +. "$topsrcdir/mobile/android/config/mozconfigs/common.override" diff --git a/testing/docker/desktop-build/REGISTRY b/testing/docker/desktop-build/REGISTRY index cb1e1bb482a2..729d6857f4c4 100644 --- a/testing/docker/desktop-build/REGISTRY +++ b/testing/docker/desktop-build/REGISTRY @@ -1 +1 @@ -taskcluster +jlund diff --git a/testing/docker/desktop-build/VERSION b/testing/docker/desktop-build/VERSION index 9faa1b7a7339..1a030947e832 100644 --- a/testing/docker/desktop-build/VERSION +++ b/testing/docker/desktop-build/VERSION @@ -1 +1 @@ -0.1.5 +0.1.9 diff --git a/testing/docker/desktop-build/bin/checkout-sources.sh b/testing/docker/desktop-build/bin/checkout-sources.sh index fe16dbec1413..356490275fbd 100644 --- a/testing/docker/desktop-build/bin/checkout-sources.sh +++ b/testing/docker/desktop-build/bin/checkout-sources.sh @@ -40,4 +40,16 @@ then fi fi +# TODO - include tools repository in EXTRA_CHECKOUT_REPOSITORIES list +for extra_repo in $EXTRA_CHECKOUT_REPOSITORIES; do + BASE_REPO="${extra_repo}_BASE_REPOSITORY" + HEAD_REPO="${extra_repo}_HEAD_REPOSITORY" + HEAD_REV="${extra_repo}_HEAD_REV" + HEAD_REF="${extra_repo}_HEAD_REF" + DEST_DIR="${extra_repo}_DEST_DIR" + + tc-vcs checkout ${!DEST_DIR} ${!BASE_REPO} ${!HEAD_REPO} ${!HEAD_REV} ${!HEAD_REF} +done + + tc-vcs checkout $WORKSPACE/build/src $GECKO_BASE_REPOSITORY $GECKO_HEAD_REPOSITORY $GECKO_HEAD_REV $GECKO_HEAD_REF diff --git a/testing/mozharness/configs/builds/releng_sub_android_configs/64_api_11_partner_sample1.py b/testing/mozharness/configs/builds/releng_sub_android_configs/64_api_11_partner_sample1.py new file mode 100644 index 000000000000..df5b77f19e59 --- /dev/null +++ b/testing/mozharness/configs/builds/releng_sub_android_configs/64_api_11_partner_sample1.py @@ -0,0 +1,9 @@ +config = { + 'base_name': 'Android armv7 API 11+ partner Sample1 %(branch)s', + 'stage_platform': 'android-api-11-partner-sample1', + 'build_type': 'api-11-partner-sample1-opt', + 'src_mozconfig': None, # use manifest to determine mozconfig src + 'src_mozconfig_manifest': 'partner/mozconfigs/mozconfig1.json', + 'tooltool_manifest_src': 'mobile/android/config/tooltool-manifests/android/releng.manifest', + 'multi_locale_config_platform': 'android', +} diff --git a/testing/mozharness/mozharness/mozilla/building/buildbase.py b/testing/mozharness/mozharness/mozilla/building/buildbase.py index d2348e85d494..4a1fb96509ca 100755 --- a/testing/mozharness/mozharness/mozilla/building/buildbase.py +++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py @@ -57,8 +57,6 @@ Please make sure that either "repo" is in your config or, if \ you are running this in buildbot, "repo_path" is in your buildbot_config.', 'comments_undetermined': '"comments" could not be determined. This may be \ because it was a forced build.', - 'src_mozconfig_path_not_found': '"abs_src_mozconfig" path could not be \ -determined. Please make sure it is a valid path off of "abs_src_dir"', 'tooltool_manifest_undetermined': '"tooltool_manifest_src" not set, \ Skipping run_tooltool...', } @@ -325,6 +323,7 @@ class BuildOptionParser(object): 'api-9-debug': 'builds/releng_sub_%s_configs/%s_api_9_debug.py', 'api-11-debug': 'builds/releng_sub_%s_configs/%s_api_11_debug.py', 'x86': 'builds/releng_sub_%s_configs/%s_x86.py', + 'api-11-partner-sample1': 'builds/releng_sub_%s_configs/%s_api_11_partner_sample1.py', } build_pool_cfg_file = 'builds/build_pool_specifics.py' branch_cfg_file = 'builds/branch_specifics.py' @@ -1016,23 +1015,32 @@ or run without that action (ie: --no-{action})" """assign mozconfig.""" c = self.config dirs = self.query_abs_dirs() - if c.get('src_mozconfig'): + abs_mozconfig_path = '' + + # first determine the mozconfig path + if c.get('src_mozconfig') and not c.get('src_mozconfig_manifest'): self.info('Using in-tree mozconfig') - abs_src_mozconfig = os.path.join(dirs['abs_src_dir'], - c.get('src_mozconfig')) - if not os.path.exists(abs_src_mozconfig): - self.info('abs_src_mozconfig: %s' % (abs_src_mozconfig,)) - self.fatal(ERROR_MSGS['src_mozconfig_path_not_found']) - self.copyfile(abs_src_mozconfig, - os.path.join(dirs['abs_src_dir'], '.mozconfig')) - self.info("mozconfig content:") - with open(abs_src_mozconfig) as mozconfig: - for line in mozconfig: - self.info(line) + abs_mozconfig_path = os.path.join(dirs['abs_src_dir'], c.get('src_mozconfig')) + elif c.get('src_mozconfig_manifest') and not c.get('src_mozconfig'): + self.info('Using mozconfig based on manifest contents') + manifest = os.path.join(dirs['abs_work_dir'], c['src_mozconfig_manifest']) + if not os.path.exists(manifest): + self.fatal('src_mozconfig_manifest: "%s" not found. Does it exist?' % (manifest,)) + with self.opened(manifest, error_level=ERROR) as (fh, err): + if err: + self.fatal("%s exists but coud not read properties" % manifest) + abs_mozconfig_path = os.path.join(dirs['abs_src_dir'], json.load(fh)['gecko_path']) else: - self.fatal("To build, you must supply a mozconfig from inside the " - "tree to use use. Please provide the path in your " - "config via 'src_mozconfig'") + self.fatal("'src_mozconfig' or 'src_mozconfig_manifest' must be " + "in the config but not both in order to determine the mozconfig.") + + # print its contents + content = self.read_from_file(abs_mozconfig_path, error_level=FATAL) + self.info("mozconfig content:") + self.info(content) + + # finally, copy the mozconfig to a path that 'mach build' expects it to be + self.copyfile(abs_mozconfig_path, os.path.join(dirs['abs_src_dir'], '.mozconfig')) # TODO: replace with ToolToolMixin def _get_tooltool_auth_file(self): diff --git a/testing/taskcluster/tasks/branches/base_job_flags.yml b/testing/taskcluster/tasks/branches/base_job_flags.yml index 6560eedb702a..bd245d6fa75a 100644 --- a/testing/taskcluster/tasks/branches/base_job_flags.yml +++ b/testing/taskcluster/tasks/branches/base_job_flags.yml @@ -35,6 +35,7 @@ flags: - aries-eng - aries-dogfood - android-api-11 + - android-partner-sample1 - linux - linux64 - linux64-st-an diff --git a/testing/taskcluster/tasks/branches/try/job_flags.yml b/testing/taskcluster/tasks/branches/try/job_flags.yml index e095f327e9f0..2602effcf542 100644 --- a/testing/taskcluster/tasks/branches/try/job_flags.yml +++ b/testing/taskcluster/tasks/branches/try/job_flags.yml @@ -138,6 +138,12 @@ builds: task: tasks/builds/opt_linux32_clobber.yml debug: task: tasks/builds/dbg_linux32_clobber.yml + android-partner-sample1: + platforms: + - Android + types: + opt: + task: tasks/builds/android_api_11_partner_sample1.yml sm-plain: platforms: - Linux64 diff --git a/testing/taskcluster/tasks/builds/android_api_11_partner_sample1.yml b/testing/taskcluster/tasks/builds/android_api_11_partner_sample1.yml new file mode 100644 index 000000000000..18e99d5b5cd6 --- /dev/null +++ b/testing/taskcluster/tasks/builds/android_api_11_partner_sample1.yml @@ -0,0 +1,78 @@ +$inherits: + from: 'tasks/builds/mobile_base.yml' + variables: + build_name: 'android-api-11-partner-sample1' + build_type: 'opt' +task: + metadata: + name: '[TC] Android armv7 API 11+ partner sample 1' + description: 'Android armv7 API 11+ partner sample 1' + + workerType: android-api-11 + + routes: + - 'index.buildbot.branches.{{project}}.android-api-11-partner-sample1' + - 'index.buildbot.revisions.{{head_rev}}.{{project}}.android-api-11-partner-sample1' + + scopes: + - 'docker-worker:cache:build-android-api-11-c6-workspace' + - 'docker-worker:cache:tooltool-cache' + - 'docker-worker:relengapi-proxy:tooltool.download.internal' + - 'docker-worker:relengapi-proxy:tooltool.download.public' + + payload: + image: '{{#docker_image}}desktop-build{{/docker_image}}' + cache: + build-android-api-11-c6-workspace: '/home/worker/workspace' + tooltool-cache: '/home/worker/tooltool-cache' + + features: + relengAPIProxy: true + + env: + # inputs to mozharness + MOZHARNESS_SCRIPT: 'mozharness/scripts/fx_desktop_build.py' + # TODO: make these additional configuration files go away + MOZHARNESS_CONFIG: > + builds/releng_base_android_64_builds.py + disable_signing.py + platform_supports_post_upload_to_latest.py + MH_CUSTOM_BUILD_VARIANT_CFG: api-11-partner-sample1 + MH_BRANCH: {{project}} + MH_BUILD_POOL: taskcluster + + # space separated list of repositories required for this build + # for each ITEM in list you want checked out, you must also supply tc-vcs args: + # e.g. ${ITEM}_BASE_REPOSITORY + EXTRA_CHECKOUT_REPOSITORIES: > + PARTNER + + PARTNER_BASE_REPOSITORY: 'https://github.com/mozilla/fennec-distribution-sample' + PARTNER_HEAD_REPOSITORY: 'https://github.com/mozilla/fennec-distribution-sample' + PARTNER_HEAD_REV: 756f0378d4cac87e5e6c405249ede5effe082da2 + PARTNER_DEST_DIR: '/home/worker/workspace/build/partner' + + # image paths + TOOLTOOL_CACHE: '/home/worker/tooltool-cache' + + # authentication + RELENGAPI_TOKEN: 'TODO' # 1164612: encrypt this secret + + maxRunTime: 36000 + + command: ["/bin/bash", "bin/build.sh"] + + extra: + treeherderEnv: + - production + - staging + treeherder: + machine: + # see https://github.com/mozilla/treeherder/blob/master/ui/js/values.js + platform: android-4-0-armv7-api11 + # Rather then enforcing particular conventions we require that all build + # tasks provide the "build" extra field to specify where the build and tests + # files are located. + locations: + build: 'public/build/target.linux-x86_64.tar.bz2' + tests: 'public/build/target.tests.zip'