From 5f84cb18027da64aee3d33e44088b0ef1086530e Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Sat, 6 Apr 2019 05:49:55 +0000 Subject: [PATCH] Bug 1458385: Repackage windows builds on linux. r=glandium,Callek This only uses cross-platform tools, so switch to running these on linux, which cuts the runtime down from ~20m to ~3m. Differential Revision: https://phabricator.services.mozilla.com/D1080 --HG-- extra : moz-landing-system : lando --- taskcluster/ci/repackage-l10n/kind.yml | 7 +++ taskcluster/ci/repackage-msi/kind.yml | 3 + taskcluster/ci/repackage/kind.yml | 7 +++ taskcluster/taskgraph/transforms/repackage.py | 59 +++++-------------- .../taskgraph/transforms/repackage_partner.py | 15 +---- .../configs/repackage/win32_partner.py | 8 +-- .../configs/repackage/win32_signed.py | 9 ++- .../configs/repackage/win64_partner.py | 8 +-- .../configs/repackage/win64_signed.py | 9 ++- testing/mozharness/scripts/repackage.py | 1 + 10 files changed, 51 insertions(+), 75 deletions(-) diff --git a/taskcluster/ci/repackage-l10n/kind.yml b/taskcluster/ci/repackage-l10n/kind.yml index 57c24ac11894..5c9a68813fdf 100644 --- a/taskcluster/ci/repackage-l10n/kind.yml +++ b/taskcluster/ci/repackage-l10n/kind.yml @@ -37,6 +37,9 @@ only-for-build-platforms: - win64-aarch64-devedition-nightly/opt job-template: + worker-type: 'aws-provisioner-v1/gecko-{level}-b-linux' + worker: + docker-image: {"in-tree": "debian7-amd64-build"} mozharness: config: by-build-platform: @@ -77,3 +80,7 @@ job-template: macosx64\b.*: [mar, dmg] win32\b.*: [mar, installer] win64\b.*: [mar, installer] + fetches: + toolchain: + - linux64-mar-tools + - linux64-upx diff --git a/taskcluster/ci/repackage-msi/kind.yml b/taskcluster/ci/repackage-msi/kind.yml index 13c7c6f08aa2..b727b19eb3d4 100644 --- a/taskcluster/ci/repackage-msi/kind.yml +++ b/taskcluster/ci/repackage-msi/kind.yml @@ -25,7 +25,10 @@ only-for-build-platforms: - win64-devedition-nightly/opt job-template: + worker-type: 'aws-provisioner-v1/gecko-{level}-b-win2012' mozharness: + use-magic-mh-args: false + use-caches: false config: by-build-platform: win32\b.*: diff --git a/taskcluster/ci/repackage/kind.yml b/taskcluster/ci/repackage/kind.yml index b0d0686f16a7..f839e175ba65 100644 --- a/taskcluster/ci/repackage/kind.yml +++ b/taskcluster/ci/repackage/kind.yml @@ -47,6 +47,9 @@ only-for-build-platforms: - win64-asan-reporter-nightly/opt job-template: + worker-type: 'aws-provisioner-v1/gecko-{level}-b-linux' + worker: + docker-image: {"in-tree": "debian7-amd64-build"} mozharness: config: by-build-platform: @@ -87,3 +90,7 @@ job-template: macosx64\b.*: [mar, dmg] win32\b.*: [mar, installer] win64\b.*: [mar, installer] + fetches: + toolchain: + - linux64-mar-tools + - linux64-upx diff --git a/taskcluster/taskgraph/transforms/repackage.py b/taskcluster/taskgraph/transforms/repackage.py index ddba741a5b53..df58ab20df89 100644 --- a/taskcluster/taskgraph/transforms/repackage.py +++ b/taskcluster/taskgraph/transforms/repackage.py @@ -17,10 +17,10 @@ from taskgraph.util.schema import ( resolve_keyed_by, ) from taskgraph.util.taskcluster import get_artifact_prefix -from taskgraph.util.platforms import archive_format, executable_extension, architecture +from taskgraph.util.platforms import archive_format, architecture from taskgraph.util.workertypes import worker_type_implementation from taskgraph.transforms.job import job_description_schema -from voluptuous import Required, Optional +from voluptuous import Required, Optional, Extra packaging_description_schema = schema.extend({ @@ -30,6 +30,9 @@ packaging_description_schema = schema.extend({ # unique label to describe this repackaging task Optional('label'): basestring, + Optional('worker-type'): basestring, + Optional('worker'): object, + # treeherder is allowed here to override any defaults we use for repackaging. See # taskcluster/taskgraph/transforms/task.py for the schema details, and the # below transforms for defaults of various values. @@ -56,6 +59,7 @@ packaging_description_schema = schema.extend({ # All l10n jobs use mozharness Required('mozharness'): { + Extra: object, # Config files passed to the mozharness script Required('config'): optionally_keyed_by('build-platform', [basestring]), @@ -76,7 +80,7 @@ packaging_description_schema = schema.extend({ # from mozharness. # - `inputs` are passed as long-options, with the filename prefixed by # `MOZ_FETCH_DIR`. The filename is interpolated by taskgraph -# (`{archive_format}`, `{executable_extension}`). +# (`{archive_format}`). # - `output` is passed to `--output`, with the filename prefixed by the output # directory. PACKAGE_FORMATS = { @@ -87,7 +91,7 @@ PACKAGE_FORMATS = { ], 'inputs': { 'input': 'target{archive_format}', - 'mar': 'mar-tools/mar{executable_extension}', + 'mar': 'mar-tools/mar', }, 'output': "target.complete.mar", }, @@ -98,7 +102,7 @@ PACKAGE_FORMATS = { ], 'inputs': { 'input': 'target{archive_format}', - 'mar': 'mar{executable_extension}', + 'mar': 'mar', }, 'output': "target.bz2.complete.mar", }, @@ -205,10 +209,6 @@ def make_job_description(config, jobs): # repackage-signing can end up with multiple deps... raise NotImplementedError( "Can't repackage a signing task with multiple dependencies") - signing_dependencies = dep_job.dependencies - # This is so we get the build task in our dependencies to - # have better beetmover support. - dependencies.update(signing_dependencies) attributes = copy_attributes_from_dependent_job(dep_job) attributes['repackage_type'] = 'repackage' @@ -230,7 +230,6 @@ def make_job_description(config, jobs): if config.kind == 'repackage-msi': treeherder['symbol'] = 'MSI({})'.format(locale or 'N') - build_task = None signing_task = None repackage_signing_task = None for dependency in dependencies.keys(): @@ -238,21 +237,12 @@ def make_job_description(config, jobs): repackage_signing_task = dependency elif 'signing' in dependency: signing_task = dependency - else: - build_task = dependency _fetch_subst_locale = 'en-US' if locale: - # XXXCallek: todo: rewrite dependency finding - # Use string splice to strip out 'nightly-l10n-' .. '-/opt' - # We need this additional dependency to support finding the mar binary - # Which is needed in order to generate a new complete.mar - dependencies['build'] = "build-{}/opt".format( - dependencies[build_task][13:dependencies[build_task].rfind('-')]) - build_task = 'build' _fetch_subst_locale = locale - level = config.params['level'] + worker_type = job['worker-type'] build_platform = attributes['build_platform'] use_stub = attributes.get('stub-installer') @@ -266,7 +256,6 @@ def make_job_description(config, jobs): command = copy.deepcopy(PACKAGE_FORMATS[format]) substs = { 'archive_format': archive_format(build_platform), - 'executable_extension': executable_extension(build_platform), '_locale': _fetch_subst_locale, 'architecture': architecture(build_platform), 'version_display': config.params['version'], @@ -298,32 +287,18 @@ def make_job_description(config, jobs): }, }) - worker = { + worker = job.get('worker', {}) + worker.update({ 'chain-of-trust': True, 'max-run-time': 7200 if build_platform.startswith('win') else 3600, # Don't add generic artifact directory. 'skip-artifacts': True, - } + }) if locale: # Make sure we specify the locale-specific upload dir worker.setdefault('env', {}).update(LOCALE=locale) - if build_platform.startswith('win'): - worker_type = 'aws-provisioner-v1/gecko-%s-b-win2012' % level - run['use-magic-mh-args'] = False - run['use-caches'] = False - else: - if build_platform.startswith(('linux', 'macosx')): - worker_type = 'aws-provisioner-v1/gecko-%s-b-linux' % level - else: - raise NotImplementedError( - 'Unsupported build_platform: "{}"'.format(build_platform) - ) - - run['tooltool-downloads'] = 'internal' - worker['docker-image'] = {"in-tree": "debian7-amd64-build"} - worker['artifacts'] = _generate_task_output_files( dep_job, worker_type_implementation(config.graph_config, worker_type), repackage_config=repackage_config, @@ -351,7 +326,7 @@ def make_job_description(config, jobs): 'extra': job.get('extra', {}), 'worker': worker, 'run': run, - 'fetches': _generate_download_config(dep_job, build_platform, build_task, + 'fetches': _generate_download_config(dep_job, build_platform, signing_task, repackage_signing_task, locale=locale, project=config.params["project"], @@ -368,7 +343,7 @@ def make_job_description(config, jobs): yield task -def _generate_download_config(task, build_platform, build_task, signing_task, +def _generate_download_config(task, build_platform, signing_task, repackage_signing_task, locale=None, project=None, existing_fetch=None): locale_path = '{}/'.format(locale) if locale else '' @@ -390,7 +365,6 @@ def _generate_download_config(task, build_platform, build_task, signing_task, 'extract': False, }, ], - "toolchain": ["linux64-mar-tools"], }) elif build_platform.startswith('win'): fetch.update({ @@ -401,9 +375,6 @@ def _generate_download_config(task, build_platform, build_task, signing_task, }, '{}setup.exe'.format(locale_path), ], - build_task: [ - 'host/bin/mar.exe', - ], }) use_stub = task.attributes.get('stub-installer') diff --git a/taskcluster/taskgraph/transforms/repackage_partner.py b/taskcluster/taskgraph/transforms/repackage_partner.py index 8a78d6359701..b8b77b1aed8f 100644 --- a/taskcluster/taskgraph/transforms/repackage_partner.py +++ b/taskcluster/taskgraph/transforms/repackage_partner.py @@ -167,19 +167,8 @@ def make_job_description(config, jobs): 'skip-artifacts': True, } - if build_platform.startswith('win'): - worker_type = 'aws-provisioner-v1/gecko-%s-b-win2012' % level - run['use-magic-mh-args'] = False - else: - if build_platform.startswith('macosx'): - worker_type = 'aws-provisioner-v1/gecko-%s-b-linux' % level - else: - raise NotImplementedError( - 'Unsupported build_platform: "{}"'.format(build_platform) - ) - - run['tooltool-downloads'] = 'internal' - worker['docker-image'] = {"in-tree": "debian7-amd64-build"} + worker_type = 'aws-provisioner-v1/gecko-%s-b-linux' % level + worker['docker-image'] = {"in-tree": "debian7-amd64-build"} worker['artifacts'] = _generate_task_output_files( dep_job, worker_type_implementation(config.graph_config, worker_type), diff --git a/testing/mozharness/configs/repackage/win32_partner.py b/testing/mozharness/configs/repackage/win32_partner.py index b0be32fdf634..f73ce73c0416 100644 --- a/testing/mozharness/configs/repackage/win32_partner.py +++ b/testing/mozharness/configs/repackage/win32_partner.py @@ -5,10 +5,10 @@ platform = "win32" config = { "repack_id": os.environ.get("REPACK_ID"), - # ToolTool - "tooltool_manifest_src": 'browser\\config\\tooltool-manifests\\{}\\releng.manifest'.format(platform), 'tooltool_url': 'https://tooltool.mozilla-releng.net/', - 'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'), - 'run_configure': False, + + 'env': { + 'PATH': "%(abs_input_dir)s/upx/bin:%(PATH)s", + } } diff --git a/testing/mozharness/configs/repackage/win32_signed.py b/testing/mozharness/configs/repackage/win32_signed.py index 6f59b1501d04..f07076f832a9 100644 --- a/testing/mozharness/configs/repackage/win32_signed.py +++ b/testing/mozharness/configs/repackage/win32_signed.py @@ -5,10 +5,9 @@ platform = "win32" config = { "locale": os.environ.get("LOCALE"), - # ToolTool - "tooltool_manifest_src": 'browser\\config\\tooltool-manifests\\{}\\releng.manifest'.format(platform), - 'tooltool_url': 'https://tooltool.mozilla-releng.net/', - 'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'), - 'run_configure': False, + + 'env': { + 'PATH': "%(abs_input_dir)s/upx/bin:%(PATH)s", + } } diff --git a/testing/mozharness/configs/repackage/win64_partner.py b/testing/mozharness/configs/repackage/win64_partner.py index e524900ffd2e..54ec61f3a629 100644 --- a/testing/mozharness/configs/repackage/win64_partner.py +++ b/testing/mozharness/configs/repackage/win64_partner.py @@ -5,10 +5,10 @@ platform = "win64" config = { "repack_id": os.environ.get("REPACK_ID"), - # ToolTool - "tooltool_manifest_src": 'browser\\config\\tooltool-manifests\\{}\\releng.manifest'.format(platform), 'tooltool_url': 'https://tooltool.mozilla-releng.net/', - 'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'), - 'run_configure': False, + + 'env': { + 'PATH': "%(abs_input_dir)s/upx/bin:%(PATH)s", + } } diff --git a/testing/mozharness/configs/repackage/win64_signed.py b/testing/mozharness/configs/repackage/win64_signed.py index 13691abda962..ff359334be3c 100644 --- a/testing/mozharness/configs/repackage/win64_signed.py +++ b/testing/mozharness/configs/repackage/win64_signed.py @@ -5,10 +5,9 @@ platform = "win64" config = { "locale": os.environ.get("LOCALE"), - # ToolTool - "tooltool_manifest_src": 'browser\\config\\tooltool-manifests\\{}\\releng.manifest'.format(platform), - 'tooltool_url': 'https://tooltool.mozilla-releng.net/', - 'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'), - 'run_configure': False, + + 'env': { + 'PATH': "%(abs_input_dir)s/upx/bin:%(PATH)s", + } } diff --git a/testing/mozharness/scripts/repackage.py b/testing/mozharness/scripts/repackage.py index 8ecf90bd5baf..ab6ccfccd2b6 100644 --- a/testing/mozharness/scripts/repackage.py +++ b/testing/mozharness/scripts/repackage.py @@ -93,6 +93,7 @@ class Repackage(BaseScript): command=command, cwd=dirs['abs_mozilla_dir'], halt_on_failure=True, + env=self.query_env(), ) def _run_tooltool(self):