Bug 1458385: Repackage windows builds on linux. r=glandium,Callek

This only uses cross-platform tools, so switch to running these on linux, which
cuts the runtime down from ~20m to ~3m.

Differential Revision: https://phabricator.services.mozilla.com/D1080

--HG--
extra : moz-landing-system : lando

taskcluster/ci/repackage-l10n/kind.yml

@@ -37,6 +37,9 @@ only-for-build-platforms:
     - win64-aarch64-devedition-nightly/opt
 
 job-template:
+    worker-type: 'aws-provisioner-v1/gecko-{level}-b-linux'
+    worker:
+        docker-image: {"in-tree": "debian7-amd64-build"}
     mozharness:
         config:
             by-build-platform:
@@ -77,3 +80,7 @@ job-template:
                     macosx64\b.*: [mar, dmg]
                     win32\b.*: [mar, installer]
                     win64\b.*: [mar, installer]
+    fetches:
+        toolchain:
+            - linux64-mar-tools
+            - linux64-upx

taskcluster/ci/repackage-msi/kind.yml

@@ -25,7 +25,10 @@ only-for-build-platforms:
     - win64-devedition-nightly/opt
 
 job-template:
+    worker-type: 'aws-provisioner-v1/gecko-{level}-b-win2012'
     mozharness:
+        use-magic-mh-args: false
+        use-caches: false
         config:
             by-build-platform:
                 win32\b.*:

taskcluster/ci/repackage/kind.yml

@@ -47,6 +47,9 @@ only-for-build-platforms:
     - win64-asan-reporter-nightly/opt
 
 job-template:
+    worker-type: 'aws-provisioner-v1/gecko-{level}-b-linux'
+    worker:
+        docker-image: {"in-tree": "debian7-amd64-build"}
     mozharness:
         config:
             by-build-platform:
@@ -87,3 +90,7 @@ job-template:
                     macosx64\b.*: [mar, dmg]
                     win32\b.*: [mar, installer]
                     win64\b.*: [mar, installer]
+    fetches:
+        toolchain:
+            - linux64-mar-tools
+            - linux64-upx

taskcluster/taskgraph/transforms/repackage.py

@@ -17,10 +17,10 @@ from taskgraph.util.schema import (
     resolve_keyed_by,
 )
 from taskgraph.util.taskcluster import get_artifact_prefix
-from taskgraph.util.platforms import archive_format, executable_extension, architecture
+from taskgraph.util.platforms import archive_format, architecture
 from taskgraph.util.workertypes import worker_type_implementation
 from taskgraph.transforms.job import job_description_schema
-from voluptuous import Required, Optional
+from voluptuous import Required, Optional, Extra
 
 
 packaging_description_schema = schema.extend({
@@ -30,6 +30,9 @@ packaging_description_schema = schema.extend({
     # unique label to describe this repackaging task
     Optional('label'): basestring,
 
+    Optional('worker-type'): basestring,
+    Optional('worker'): object,
+
     # treeherder is allowed here to override any defaults we use for repackaging.  See
     # taskcluster/taskgraph/transforms/task.py for the schema details, and the
     # below transforms for defaults of various values.
@@ -56,6 +59,7 @@ packaging_description_schema = schema.extend({
 
     # All l10n jobs use mozharness
     Required('mozharness'): {
+        Extra: object,
         # Config files passed to the mozharness script
         Required('config'): optionally_keyed_by('build-platform', [basestring]),
 
@@ -76,7 +80,7 @@ packaging_description_schema = schema.extend({
 #    from mozharness.
 # - `inputs` are passed as long-options, with the filename prefixed by
 #   `MOZ_FETCH_DIR`. The filename is interpolated by taskgraph
-#   (`{archive_format}`, `{executable_extension}`).
+#   (`{archive_format}`).
 # - `output` is passed to `--output`, with the filename prefixed by the output
 #   directory.
 PACKAGE_FORMATS = {
@@ -87,7 +91,7 @@ PACKAGE_FORMATS = {
         ],
         'inputs': {
             'input': 'target{archive_format}',
-            'mar': 'mar-tools/mar{executable_extension}',
+            'mar': 'mar-tools/mar',
         },
         'output': "target.complete.mar",
     },
@@ -98,7 +102,7 @@ PACKAGE_FORMATS = {
         ],
         'inputs': {
             'input': 'target{archive_format}',
-            'mar': 'mar{executable_extension}',
+            'mar': 'mar',
         },
         'output': "target.bz2.complete.mar",
     },
@@ -205,10 +209,6 @@ def make_job_description(config, jobs):
             # repackage-signing can end up with multiple deps...
             raise NotImplementedError(
                 "Can't repackage a signing task with multiple dependencies")
-        signing_dependencies = dep_job.dependencies
-        # This is so we get the build task in our dependencies to
-        # have better beetmover support.
-        dependencies.update(signing_dependencies)
 
         attributes = copy_attributes_from_dependent_job(dep_job)
         attributes['repackage_type'] = 'repackage'
@@ -230,7 +230,6 @@ def make_job_description(config, jobs):
         if config.kind == 'repackage-msi':
             treeherder['symbol'] = 'MSI({})'.format(locale or 'N')
 
-        build_task = None
         signing_task = None
         repackage_signing_task = None
         for dependency in dependencies.keys():
@@ -238,21 +237,12 @@ def make_job_description(config, jobs):
                 repackage_signing_task = dependency
             elif 'signing' in dependency:
                 signing_task = dependency
-            else:
-                build_task = dependency
 
         _fetch_subst_locale = 'en-US'
         if locale:
-            # XXXCallek: todo: rewrite dependency finding
-            # Use string splice to strip out 'nightly-l10n-' .. '-<chunk>/opt'
-            # We need this additional dependency to support finding the mar binary
-            # Which is needed in order to generate a new complete.mar
-            dependencies['build'] = "build-{}/opt".format(
-                dependencies[build_task][13:dependencies[build_task].rfind('-')])
-            build_task = 'build'
             _fetch_subst_locale = locale
 
-        level = config.params['level']
+        worker_type = job['worker-type']
         build_platform = attributes['build_platform']
 
         use_stub = attributes.get('stub-installer')
@@ -266,7 +256,6 @@ def make_job_description(config, jobs):
             command = copy.deepcopy(PACKAGE_FORMATS[format])
             substs = {
                 'archive_format': archive_format(build_platform),
-                'executable_extension': executable_extension(build_platform),
                 '_locale': _fetch_subst_locale,
                 'architecture': architecture(build_platform),
                 'version_display': config.params['version'],
@@ -298,32 +287,18 @@ def make_job_description(config, jobs):
             },
         })
 
-        worker = {
+        worker = job.get('worker', {})
+        worker.update({
             'chain-of-trust': True,
             'max-run-time': 7200 if build_platform.startswith('win') else 3600,
             # Don't add generic artifact directory.
             'skip-artifacts': True,
-        }
+        })
 
         if locale:
             # Make sure we specify the locale-specific upload dir
             worker.setdefault('env', {}).update(LOCALE=locale)
 
-        if build_platform.startswith('win'):
-            worker_type = 'aws-provisioner-v1/gecko-%s-b-win2012' % level
-            run['use-magic-mh-args'] = False
-            run['use-caches'] = False
-        else:
-            if build_platform.startswith(('linux', 'macosx')):
-                worker_type = 'aws-provisioner-v1/gecko-%s-b-linux' % level
-            else:
-                raise NotImplementedError(
-                    'Unsupported build_platform: "{}"'.format(build_platform)
-                )
-
-            run['tooltool-downloads'] = 'internal'
-            worker['docker-image'] = {"in-tree": "debian7-amd64-build"}
-
         worker['artifacts'] = _generate_task_output_files(
             dep_job, worker_type_implementation(config.graph_config, worker_type),
             repackage_config=repackage_config,
@@ -351,7 +326,7 @@ def make_job_description(config, jobs):
             'extra': job.get('extra', {}),
             'worker': worker,
             'run': run,
-            'fetches': _generate_download_config(dep_job, build_platform, build_task,
+            'fetches': _generate_download_config(dep_job, build_platform,
                                                  signing_task, repackage_signing_task,
                                                  locale=locale,
                                                  project=config.params["project"],
@@ -368,7 +343,7 @@ def make_job_description(config, jobs):
         yield task
 
 
-def _generate_download_config(task, build_platform, build_task, signing_task,
+def _generate_download_config(task, build_platform, signing_task,
                               repackage_signing_task, locale=None, project=None,
                               existing_fetch=None):
     locale_path = '{}/'.format(locale) if locale else ''
@@ -390,7 +365,6 @@ def _generate_download_config(task, build_platform, build_task, signing_task,
                     'extract': False,
                 },
             ],
-            "toolchain": ["linux64-mar-tools"],
         })
     elif build_platform.startswith('win'):
         fetch.update({
@@ -401,9 +375,6 @@ def _generate_download_config(task, build_platform, build_task, signing_task,
                 },
                 '{}setup.exe'.format(locale_path),
             ],
-            build_task: [
-                'host/bin/mar.exe',
-            ],
         })
 
         use_stub = task.attributes.get('stub-installer')

taskcluster/taskgraph/transforms/repackage_partner.py

@@ -167,19 +167,8 @@ def make_job_description(config, jobs):
             'skip-artifacts': True,
         }
 
-        if build_platform.startswith('win'):
-            worker_type = 'aws-provisioner-v1/gecko-%s-b-win2012' % level
-            run['use-magic-mh-args'] = False
-        else:
-            if build_platform.startswith('macosx'):
-                worker_type = 'aws-provisioner-v1/gecko-%s-b-linux' % level
-            else:
-                raise NotImplementedError(
-                    'Unsupported build_platform: "{}"'.format(build_platform)
-                )
-
-            run['tooltool-downloads'] = 'internal'
-            worker['docker-image'] = {"in-tree": "debian7-amd64-build"}
+        worker_type = 'aws-provisioner-v1/gecko-%s-b-linux' % level
+        worker['docker-image'] = {"in-tree": "debian7-amd64-build"}
 
         worker['artifacts'] = _generate_task_output_files(
             dep_job, worker_type_implementation(config.graph_config, worker_type),

testing/mozharness/configs/repackage/win32_partner.py

@@ -5,10 +5,10 @@ platform = "win32"
 config = {
     "repack_id": os.environ.get("REPACK_ID"),
 
-    # ToolTool
-    "tooltool_manifest_src": 'browser\\config\\tooltool-manifests\\{}\\releng.manifest'.format(platform),
     'tooltool_url': 'https://tooltool.mozilla-releng.net/',
-    'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'),
-
     'run_configure': False,
+
+    'env': {
+        'PATH': "%(abs_input_dir)s/upx/bin:%(PATH)s",
+    }
 }

testing/mozharness/configs/repackage/win32_signed.py

@@ -5,10 +5,9 @@ platform = "win32"
 config = {
     "locale": os.environ.get("LOCALE"),
 
-    # ToolTool
-    "tooltool_manifest_src": 'browser\\config\\tooltool-manifests\\{}\\releng.manifest'.format(platform),
-    'tooltool_url': 'https://tooltool.mozilla-releng.net/',
-    'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'),
-
     'run_configure': False,
+
+    'env': {
+        'PATH': "%(abs_input_dir)s/upx/bin:%(PATH)s",
+    }
 }

testing/mozharness/configs/repackage/win64_partner.py

@@ -5,10 +5,10 @@ platform = "win64"
 config = {
     "repack_id": os.environ.get("REPACK_ID"),
 
-    # ToolTool
-    "tooltool_manifest_src": 'browser\\config\\tooltool-manifests\\{}\\releng.manifest'.format(platform),
     'tooltool_url': 'https://tooltool.mozilla-releng.net/',
-    'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'),
-
     'run_configure': False,
+
+    'env': {
+        'PATH': "%(abs_input_dir)s/upx/bin:%(PATH)s",
+    }
 }

testing/mozharness/configs/repackage/win64_signed.py

@@ -5,10 +5,9 @@ platform = "win64"
 config = {
     "locale": os.environ.get("LOCALE"),
 
-    # ToolTool
-    "tooltool_manifest_src": 'browser\\config\\tooltool-manifests\\{}\\releng.manifest'.format(platform),
-    'tooltool_url': 'https://tooltool.mozilla-releng.net/',
-    'tooltool_cache': os.environ.get('TOOLTOOL_CACHE'),
-
     'run_configure': False,
+
+    'env': {
+        'PATH': "%(abs_input_dir)s/upx/bin:%(PATH)s",
+    }
 }

testing/mozharness/scripts/repackage.py

@@ -93,6 +93,7 @@ class Repackage(BaseScript):
                 command=command,
                 cwd=dirs['abs_mozilla_dir'],
                 halt_on_failure=True,
+                env=self.query_env(),
             )
 
     def _run_tooltool(self):