Bug 1342392 Migrate partial update generation in-tree r=rail

MozReview-Commit-ID: 84fP48sMYhS --HG-- rename : taskcluster/docker/funsize-update-generator/Dockerfile => taskcluster/docker/partial-update-generator/Dockerfile rename : taskcluster/docker/funsize-update-generator/Makefile => taskcluster/docker/partial-update-generator/Makefile rename : taskcluster/docker/funsize-update-generator/README => taskcluster/docker/partial-update-generator/README rename : taskcluster/docker/funsize-update-generator/dep.pubkey => taskcluster/docker/partial-update-generator/dep.pubkey rename : taskcluster/docker/funsize-update-generator/nightly_sha1.pubkey => taskcluster/docker/partial-update-generator/nightly_sha1.pubkey rename : taskcluster/docker/funsize-update-generator/nightly_sha384.pubkey => taskcluster/docker/partial-update-generator/nightly_sha384.pubkey rename : taskcluster/docker/funsize-update-generator/recompress.sh => taskcluster/docker/partial-update-generator/recompress.sh rename : taskcluster/docker/funsize-update-generator/release_sha1.pubkey => taskcluster/docker/partial-update-generator/release_sha1.pubkey rename : taskcluster/docker/funsize-update-generator/release_sha384.pubkey => taskcluster/docker/partial-update-generator/release_sha384.pubkey rename : taskcluster/docker/funsize-update-generator/requirements.txt => taskcluster/docker/partial-update-generator/requirements.txt rename : taskcluster/docker/funsize-update-generator/runme.sh => taskcluster/docker/partial-update-generator/runme.sh rename : taskcluster/docker/funsize-update-generator/scripts/funsize.py => taskcluster/docker/partial-update-generator/scripts/funsize.py rename : taskcluster/docker/funsize-update-generator/scripts/mbsdiff_hook.sh => taskcluster/docker/partial-update-generator/scripts/mbsdiff_hook.sh rename : taskcluster/docker/funsize-update-generator/scripts/recompress.py => taskcluster/docker/partial-update-generator/scripts/recompress.py extra : rebase_source : 5c6e1e909339f58ada48285294d97f3a4de46c28
2017-09-15 11:04:35 +01:00 · 2017-09-15 11:04:35 +01:00 · 9ba847f43d
--- a/taskcluster/ci/beetmover-repackage/kind.yml
+++ b/taskcluster/ci/beetmover-repackage/kind.yml
@ -12,6 +12,7 @@ transforms:

 kind-dependencies:
  - repackage-signing
+  - partials-signing

 only-for-build-platforms:
  - linux-nightly/opt
--- a/taskcluster/ci/docker-image/kind.yml
+++ b/taskcluster/ci/docker-image/kind.yml
@ -26,3 +26,5 @@ jobs:
    symbol: I(agb)
  index-task:
    symbol: I(idx)
+  partial-update-generator:
+    symbol: I(pg)
--- a/taskcluster/ci/partials-signing/kind.yml
+++ b/taskcluster/ci/partials-signing/kind.yml
@ -0,0 +1,13 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.single_dep:loader
+
+transforms:
+  - taskgraph.transforms.name_sanity:transforms
+  - taskgraph.transforms.partials_signing:transforms
+  - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - partials
--- a/taskcluster/ci/partials/kind.yml
+++ b/taskcluster/ci/partials/kind.yml
@ -0,0 +1,23 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.single_dep:loader
+
+transforms:
+  - taskgraph.transforms.name_sanity:transforms
+  - taskgraph.transforms.partials:transforms
+  - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - repackage-signing
+
+only-for-attributes:
+  - nightly
+
+only-for-build-platforms:
+  - macosx64-nightly/opt
+  - win32-nightly/opt
+  - win64-nightly/opt
+  - linux-nightly/opt
+  - linux64-nightly/opt
--- a/taskcluster/docker/partial-update-generator/Dockerfile
+++ b/taskcluster/docker/partial-update-generator/Dockerfile
@ -25,6 +25,7 @@ RUN pip install -r /tmp/requirements.txt
 # scripts
 RUN mkdir /home/worker/bin
 COPY scripts/* /home/worker/bin/
+
 COPY runme.sh /runme.sh
 COPY recompress.sh /recompress.sh
 RUN chmod 755 /home/worker/bin/* /*.sh
@ -35,3 +36,5 @@ ENV           HOME          /home/worker
 ENV           SHELL         /bin/bash
 ENV           USER          worker
 ENV           LOGNAME       worker
+
+CMD ["/runme.sh"]
--- a/taskcluster/docker/partial-update-generator/Makefile
+++ b/taskcluster/docker/partial-update-generator/Makefile
@ -1,5 +1,5 @@
 DOCKERIO_USERNAME =$(error DOCKERIO_USERNAME should be set)
-IMAGE_NAME = funsize-update-generator
+IMAGE_NAME = partial-update-generator
 FULL_IMAGE_NAME = $(DOCKERIO_USERNAME)/$(IMAGE_NAME)

 build:
--- a/taskcluster/docker/partial-update-generator/README
+++ b/taskcluster/docker/partial-update-generator/README
@ -3,5 +3,5 @@ To run this locally for testing/development purposes:

 1. Find a funsize generating task ID
 2. make pull DOCKERIO_USERNAME=mozillareleases
-3. docker run -t -e SHA1_SIGNING_CERT='nightly_sha1' -e SHA384_SIGNING_CERT='nightly_sha384' -e TASK_ID=LD5HUGP5QNeQdFKNTTuyCg mozillareleases/funsize-update-generator /runme.sh
+3. docker run -t -e SHA1_SIGNING_CERT='nightly_sha1' -e SHA384_SIGNING_CERT='nightly_sha384' -e TASK_ID=LD5HUGP5QNeQdFKNTTuyCg mozillareleases/partial-update-generator /runme.sh

--- a/taskcluster/docker/partial-update-generator/dep.pubkey
+++ b/taskcluster/docker/partial-update-generator/dep.pubkey
--- a/taskcluster/docker/partial-update-generator/nightly_sha1.pubkey
+++ b/taskcluster/docker/partial-update-generator/nightly_sha1.pubkey
--- a/taskcluster/docker/partial-update-generator/nightly_sha384.pubkey
+++ b/taskcluster/docker/partial-update-generator/nightly_sha384.pubkey
--- a/taskcluster/docker/partial-update-generator/recompress.sh
+++ b/taskcluster/docker/partial-update-generator/recompress.sh
--- a/taskcluster/docker/partial-update-generator/release_sha1.pubkey
+++ b/taskcluster/docker/partial-update-generator/release_sha1.pubkey
--- a/taskcluster/docker/partial-update-generator/release_sha384.pubkey
+++ b/taskcluster/docker/partial-update-generator/release_sha384.pubkey
--- a/taskcluster/docker/partial-update-generator/requirements.txt
+++ b/taskcluster/docker/partial-update-generator/requirements.txt
--- a/taskcluster/docker/partial-update-generator/runme.sh
+++ b/taskcluster/docker/partial-update-generator/runme.sh
--- a/taskcluster/docker/partial-update-generator/scripts/funsize.py
+++ b/taskcluster/docker/partial-update-generator/scripts/funsize.py
@ -1,5 +1,4 @@
 #!/usr/bin/env python
-from __future__ import absolute_import, print_function

 import ConfigParser
 import argparse
@ -27,6 +26,7 @@ ALLOWED_URL_PREFIXES = [
    "http://ftp.mozilla.org/",
    "http://download.mozilla.org/",
    "https://archive.mozilla.org/",
+    "https://queue.taskcluster.net/v1/task/",
 ]

 DEFAULT_FILENAME_TEMPLATE = "{appName}-{branch}-{version}-{platform}-" \
@ -286,7 +286,11 @@ def main():
        # if branch not set explicitly use repo-name
        mar_data["branch"] = e.get("branch",
                                   mar_data["repo"].rstrip("/").split("/")[-1])
-        mar_name = args.filename_template.format(**mar_data)
+        if 'dest_mar' in e:
+            mar_name = e['dest_mar']
+        else:
+            # default to formatted name if not specified
+            mar_name = args.filename_template.format(**mar_data)
        mar_data["mar"] = mar_name
        dest_mar = os.path.join(work_env.workdir, mar_name)
        # TODO: download these once
--- a/taskcluster/docker/partial-update-generator/scripts/mbsdiff_hook.sh
+++ b/taskcluster/docker/partial-update-generator/scripts/mbsdiff_hook.sh
--- a/taskcluster/docker/partial-update-generator/scripts/recompress.py
+++ b/taskcluster/docker/partial-update-generator/scripts/recompress.py
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@ -233,3 +233,13 @@ repackage-signing
 -----------------
 Repackage-signing take the repackaged installers (windows) and update packaging (with
 the signed internal bits) and signs them.
+
+partials
+--------
+Partials takes the complete.mar files produced in previous tasks and generates partial
+updates between previous nightly releases and the new one. Requires a release_history
+in the parameters. See ``mach release-history`` if doing this manually.
+
+partials-signing
+----------------
+Partials-signing takes the partial updates produced in Partials and signs them.
--- a/taskcluster/docs/parameters.rst
+++ b/taskcluster/docs/parameters.rst
@ -107,6 +107,12 @@ syntax or reading a project-specific configuration file).
 ``include_nightly``
    If true, then nightly tasks are eligible for optimization.

+``release_history``
+   History of recent releases by platform and locale, used when generating
+   partial updates for nightly releases.
+   Suitable contents can be generated with ``mach release-history``,
+   which will print to the console by default.
+
 Morphed Set
 -----------

--- a/taskcluster/mach_commands.py
+++ b/taskcluster/mach_commands.py
@ -504,3 +504,22 @@ class TaskClusterImagesProvider(object):
        except Exception:
            traceback.print_exc()
            sys.exit(1)
+
+@CommandProvider
+class TaskClusterPartialsData(object):
+    @Command('release-history', category="ci",
+             description="Query balrog for release history used by enable partials generation")
+    @CommandArgument('-b', '--branch',
+                     help="The gecko project branch used in balrog, such as "
+                          "mozilla-central, release, date")
+    @CommandArgument('--product', default='Firefox',
+                     help="The product identifier, such as 'Firefox'")
+    def generate_partials_builds(self, product, branch):
+        from taskgraph.util.partials import populate_release_history
+        try:
+            import yaml
+            release_history = {'release_history': populate_release_history(product, branch)}
+            print(yaml.safe_dump(release_history, allow_unicode=True, default_flow_style=False))
+        except Exception:
+            traceback.print_exc()
+            sys.exit(1)
--- a/taskcluster/taskgraph/decision.py
+++ b/taskcluster/taskgraph/decision.py
@ -18,6 +18,7 @@ from .create import create_tasks
 from .parameters import Parameters
 from .taskgraph import TaskGraph
 from .actions import render_actions_json
+from taskgraph.util.partials import populate_release_history
 from . import GECKO

 from taskgraph.util.templates import Templates
@ -107,6 +108,7 @@ def taskgraph_decision(options):
    """

    parameters = get_decision_parameters(options)
+
    # create a TaskGraphGenerator instance
    tgg = TaskGraphGenerator(
        root_dir=options['root'],
@ -202,6 +204,13 @@ def get_decision_parameters(options):
    if options.get('target_tasks_method'):
        parameters['target_tasks_method'] = options['target_tasks_method']

+    # If the target method is nightly, we should build partials. This means
+    # knowing what has been released previously.
+    # An empty release_history is fine, it just means no partials will be built
+    parameters.setdefault('release_history', dict())
+    if 'nightly' in parameters.get('target_tasks_method', ''):
+        parameters['release_history'] = populate_release_history('Firefox', project)
+
    return Parameters(parameters)


--- a/taskcluster/taskgraph/parameters.py
+++ b/taskcluster/taskgraph/parameters.py
@ -28,6 +28,7 @@ PARAMETER_NAMES = set([
    'project',
    'pushdate',
    'pushlog_id',
+    'release_history',
    'target_task_labels',
    'target_tasks_method',
 ])
--- a/taskcluster/taskgraph/transforms/balrog.py
+++ b/taskcluster/taskgraph/transforms/balrog.py
@ -56,7 +56,7 @@ def make_task_description(config, jobs):
        dep_job = job['dependent-task']

        treeherder = job.get('treeherder', {})
-        treeherder.setdefault('symbol', 'tc-Up(N)')
+        treeherder.setdefault('symbol', 'c-Up(N)')
        dep_th_platform = dep_job.task.get('extra', {}).get(
            'treeherder', {}).get('machine', {}).get('platform', '')
        treeherder.setdefault('platform',
@ -66,11 +66,14 @@ def make_task_description(config, jobs):

        attributes = copy_attributes_from_dependent_job(dep_job)

+        treeherder_job_symbol = dep_job.attributes.get('locale', 'N')
+
        if dep_job.attributes.get('locale'):
-            treeherder['symbol'] = 'tc-Up({})'.format(dep_job.attributes.get('locale'))
+            treeherder['symbol'] = 'c-Up({})'.format(treeherder_job_symbol)
            attributes['locale'] = dep_job.attributes.get('locale')

        label = job['label']
+
        description = (
            "Balrog submission for locale '{locale}' for build '"
            "{build_platform}/{build_type}'".format(
@ -94,7 +97,6 @@ def make_task_description(config, jobs):
        task = {
            'label': label,
            'description': description,
-            # do we have to define worker type somewhere?
            'worker-type': 'scriptworker-prov-v1/balrogworker-v1',
            'worker': {
                'implementation': 'balrog',
--- a/taskcluster/taskgraph/transforms/beetmover_repackage.py
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage.py
@ -9,6 +9,9 @@ from __future__ import absolute_import, print_function, unicode_literals

 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
+from taskgraph.util.partials import (get_balrog_platform_name,
+                                     get_partials_artifacts,
+                                     get_partials_artifact_map)
 from taskgraph.util.schema import validate_schema, Schema
 from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
                                         get_beetmover_action_scope)
@ -215,6 +218,14 @@ def make_task_description(config, jobs):
                                  }
        dependencies.update(repackage_dependencies)

+        # If this isn't a direct dependency, it won't be in there.
+        if 'repackage-signing' not in dependencies:
+            repackage_signing_name = "repackage-signing"
+            repackage_signing_deps = {"repackage-signing":
+                                      dep_job.dependencies[repackage_signing_name]
+                                      }
+            dependencies.update(repackage_signing_deps)
+
        attributes = copy_attributes_from_dependent_job(dep_job)
        if job.get('locale'):
            attributes['locale'] = job['locale']
@ -273,7 +284,6 @@ def generate_upstream_artifacts(build_task_ref, build_signing_task_ref,
                _check_platform_matched_only_one_regex(
                    tasktype, platform, plarform_was_previously_matched_by_regex, platform_regex
                )
-
                upstream_artifacts.append({
                    "taskId": {"task-reference": ref},
                    "taskType": tasktype,
@ -285,6 +295,23 @@ def generate_upstream_artifacts(build_task_ref, build_signing_task_ref,
    return upstream_artifacts


+def generate_partials_upstream_artifacts(artifacts, platform, locale=None):
+    if not locale or locale == 'en-US':
+        artifact_prefix = 'public/build'
+    else:
+        artifact_prefix = 'public/build/{}'.format(locale)
+
+    upstream_artifacts = [{
+        'taskId': {'task-reference': '<partials-signing>'},
+        'taskType': 'signing',
+        'paths': ["{}/{}".format(artifact_prefix, p)
+                  for p in artifacts],
+        'locale': locale or 'en-US',
+    }]
+
+    return upstream_artifacts
+
+
 def _check_platform_matched_only_one_regex(
    task_type, platform, plarform_was_previously_matched_by_regex, platform_regex
 ):
@ -299,8 +326,12 @@ least 2 regular expressions. First matched: "{first_matched}". Second matched: \


 def is_valid_beetmover_job(job):
-    # windows builds don't have docker-image, so fewer dependencies
-    if any(b in job['attributes']['build_platform'] for b in _WINDOWS_BUILD_PLATFORMS):
+    # beetmover after partials-signing should have six dependencies.
+    # windows builds w/o partials don't have docker-image, so fewer
+    # dependencies
+    if 'partials-signing' in job['dependencies'].keys():
+        expected_dep_count = 6
+    elif any(b in job['attributes']['build_platform'] for b in _WINDOWS_BUILD_PLATFORMS):
        expected_dep_count = 4
    else:
        expected_dep_count = 5
@ -321,6 +352,7 @@ def make_task_worker(config, jobs):
        build_signing_task = None
        repackage_task = None
        repackage_signing_task = None
+
        for dependency in job["dependencies"].keys():
            if 'repackage-signing' in dependency:
                repackage_signing_task = dependency
@ -348,3 +380,57 @@ def make_task_worker(config, jobs):
        job["worker"] = worker

        yield job
+
+
+@transforms.add
+def make_partials_artifacts(config, jobs):
+    for job in jobs:
+        locale = job["attributes"].get("locale")
+        if not locale:
+            locale = 'en-US'
+
+        # Remove when proved reliable
+        # job['treeherder']['tier'] = 3
+
+        platform = job["attributes"]["build_platform"]
+
+        balrog_platform = get_balrog_platform_name(platform)
+
+        artifacts = get_partials_artifacts(config.params.get('release_history'),
+                                           balrog_platform, locale)
+
+        # Dependency:        | repackage-signing | partials-signing
+        # Partials artifacts |              Skip | Populate & yield
+        # No partials        |             Yield |         continue
+        if len(artifacts) == 0:
+            if 'partials-signing' in job['dependencies']:
+                continue
+            else:
+                yield job
+                continue
+        else:
+            if 'partials-signing' not in job['dependencies']:
+                continue
+
+        upstream_artifacts = generate_partials_upstream_artifacts(
+            artifacts, balrog_platform, locale
+        )
+
+        job['worker']['upstream-artifacts'].extend(upstream_artifacts)
+
+        extra = list()
+
+        artifact_map = get_partials_artifact_map(
+            config.params.get('release_history'), balrog_platform, locale)
+        for artifact in artifact_map:
+            extra.append({
+                'locale': locale,
+                'artifact_name': artifact,
+                'buildid': artifact_map[artifact],
+                'platform': balrog_platform,
+            })
+
+        job.setdefault('extra', {})
+        job['extra']['partials'] = extra
+
+        yield job
--- a/taskcluster/taskgraph/transforms/partials.py
+++ b/taskcluster/taskgraph/transforms/partials.py
@ -0,0 +1,137 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the partials task into an actual task description.
+"""
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.attributes import copy_attributes_from_dependent_job
+from taskgraph.util.partials import get_balrog_platform_name, get_builds
+from taskgraph.util.taskcluster import get_taskcluster_artifact_prefix
+
+import logging
+logger = logging.getLogger(__name__)
+
+transforms = TransformSequence()
+
+
+def _generate_task_output_files(filenames, locale=None):
+    locale_output_path = '{}/'.format(locale) if locale else ''
+
+    data = list()
+    for filename in filenames:
+        data.append({
+            'type': 'file',
+            'path': '/home/worker/artifacts/{}'.format(filename),
+            'name': 'public/build/{}{}'.format(locale_output_path, filename)
+        })
+    data.append({
+        'type': 'file',
+        'path': '/home/worker/artifacts/manifest.json',
+        'name': 'public/build/{}manifest.json'.format(locale_output_path)
+    })
+    return data
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    # If no balrog release history, then don't generate partials
+    if not config.params.get('release_history'):
+        return
+    for job in jobs:
+        dep_job = job['dependent-task']
+
+        treeherder = job.get('treeherder', {})
+        treeherder.setdefault('symbol', 'p(N)')
+
+        label = job.get('label', "partials-{}".format(dep_job.label))
+        dep_th_platform = dep_job.task.get('extra', {}).get(
+            'treeherder', {}).get('machine', {}).get('platform', '')
+
+        treeherder.setdefault('platform',
+                              "{}/opt".format(dep_th_platform))
+        treeherder.setdefault('kind', 'build')
+        treeherder.setdefault('tier', 1)
+
+        dependent_kind = str(dep_job.kind)
+        dependencies = {dependent_kind: dep_job.label}
+        signing_dependencies = dep_job.dependencies
+        # This is so we get the build task etc in our dependencies to
+        # have better beetmover support.
+        dependencies.update(signing_dependencies)
+
+        attributes = copy_attributes_from_dependent_job(dep_job)
+        locale = dep_job.attributes.get('locale')
+        if locale:
+            attributes['locale'] = locale
+            treeherder['symbol'] = "p({})".format(locale)
+
+        build_locale = locale or 'en-US'
+
+        builds = get_builds(config.params['release_history'], dep_th_platform,
+                            build_locale)
+
+        # If the list is empty there's no available history for this platform
+        # and locale combination, so we can't build any partials.
+        if not builds:
+            continue
+
+        signing_task = None
+        for dependency in sorted(dependencies.keys()):
+            if 'repackage-signing' in dependency:
+                signing_task = dependency
+                break
+        signing_task_ref = '<{}>'.format(signing_task)
+
+        extra = {'funsize': {'partials': list()}}
+        update_number = 1
+        artifact_path = "{}{}".format(
+            get_taskcluster_artifact_prefix(signing_task_ref, locale=locale),
+            'target.complete.mar'
+        )
+        for build in builds:
+            extra['funsize']['partials'].append({
+                'locale': build_locale,
+                'from_mar': builds[build]['mar_url'],
+                'to_mar': {'task-reference': artifact_path},
+                'platform': get_balrog_platform_name(dep_th_platform),
+                'branch': config.params['project'],
+                'update_number': update_number,
+                'dest_mar': build,
+            })
+            update_number += 1
+
+        cot = extra.setdefault('chainOfTrust', {})
+        cot.setdefault('inputs', {})['docker-image'] = {"task-reference": "<docker-image>"}
+
+        worker = {
+            'artifacts': _generate_task_output_files(builds.keys(), locale),
+            'implementation': 'docker-worker',
+            'docker-image': {'in-tree': 'partial-update-generator'},
+            'os': 'linux',
+            'max-run-time': 3600,
+            'chain-of-trust': True,
+            'env': {
+                'SHA1_SIGNING_CERT': 'nightly_sha1',
+                'SHA384_SIGNING_CERT': 'nightly_sha384'
+            }
+        }
+
+        level = config.params['level']
+
+        task = {
+            'label': label,
+            'description': "{} Partials".format(
+                dep_job.task["metadata"]["description"]),
+            'worker-type': 'aws-provisioner-v1/gecko-%s-b-linux' % level,
+            'dependencies': dependencies,
+            'attributes': attributes,
+            'run-on-projects': dep_job.attributes.get('run_on_projects'),
+            'treeherder': treeherder,
+            'extra': extra,
+            'worker': worker,
+        }
+
+        yield task
--- a/taskcluster/taskgraph/transforms/partials_signing.py
+++ b/taskcluster/taskgraph/transforms/partials_signing.py
@ -0,0 +1,95 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the partials task into an actual task description.
+"""
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.attributes import copy_attributes_from_dependent_job
+from taskgraph.util.scriptworker import get_signing_cert_scope_per_platform
+from taskgraph.util.partials import get_balrog_platform_name, get_partials_artifacts
+
+import logging
+logger = logging.getLogger(__name__)
+
+transforms = TransformSequence()
+
+
+def generate_upstream_artifacts(release_history, platform, locale=None):
+    artifact_prefix = 'public/build'
+    if locale:
+        artifact_prefix = 'public/build/{}'.format(locale)
+    else:
+        locale = 'en-US'
+
+    artifacts = get_partials_artifacts(release_history, platform, locale)
+
+    upstream_artifacts = [{
+        "taskId": {"task-reference": '<partials>'},
+        "taskType": 'partials',
+        "paths": ["{}/{}".format(artifact_prefix, p)
+                  for p in artifacts],
+        "formats": ["mar_sha384"],
+    }]
+
+    return upstream_artifacts
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
+
+        treeherder = job.get('treeherder', {})
+        treeherder.setdefault('symbol', 'ps(N)')
+
+        dep_th_platform = dep_job.task.get('extra', {}).get(
+            'treeherder', {}).get('machine', {}).get('platform', '')
+        label = job.get('label', "partials-signing-{}".format(dep_job.label))
+        dep_th_platform = dep_job.task.get('extra', {}).get(
+            'treeherder', {}).get('machine', {}).get('platform', '')
+        treeherder.setdefault('platform',
+                              "{}/opt".format(dep_th_platform))
+        treeherder.setdefault('kind', 'build')
+        treeherder.setdefault('tier', 1)
+
+        dependent_kind = str(dep_job.kind)
+        dependencies = {dependent_kind: dep_job.label}
+        signing_dependencies = dep_job.dependencies
+        # This is so we get the build task etc in our dependencies to
+        # have better beetmover support.
+        dependencies.update(signing_dependencies)
+
+        attributes = copy_attributes_from_dependent_job(dep_job)
+        locale = dep_job.attributes.get('locale')
+        if locale:
+            attributes['locale'] = locale
+            treeherder['symbol'] = 'ps({})'.format(locale)
+
+        balrog_platform = get_balrog_platform_name(dep_th_platform)
+        upstream_artifacts = generate_upstream_artifacts(config.params['release_history'], balrog_platform, locale)
+
+        build_platform = dep_job.attributes.get('build_platform')
+        is_nightly = dep_job.attributes.get('nightly')
+        signing_cert_scope = get_signing_cert_scope_per_platform(
+            build_platform, is_nightly, config
+        )
+        scopes = [signing_cert_scope, 'project:releng:signing:format:mar_sha384']
+        task = {
+            'label': label,
+            'description': "{} Partials".format(
+                dep_job.task["metadata"]["description"]),
+            'worker-type': 'scriptworker-prov-v1/signing-linux-v1',
+            'worker': {'implementation': 'scriptworker-signing',
+                           'upstream-artifacts': upstream_artifacts,
+                           'max-run-time': 3600},
+            'dependencies': dependencies,
+            'attributes': attributes,
+            'scopes': scopes,
+            'run-on-projects': dep_job.attributes.get('run_on_projects'),
+            'treeherder': treeherder,
+        }
+
+        yield task
--- a/taskcluster/taskgraph/transforms/repackage.py
+++ b/taskcluster/taskgraph/transforms/repackage.py
@ -10,12 +10,10 @@ from __future__ import absolute_import, print_function, unicode_literals
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
 from taskgraph.util.schema import validate_schema, Schema
+from taskgraph.util.taskcluster import get_taskcluster_artifact_prefix
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Any, Required, Optional

-_TC_ARTIFACT_LOCATION = \
-        'https://queue.taskcluster.net/v1/task/{task_id}/artifacts/public/build/{postfix}'
-
 transforms = TransformSequence()

 # Voluptuous uses marker objects as dictionary *keys*, but they are not
@ -203,8 +201,8 @@ def _generate_task_mozharness_config(build_platform):


 def _generate_task_env(build_platform, build_task_ref, signing_task_ref, locale=None):
-    mar_prefix = _generate_taskcluster_prefix(build_task_ref, postfix='host/bin/', locale=None)
-    signed_prefix = _generate_taskcluster_prefix(signing_task_ref, locale=locale)
+    mar_prefix = get_taskcluster_artifact_prefix(build_task_ref, postfix='host/bin/', locale=None)
+    signed_prefix = get_taskcluster_artifact_prefix(signing_task_ref, locale=locale)

    if build_platform.startswith('linux') or build_platform.startswith('macosx'):
        tarball_extension = 'bz2' if build_platform.startswith('linux') else 'gz'
@ -231,13 +229,6 @@ def _generate_task_env(build_platform, build_task_ref, signing_task_ref, locale=
    raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform))


-def _generate_taskcluster_prefix(task_id, postfix='', locale=None):
-    if locale:
-        postfix = '{}/{}'.format(locale, postfix)
-
-    return _TC_ARTIFACT_LOCATION.format(task_id=task_id, postfix=postfix)
-
-
 def _generate_task_output_files(build_platform, locale=None):
    locale_output_path = '{}/'.format(locale) if locale else ''

--- a/taskcluster/taskgraph/transforms/repackage_signing.py
+++ b/taskcluster/taskgraph/transforms/repackage_signing.py
@ -129,20 +129,6 @@ def make_repackage_signing_description(config, jobs):
            'treeherder': treeherder,
        }

-        funsize_platforms = [
-            'linux-nightly',
-            'linux64-nightly',
-            'macosx64-nightly',
-            'win32-nightly',
-            'win64-nightly'
-        ]
-        if build_platform in funsize_platforms and is_nightly:
-            route_template = "project.releng.funsize.level-{level}.{project}"
-            task['routes'] = [
-                route_template.format(project=config.params['project'],
-                                      level=config.params['level'])
-            ]
-
        yield task


--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@ -515,7 +515,7 @@ GROUP_NAMES = {
    'tc-L10n-Rpk': 'Localized Repackaged Repacks executed by Taskcluster',
    'tc-BM-L10n': 'Beetmover for locales executed by Taskcluster',
    'tc-BMR-L10n': 'Beetmover repackages for locales executed by Taskcluster',
-    'tc-Up': 'Balrog submission of updates, executed by Taskcluster',
+    'c-Up': 'Balrog submission of complete updates',
    'tc-cs': 'Checksum signing executed by Taskcluster',
    'tc-rs': 'Repackage signing executed by Taskcluster',
    'tc-BMcs': 'Beetmover checksums, executed by Taskcluster',
@ -528,7 +528,10 @@ GROUP_NAMES = {
    'TW64': 'Toolchain builds for Windows 64-bits',
    'SM-tc': 'Spidermonkey builds',
    'pub': 'APK publishing',
+    'p': 'Partial generation',
+    'ps': 'Partials signing',
 }
+
 UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__

 V2_ROUTE_TEMPLATES = [
--- a/taskcluster/taskgraph/util/partials.py
+++ b/taskcluster/taskgraph/util/partials.py
@ -0,0 +1,193 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+import json
+import os
+
+import requests
+import redo
+
+import logging
+logger = logging.getLogger(__name__)
+
+BALROG_API_ROOT = 'https://aus5.mozilla.org/api/v1'
+
+PLATFORM_RENAMES = {
+    'windows2012-32': 'win32',
+    'windows2012-64': 'win64',
+    'osx-cross': 'macosx64',
+}
+
+BALROG_PLATFORM_MAP = {
+    "linux": [
+        "Linux_x86-gcc3"
+    ],
+    "linux64": [
+        "Linux_x86_64-gcc3"
+    ],
+    "macosx64": [
+        "Darwin_x86_64-gcc3-u-i386-x86_64",
+        "Darwin_x86-gcc3-u-i386-x86_64",
+        "Darwin_x86-gcc3",
+        "Darwin_x86_64-gcc3"
+    ],
+    "win32": [
+        "WINNT_x86-msvc",
+        "WINNT_x86-msvc-x86",
+        "WINNT_x86-msvc-x64"
+    ],
+    "win64": [
+        "WINNT_x86_64-msvc",
+        "WINNT_x86_64-msvc-x64"
+    ]
+}
+
+
+def get_balrog_platform_name(platform):
+    """Convert build platform names into balrog platform names"""
+    if '-nightly' in platform:
+        platform = platform.replace('-nightly', '')
+    if '-devedition' in platform:
+        platform = platform.replace('-devedition', '')
+    return PLATFORM_RENAMES.get(platform, platform)
+
+
+def _sanitize_platform(platform):
+    platform = get_balrog_platform_name(platform)
+    if platform not in BALROG_PLATFORM_MAP:
+        return platform
+    return BALROG_PLATFORM_MAP[platform][0]
+
+
+def get_builds(release_history, platform, locale):
+    """Examine cached balrog release history and return the list of
+    builds we need to generate diffs from"""
+    platform = _sanitize_platform(platform)
+    return release_history.get(platform, {}).get(locale, {})
+
+
+def get_partials_artifacts(release_history, platform, locale):
+    platform = _sanitize_platform(platform)
+    return release_history.get(platform, {}).get(locale, {}).keys()
+
+
+def get_partials_artifact_map(release_history, platform, locale):
+    platform = _sanitize_platform(platform)
+    return {k: release_history[platform][locale][k]['buildid'] for k in release_history.get(platform, {}).get(locale, {})}
+
+
+def _retry_on_http_errors(url, verify, params, errors):
+    if params:
+        params_str = "&".join("=".join([k, str(v)])
+                              for k, v in params.iteritems())
+    else:
+        params_str = ''
+    logger.info("Connecting to %s?%s", url, params_str)
+    for _ in redo.retrier(sleeptime=5, max_sleeptime=30, attempts=10):
+        try:
+            req = requests.get(url, verify=verify, params=params, timeout=4)
+            req.raise_for_status()
+            return req
+        except requests.HTTPError as e:
+            if e.response.status_code in errors:
+                logger.exception("Got HTTP %s trying to reach %s",
+                                 e.response.status_code, url)
+            else:
+                raise
+    else:
+        raise
+
+
+def get_sorted_releases(product, branch):
+    """Returns a list of release names from Balrog.
+    :param product: product name, AKA appName
+    :param branch: branch name, e.g. mozilla-central
+    :return: a sorted list of release names, most recent first.
+    """
+    url = "{}/releases".format(BALROG_API_ROOT)
+    params = {
+        "product": product,
+        # Adding -nightly-2 (2 stands for the beginning of build ID
+        # based on date) should filter out release and latest blobs.
+        # This should be changed to -nightly-3 in 3000 ;)
+        "name_prefix": "{}-{}-nightly-2".format(product, branch),
+        "names_only": True
+    }
+    req = _retry_on_http_errors(
+        url=url, verify=True, params=params,
+        errors=[500])
+    releases = req.json()["names"]
+    releases = sorted(releases, reverse=True)
+    return releases
+
+
+def get_release_builds(release):
+    url = "{}/releases/{}".format(BALROG_API_ROOT, release)
+    req = _retry_on_http_errors(
+        url=url, verify=True, params=None,
+        errors=[500])
+    return req.json()
+
+
+def populate_release_history(product, branch, maxbuilds=4, maxsearch=10):
+    """Find relevant releases in Balrog
+    Not all releases have all platforms and locales, due
+    to Taskcluster migration.
+
+        Args:
+            product (str): capitalized product name, AKA appName, e.g. Firefox
+            branch (str): branch name (mozilla-central)
+            maxbuilds (int): Maximum number of historical releases to populate
+            maxsearch(int): Traverse at most this many releases, to avoid
+                working through the entire history.
+        Returns:
+            json object based on data from balrog api
+
+            results = {
+                'platform1': {
+                    'locale1': {
+                        'buildid1': mar_url,
+                        'buildid2': mar_url,
+                        'buildid3': mar_url,
+                    },
+                    'locale2': {
+                        'target.partial-1.mar': {'buildid1': 'mar_url'},
+                    }
+                },
+                'platform2': {
+                }
+            }
+        """
+    last_releases = get_sorted_releases(product, branch)
+
+    partial_mar_tmpl = 'target.partial-{}.mar'
+
+    builds = dict()
+    for release in last_releases[:maxsearch]:
+        # maxbuilds in all categories, don't make any more queries
+        full = len(builds) > 0 and all(
+            len(builds[platform][locale]) >= maxbuilds for platform in builds for locale in builds[platform])
+        if full:
+            break
+        history = get_release_builds(release)
+
+        for platform in history['platforms']:
+            if 'alias' in history['platforms'][platform]:
+                continue
+            if platform not in builds:
+                builds[platform] = dict()
+            for locale in history['platforms'][platform]['locales']:
+                if locale not in builds[platform]:
+                    builds[platform][locale] = dict()
+                if len(builds[platform][locale]) >= maxbuilds:
+                    continue
+                buildid = history['platforms'][platform]['locales'][locale]['buildID']
+                url = history['platforms'][platform]['locales'][locale]['completes'][0]['fileUrl']
+                nextkey = len(builds[platform][locale]) + 1
+                builds[platform][locale][partial_mar_tmpl.format(nextkey)] = {
+                    'buildid': buildid,
+                    'mar_url': url,
+                }
+    return builds
--- a/taskcluster/taskgraph/util/taskcluster.py
+++ b/taskcluster/taskgraph/util/taskcluster.py
@ -13,6 +13,9 @@ from mozbuild.util import memoize
 from requests.packages.urllib3.util.retry import Retry
 from requests.adapters import HTTPAdapter

+_TC_ARTIFACT_LOCATION = \
+        'https://queue.taskcluster.net/v1/task/{task_id}/artifacts/public/build/{postfix}'
+

@memoize
 def get_session():
@ -101,3 +104,10 @@ def get_task_url(task_id, use_proxy=False):
 def get_task_definition(task_id, use_proxy=False):
    response = _do_request(get_task_url(task_id, use_proxy))
    return response.json()
+
+
+def get_taskcluster_artifact_prefix(task_id, postfix='', locale=None):
+    if locale:
+        postfix = '{}/{}'.format(locale, postfix)
+
+    return _TC_ARTIFACT_LOCATION.format(task_id=task_id, postfix=postfix)