Bug 1538703 - Full rollout of Firefox to mozilla-beta. r=sfraser a=release

--HG--
extra : rebase_source : 8ea111a2bcc62f92a2cfe9fa718d2ac970fc1005

taskcluster/ci/beetmover-checksums/kind.yml

@@ -26,6 +26,10 @@ job-template:
                by-platform:
                   android.*: taskcluster/taskgraph/manifests/fennec_nightly_checksums.yml
                   default: taskcluster/taskgraph/manifests/firefox_nightly_checksums.yml
+            try:
+               by-platform:
+                  android.*: taskcluster/taskgraph/manifests/fennec_candidates_checksums.yml
+                  default: taskcluster/taskgraph/manifests/firefox_candidates_checksums.yml
             mozilla-beta:
                by-platform:
                   android.*: taskcluster/taskgraph/manifests/fennec_candidates_checksums.yml

taskcluster/ci/beetmover-repackage/kind.yml

@@ -58,6 +58,7 @@ job-template:
     attributes:
         artifact_map:
             by-project:
-                mozilla-release: taskcluster/taskgraph/manifests/firefox_candidates.yml
-                mozilla-beta: taskcluster/taskgraph/manifests/firefox_candidates.yml
                 default: taskcluster/taskgraph/manifests/firefox_nightly.yml
+                try: taskcluster/taskgraph/manifests/firefox_candidates.yml
+                mozilla-beta: taskcluster/taskgraph/manifests/firefox_candidates.yml
+                mozilla-release: taskcluster/taskgraph/manifests/firefox_candidates.yml

taskcluster/ci/release-beetmover-signed-langpacks-checksums/kind.yml

@@ -14,3 +14,6 @@ kind-dependencies:
 
 job-template:
    shipping-phase: promote
+   attributes:
+      artifact_prefix: public
+      artifact_map: taskcluster/taskgraph/manifests/firefox_candidates_checksums.yml

taskcluster/ci/release-beetmover-signed-langpacks/kind.yml

@@ -24,3 +24,5 @@ job-template:
          staging: scriptworker-prov-v1/beetmoverworker-dev
    run-on-projects: []
    shipping-phase: promote
+   attributes:
+      artifact_map: taskcluster/taskgraph/manifests/firefox_candidates.yml

taskcluster/taskgraph/manifests/firefox_candidates.yml

@@ -0,0 +1,353 @@
+# This file contains exhaustive information about all the release artifacs that
+# are needed within a type of release.
+#
+# Structure
+# --------
+# `s3_bucket_paths`       -- prefix to be used per product to correctly access our S3 buckets
+# `default_locales`       -- list of locales to be used when composing upstream artifacts or the list of
+#                      destinations. If given an empty locale, it uses these locales instead.
+# `tasktype_map`          -- mapping between task reference and task type, particularly usefule when
+#                      composing the upstreamArtifacts for scriptworker.
+# `platform_names`        -- various platform mappings used in reckoning artifacts or other paths
+# `default`               -- a default entry, which the mappings extend and override in such a way that
+#                      final path full-destinations will be a concatenation of the following:
+#                      `s3_bucket_paths`, `destinations`, `locale_prefix`, `pretty_name`
+# `from`                  -- specifies the dependency(ies) from which to expect the particular artifact
+# `all_locales`           -- boolean argument to specify whether that particular artifact is to be expected
+#                      for all locales or just the default one
+# `description`           -- brief summary of what that artifact is
+# `locale_prefix`         -- prefix to be used in the final destination paths, whether that's for default locale or not
+# `source_path_modifier`  -- any parent dir that might be used in between artifact prefix and filename at source location
+#                            for example `public/build` vs `public/build/ach/`.
+# `destinations`          -- final list of directories where to push the artifacts in S3
+# `pretty_name`           -- the final name the artifact will have at destination
+# `checksums_path`        -- the name to identify one artifact within the checksums file
+# `not_for_platforms`     -- filtering option to avoid associating an artifact with a specific platform
+# `only_for_platforms`    -- filtering option to exclusively include the association of an artifact for a specific platform
+# `partials_only`         -- filtering option to avoid associating an artifact unless this flag is present
+# `update_balrog_manifest`-- flag needed downstream in beetmover jobs to reckon the balrog manifest
+# `from_buildid`          -- flag needed downstream in beetmover jobs to reckon the balrog manifest
+
+---
+s3_bucket_paths:
+  - pub/firefox/candidates
+default_locales:
+  - en-US
+tasktype_map:
+  build: build
+  signing: signing
+  mar-signing: signing
+  partials-signing: signing
+  repackage: repackage
+  repackage-signing: repackage
+  repackage-signing-msi: repackage
+  release-sign-and-push-langpacks: scriptworker
+platform_names:
+  path_platform:
+    by-platform:
+      linux-nightly: 'linux-i686'
+      linux64-nightly: 'linux-x86_64'
+      linux64-asan-reporter-nightly: 'linux-x86_64-asan-reporter'
+      macosx64-nightly: 'mac'
+      win32-nightly: 'win32'
+      win64-nightly: 'win64'
+      win64-aarch64-nightly: 'win64-aarch64'
+      win64-asan-reporter-nightly: 'win64-asan-reporter'
+  filename_platform:
+    by-platform:
+      linux-nightly: 'linux-i686'
+      linux64-nightly: 'linux-x86_64'
+      linux64-asan-reporter-nightly: 'linux-x86_64-asan-reporter'
+      macosx64-nightly: 'mac'
+      win32-nightly: 'win32'
+      win64-nightly: 'win64'
+      win64-aarch64-nightly: 'win64-aarch64'
+      win64-asan-reporter-nightly: 'win64-asan-reporter'
+  stage_platform:
+    by-platform:
+      linux-nightly: 'linux'
+      linux64-nightly: 'linux64'
+      linux64-asan-reporter-nightly: 'linux-x86_64-asan-reporter'
+      macosx64-nightly: 'macosx64'
+      win32-nightly: 'win32'
+      win64-nightly: 'win64'
+      win64-aarch64-nightly: 'win64-aarch64'
+      win64-asan-reporter-nightly: 'win64-asan-reporter'
+
+default: &default
+  from:
+    - build
+  all_locales: false
+  description: "TO_BE_OVERRIDDEN"
+  locale_prefix: '${locale}/'
+  source_path_modifier:
+    by-locale:
+      default: '${locale}'
+      en-US: ''
+  destinations:
+    - ${version}-candidates/build${build_number}/${path_platform}
+
+mapping:
+  buildhub.json:
+    <<: *default
+    all_locales: false
+    description: "Build related information to be consumed by Buildhub service"
+    pretty_name: buildhub.json
+    checksums_path: ${path_platform}/${locale}/buildhub.json
+  target.common.tests.tar.gz:
+    <<: *default
+    description: "Mixture of reftests, mochitests, UI and others, commonly bundled together in a test suite"
+    pretty_name: firefox-${version}.common.tests.tar.gz
+    checksums_path: ${path_platform}/${locale}/firefox-${version}.common.tests.tar.gz
+  target.cppunittest.tests.tar.gz:
+    <<: *default
+    description: "C++ unittests related in-tree test infrastructure"
+    pretty_name: firefox-${version}.cppunittest.tests.tar.gz
+    checksums_path: ${path_platform}/${locale}/firefox-${version}.cppunittest.tests.tar.gz
+  target.crashreporter-symbols.zip:
+    <<: *default
+    description: "Crashreporter symbols to be consumed by Socorro"
+    pretty_name: firefox-${version}.crashreporter-symbols.zip
+    checksums_path: ${path_platform}/${locale}/firefox-${version}.crashreporter-symbols.zip
+  target.json:
+    <<: *default
+    description: "Various compile and moz_app flags baked together in a json file"
+    pretty_name: firefox-${version}.json
+    checksums_path: ${path_platform}/${locale}/firefox-${version}.json
+  target.mochitest.tests.tar.gz:
+    <<: *default
+    description: "Results for running the mochitest testing framework via Javascript function calls"
+    pretty_name: firefox-${version}.mochitest.tests.tar.gz
+    checksums_path: ${path_platform}/${locale}/firefox-${version}.mochitest.tests.tar.gz
+  target.mozinfo.json:
+    <<: *default
+    description: "Various compile and moz_app flags baked together in a json file"
+    pretty_name: firefox-${version}.mozinfo.json
+    checksums_path: ${path_platform}/${locale}/firefox-${version}.mozinfo.json
+  target.reftest.tests.tar.gz:
+    <<: *default
+    description: "Results for running the reftest testing framework via display of two Web pages comparison"
+    pretty_name: firefox-${version}.reftest.tests.tar.gz
+    checksums_path: ${path_platform}/${locale}/firefox-${version}.reftest.tests.tar.gz
+  target.talos.tests.tar.gz:
+    <<: *default
+    description: "Results for running the talos testing framework to measure performance"
+    pretty_name: firefox-${version}.talos.tests.tar.gz
+    checksums_path: ${path_platform}/${locale}/firefox-${version}.talos.tests.tar.gz
+  target.awsy.tests.tar.gz:
+    <<: *default
+    description: "Results for running the awsy testing framework to track memory usage"
+    pretty_name: firefox-${version}.awsy.tests.tar.gz
+    checksums_path: ${path_platform}/${locale}/firefox-${version}.awsy.tests.tar.gz
+  target.test_packages.json:
+    <<: *default
+    description: "File containing metadata about all other files and testing harnesses specifics"
+    pretty_name: firefox-${version}.test_packages.json
+    checksums_path: ${path_platform}/${locale}/firefox-${version}.test_packages.json
+  target.txt:
+    <<: *default
+    description: "File containing buildid and revision"
+    pretty_name: firefox-${version}.txt
+    checksums_path: ${path_platform}/${locale}/firefox-${version}.txt
+  target.web-platform.tests.tar.gz:
+    <<: *default
+    description: "Results for running the webplatform testing framework to cover standard Web platform features"
+    pretty_name: firefox-${version}.web-platform.tests.tar.gz
+    checksums_path: ${path_platform}/${locale}/firefox-${version}.web-platform.tests.tar.gz
+  target.xpcshell.tests.tar.gz:
+    <<: *default
+    description: "Results for running the xpcshell testing framework to enable XPConnect console application"
+    pretty_name: firefox-${version}.xpcshell.tests.tar.gz
+    checksums_path: ${path_platform}/${locale}/firefox-${version}.xpcshell.tests.tar.gz
+  target_info.txt:
+    <<: *default
+    description: "File containing the buildID"
+    locale_prefix: ''
+    pretty_name: ${stage_platform}_info.txt
+    checksums_path: ${stage_platform}_info.txt
+    destinations:
+      - ${version}-candidates/build${build_number}
+  mozharness.zip:
+    <<: *default
+    description: "File containing the mozharness set of scripts and configuration used by various automation tools"
+    pretty_name: mozharness.zip
+    checksums_path: ${path_platform}/${locale}/mozharness.zip
+  target.jsshell.zip:
+    <<: *default
+    description: "Set of shells to allow test snippets of Javascript code without needing to reload the page"
+    locale_prefix: ''
+    pretty_name: jsshell-${filename_platform}.zip
+    checksums_path: jsshell/jsshell-${filename_platform}.zip
+    destinations:
+      - ${version}-candidates/build${build_number}/jsshell
+  target.langpack.xpi:
+    <<: *default
+    all_locales: true
+    description: "Localized repack that grabs a packaged en-US Firefox and repackages it as locale-specific Firefox"
+    locale_prefix: ''
+    source_path_modifier: '${locale}'
+    from:
+      - release-sign-and-push-langpacks
+    only_for_platforms:
+      - linux-nightly
+      - linux64-nightly
+      - macosx64-nightly
+      - win32-nightly
+      - win64-nightly
+    pretty_name: ${locale}.xpi
+    checksums_path: ${path_platform}/xpi/${locale}.xpi
+    destinations:
+      - ${version}-candidates/build${build_number}/${path_platform}/xpi
+  mar:
+    <<: *default
+    description: "Alongside `mbsdiff`, a tool used to generate partials"
+    locale_prefix: ''
+    source_path_modifier: 'host/bin'
+    pretty_name: ${stage_platform}/mar
+    checksums_path: mar-tools/${stage_platform}/mar
+    not_for_platforms:
+      - win32-nightly
+      - win64-nightly
+      - win64-aarch64-nightly
+    destinations:
+      - ${version}-candidates/build${build_number}/mar-tools
+  mbsdiff:
+    <<: *default
+    description: "Alongside `mar`, a tool used to generate partials"
+    locale_prefix: ''
+    source_path_modifier: 'host/bin'
+    pretty_name: ${stage_platform}/mbsdiff
+    checksums_path: mar-tools/${stage_platform}/mbsdiff
+    not_for_platforms:
+      - win32-nightly
+      - win64-nightly
+      - win64-aarch64-nightly
+    destinations:
+      - ${version}-candidates/build${build_number}/mar-tools
+  mar.exe:
+    <<: *default
+    description: "Alongside `mbsdiff.exe`, a tool used to generate partials for Windows platforms"
+    locale_prefix: ''
+    source_path_modifier: 'host/bin'
+    pretty_name: ${stage_platform}/mar.exe
+    checksums_path: mar-tools/${stage_platform}/mar.exe
+    only_for_platforms:
+      - win32-nightly
+      - win64-nightly
+      - win64-aarch64-nightly
+    destinations:
+      - ${version}-candidates/build${build_number}/mar-tools
+  mbsdiff.exe:
+    <<: *default
+    locale_prefix: ''
+    description: "Alongside `mar.exe`, a tool used to generate partials for Windows platforms"
+    source_path_modifier: 'host/bin'
+    pretty_name: ${stage_platform}/mbsdiff.exe
+    checksums_path: mar-tools/${stage_platform}/mbsdiff.exe
+    only_for_platforms:
+      - win32-nightly
+      - win64-nightly
+      - win64-aarch64-nightly
+    destinations:
+      - ${version}-candidates/build${build_number}/mar-tools
+  target.tar.bz2:
+    <<: *default
+    description: "Main installer for Linux platforms"
+    all_locales: true
+    from:
+      - signing
+    only_for_platforms:
+      - linux-nightly
+      - linux64-nightly
+    pretty_name: firefox-${version}.tar.bz2
+    checksums_path: ${path_platform}/${locale}/firefox-${version}.tar.bz2
+  target.tar.bz2.asc:
+    <<: *default
+    description: "Detached signature for the checksums file"
+    all_locales: true
+    from:
+      - signing
+    only_for_platforms:
+      - linux-nightly
+      - linux64-nightly
+    pretty_name: firefox-${version}.tar.bz2.asc
+    checksums_path: ${path_platform}/${locale}/firefox-${version}.tar.bz2.asc
+  target.dmg:
+    <<: *default
+    description: "Main package installer for Mac OS X platforms"
+    all_locales: true
+    from:
+      - repackage
+    only_for_platforms:
+      - macosx64-nightly
+    pretty_name: Firefox ${version}.dmg
+    checksums_path: ${path_platform}/${locale}/Firefox ${version}.dmg
+  target.zip:
+    <<: *default
+    description: "Main package installer for Windows platforms"
+    all_locales: true
+    from:
+      - signing
+    only_for_platforms:
+      - win64-nightly
+      - win32-nightly
+      - win64-aarch64-nightly
+    pretty_name: firefox-${version}.zip
+    checksums_path: ${path_platform}/${locale}/firefox-${version}.zip
+  target.installer.exe:
+    <<: *default
+    description: "Main installer for Windows platforms"
+    all_locales: true
+    from:
+      - repackage-signing
+    only_for_platforms:
+      - win64-nightly
+      - win32-nightly
+      - win64-aarch64-nightly
+    pretty_name: Firefox Setup ${version}.exe
+    checksums_path: ${path_platform}/${locale}/Firefox Setup ${version}.exe
+  target.stub-installer.exe:
+    <<: *default
+    description: "Stub installer for Win32 platforms"
+    all_locales: true
+    from:
+      - repackage-signing
+    only_for_platforms:
+      - win32-nightly
+    pretty_name: Firefox Installer.exe
+    checksums_path: ${path_platform}/${locale}/Firefox Installer.exe
+  target.installer.msi:
+    <<: *default
+    description: "Windows installer for MSI platform"
+    all_locales: true
+    from:
+      - repackage-signing-msi
+    only_for_platforms:
+      - win64-nightly
+      - win32-nightly
+    pretty_name: Firefox Setup ${version}.msi
+    checksums_path: ${path_platform}/${locale}/Firefox Setup ${version}.msi
+  target.complete.mar:
+    <<: *default
+    description: "The main installer we ship our mobile products baked within"
+    all_locales: true
+    from:
+      - mar-signing
+    pretty_name: firefox-${version}.complete.mar
+    checksums_path: update/${path_platform}/${locale}/firefox-${version}.complete.mar
+    update_balrog_manifest: true
+    destinations:
+      - ${version}-candidates/build${build_number}/update/${path_platform}
+  ${partial}:
+    <<: *default
+    description: "Partials MAR files to serve as updates"
+    all_locales: true
+    from:
+      - partials-signing
+    partials_only: true
+    pretty_name: firefox-${previous_version}-${version}.partial.mar
+    checksums_path: update/${path_platform}/${locale}/firefox-${previous_version}-${version}.partial.mar
+    update_balrog_manifest: true
+    from_buildid: ${from_buildid}
+    destinations:
+      - ${version}-candidates/build${build_number}/update/${path_platform}

taskcluster/taskgraph/manifests/firefox_candidates_checksums.yml

@@ -0,0 +1,117 @@
+# This file contains exhaustive information about all the release artifacs that
+# are needed within a type of release.
+#
+# Structure
+# --------
+# `s3_bucket_paths`       -- prefix to be used per product to correctly access our S3 buckets
+# `default_locales`       -- list of locales to be used when composing upstream artifacts or the list of
+#                      destinations. If given an empty locale, it uses these locales instead.
+# `tasktype_map`          -- mapping between task reference and task type, particularly usefule when
+#                      composing the upstreamArtifacts for scriptworker.
+# `platform_names`        -- various platform mappings used in reckoning artifacts or other paths
+# `default`               -- a default entry, which the mappings extend and override in such a way that
+#                      final path full-destinations will be a concatenation of the following:
+#                      `s3_bucket_paths`, `destinations`, `locale_prefix`, `pretty_name`
+# `from`                  -- specifies the dependency(ies) from which to expect the particular artifact
+# `all_locales`           -- boolean argument to specify whether that particular artifact is to be expected
+#                      for all locales or just the default one
+# `description`           -- brief summary of what that artifact is
+# `locale_prefix`         -- prefix to be used in the final destination paths, whether that's for default locale or not
+# `source_path_modifier`  -- any parent dir that might be used in between artifact prefix and filename at source location
+#                            for example `public/build` vs `public/build/ach/`.
+# `destinations`          -- final list of directories where to push the artifacts in S3
+# `pretty_name`           -- the final name the artifact will have at destination
+# `checksums_path`        -- the name to identify one artifact within the checksums file
+# `not_for_platforms`     -- filtering option to avoid associating an artifact with a specific platform
+# `only_for_platforms`    -- filtering option to exclusively include the association of an artifact for a specific platform
+# `partials_only`         -- filtering option to avoid associating an artifact unless this flag is present
+# `update_balrog_manifest`-- flag needed downstream in beetmover jobs to reckon the balrog manifest
+# `from_buildid`          -- flag needed downstream in beetmover jobs to reckon the balrog manifest
+---
+s3_bucket_paths:
+  by-platform:
+    .*devedition.*:
+      - pub/devedition/candidates
+    default:
+      - pub/firefox/candidates
+default_locales:
+  - en-US
+tasktype_map:
+  checksums-signing: signing
+  release-beetmover-signed-langpacks: signing
+platform_names:
+  path_platform:
+    by-platform:
+      linux-nightly: 'linux-i686'
+      linux64-nightly: 'linux-x86_64'
+      linux64-asan-reporter-nightly: 'linux-x86_64-asan-reporter'
+      macosx64-nightly: 'mac'
+      win32-nightly: 'win32'
+      win64-nightly: 'win64'
+      win64-aarch64-nightly: 'win64-aarch64'
+      win64-asan-reporter-nightly: 'win64-asan-reporter'
+      linux-shippable: 'linux-i686'
+      linux64-shippable: 'linux-x86_64'
+      macosx64-shippable: 'mac'
+      win32-shippable: 'win32'
+      win64-shippable: 'win64'
+      win64-aarch64-shippable: 'win64-aarch64'
+      linux: 'linux-i686'
+      linux64: 'linux-x86_64'
+      macosx64: 'mac'
+      win32: 'win32'
+      win64: 'win64'
+  filename_platform:
+    by-platform:
+      linux-nightly: 'linux-i686'
+      linux64-nightly: 'linux-x86_64'
+      linux64-asan-reporter-nightly: 'linux-x86_64-asan-reporter'
+      macosx64-nightly: 'mac'
+      win32-nightly: 'win32'
+      win64-nightly: 'win64'
+      win64-aarch64-nightly: 'win64-aarch64'
+      win64-asan-reporter-nightly: 'win64-asan-reporter'
+      linux-shippable: 'linux-i686'
+      linux64-shippable: 'linux-x86_64'
+      macosx64-shippable: 'mac'
+      win32-shippable: 'win32'
+      win64-shippable: 'win64'
+      win64-aarch64-shippable: 'win64-aarch64'
+      linux: 'linux-i686'
+      linux64: 'linux-x86_64'
+      macosx64: 'mac'
+      win32: 'win32'
+      win64: 'win64'
+  stage_platform: ''
+
+default: &default
+  from:
+    - checksums-signing
+  all_locales: true
+  description: "TO_BE_OVERRIDDEN"
+  locale_prefix: '${locale}/'
+  source_path_modifier: ''
+  destinations:
+    - ${version}-candidates/build${build_number}/beetmover-checksums/${path_platform}
+
+mapping:
+  target.checksums:
+    <<: *default
+    description: "Checksums file containing size, hash, sha algorithm and filename"
+    pretty_name: firefox-${version}.checksums.beet
+    checksums_path: beetmover-checksums/${path_platform}/${locale}/firefox-${version}.checksums.beet
+  target.checksums.asc:
+    <<: *default
+    description: "Detached signature for the checksums file"
+    pretty_name: firefox-${version}.checksums.asc
+    checksums_path: beetmover-checksums/${path_platform}/${locale}/firefox-${version}.checksums.asc
+  target-langpack.checksums:
+    <<: *default
+    description: "Checksums file containing size, hash, sha algorithm and filename for the langpack"
+    locale_prefix: ''
+    from:
+      - release-beetmover-signed-langpacks
+    pretty_name: ${locale}.checksums.beet
+    checksums_path: beetmover-checksums/${path_platform}/xpi/${locale}.checksums.beet
+    destinations:
+      - ${version}-candidates/build${build_number}/beetmover-checksums/${path_platform}/xpi

taskcluster/taskgraph/transforms/beetmover_langpack_checksums.py

@@ -11,14 +11,18 @@ from taskgraph.loader.single_dep import schema
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.transforms.beetmover import craft_release_properties
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
-from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
+from taskgraph.util.scriptworker import (generate_beetmover_artifact_map,
+                                         generate_beetmover_upstream_artifacts,
                                          get_beetmover_action_scope,
-                                         get_worker_type_for_scope)
+                                         get_beetmover_bucket_scope,
+                                         get_worker_type_for_scope,
+                                         should_use_artifact_map)
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Required, Optional
 
 beetmover_checksums_description_schema = schema.extend({
     Required('depname', default='build'): basestring,
+    Required('attributes'): {basestring: object},
     Optional('label'): basestring,
     Optional('treeherder'): task_description_schema['treeherder'],
     Optional('locale'): basestring,
@@ -67,6 +71,7 @@ def make_beetmover_checksums_description(config, jobs):
         attributes = copy_attributes_from_dependent_job(dep_job)
         if 'chunk_locales' in dep_job.attributes:
             attributes['chunk_locales'] = dep_job.attributes['chunk_locales']
+        attributes.update(job.get('attributes', {}))
 
         bucket_scope = get_beetmover_bucket_scope(config)
         action_scope = get_beetmover_action_scope(config)
@@ -136,11 +141,20 @@ def make_beetmover_checksums_worker(config, jobs):
         worker = {
             'implementation': 'beetmover',
             'release-properties': craft_release_properties(config, job),
-            'upstream-artifacts': generate_upstream_artifacts(
-                refs, platform, locales
-            ),
         }
 
+        if should_use_artifact_map(platform, config.params['project']):
+            upstream_artifacts = generate_beetmover_upstream_artifacts(
+                config, job, platform, locales
+            )
+            worker['artifact-map'] = generate_beetmover_artifact_map(
+                config, job, platform=platform, locale=locales)
+        else:
+            upstream_artifacts = generate_upstream_artifacts(
+                refs, platform, locales
+            )
+
+        worker['upstream-artifacts'] = upstream_artifacts
         job["worker"] = worker
 
         yield job

taskcluster/taskgraph/transforms/release_beetmover_signed_addons.py

@@ -13,7 +13,10 @@ from taskgraph.transforms.beetmover import craft_release_properties
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
 from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by
 from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
-                                         get_beetmover_action_scope)
+                                         get_beetmover_action_scope,
+                                         generate_beetmover_upstream_artifacts,
+                                         generate_beetmover_artifact_map,
+                                         should_use_artifact_map)
 from taskgraph.transforms.task import task_description_schema
 from taskgraph.transforms.release_sign_and_push_langpacks import get_upstream_task_ref
 from voluptuous import Required, Optional
@@ -31,6 +34,9 @@ beetmover_description_schema = schema.extend({
     # depname is used in taskref's to identify the taskID of the unsigned things
     Required('depname', default='build'): basestring,
 
+    # attributes is used for enabling artifact-map by declarative artifacts
+    Required('attributes'): {basestring: object},
+
     # unique label to describe this beetmover task, defaults to {dep.label}-beetmover
     Optional('label'): basestring,
 
@@ -66,10 +72,14 @@ transforms.add_validate(beetmover_description_schema)
 @transforms.add
 def resolve_keys(config, jobs):
     for job in jobs:
-        resolve_keyed_by(
+        for field in ('worker-type', 'attributes.artifact_map'):
-            job, 'worker-type', item_name=job['label'],
+            resolve_keyed_by(
-            **{'release-level': config.params.release_level()}
+                job, field, item_name=job['label'],
-        )
+                **{
+                    'release-level': config.params.release_level(),
+                    'project': config.params['project']
+                }
+            )
         yield job
 
 
@@ -88,7 +98,7 @@ def make_task_description(config, jobs):
         treeherder.setdefault('tier', 1)
         treeherder.setdefault('kind', 'build')
 
-        job['attributes'] = copy_attributes_from_dependent_job(dep_job)
+        job['attributes'].update(copy_attributes_from_dependent_job(dep_job))
         job['attributes']['chunk_locales'] = dep_job.attributes['chunk_locales']
 
         job['description'] = job['description'].format(
@@ -118,14 +128,26 @@ def make_task_worker(config, jobs):
             job, expected_kinds=('release-sign-and-push-langpacks',)
         )
 
+        platform = job["attributes"]["build_platform"]
+        locale = job["attributes"]["chunk_locales"]
+        if should_use_artifact_map(platform, config.params['project']):
+            upstream_artifacts = generate_beetmover_upstream_artifacts(
+                config, job, platform, locale,
+            )
+        else:
+            upstream_artifacts = generate_upstream_artifacts(
+                signing_task_ref, job['attributes']['chunk_locales']
+            )
         job['worker'] = {
             'implementation': 'beetmover',
             'release-properties': craft_release_properties(config, job),
-            'upstream-artifacts': generate_upstream_artifacts(
+            'upstream-artifacts': upstream_artifacts,
-                signing_task_ref, job['attributes']['chunk_locales']
-            ),
         }
 
+        if should_use_artifact_map(platform, config.params['project']):
+            job['worker']['artifact-map'] = generate_beetmover_artifact_map(
+                config, job, platform=platform, locale=locale)
+
         yield job
 
 
@@ -167,7 +189,7 @@ def yield_all_platform_jobs(config, jobs):
                         platform in ('macosx64', 'macosx64-devedition'):
                     platform_job = _strip_ja_data_from_linux_job(platform_job)
 
-                platform_job = _change_platform_data(platform_job, platform)
+                platform_job = _change_platform_data(config, platform_job, platform)
 
                 yield platform_job
 
@@ -187,10 +209,25 @@ def _strip_ja_data_from_linux_job(platform_job):
     return platform_job
 
 
-def _change_platform_data(platform_job, platform):
+def _change_platform_in_artifact_map_paths(paths, orig_platform, new_platform):
+    amended_paths = {}
+    for artifact, artifact_info in paths.iteritems():
+        amended_artifact_info = {
+            'checksums_path': artifact_info['checksums_path'].replace(orig_platform, new_platform),
+            'destinations': [
+                d.replace(orig_platform, new_platform) for d in artifact_info['destinations']
+            ]
+        }
+        amended_paths[artifact] = amended_artifact_info
+
+    return amended_paths
+
+
+def _change_platform_data(config, platform_job, platform):
     orig_platform = 'linux64'
     if 'devedition' in platform:
         orig_platform = 'linux64-devedition'
+    backup_platform = platform_job['attributes']['build_platform']
     platform_job['attributes']['build_platform'] = platform
     platform_job['label'] = platform_job['label'].replace(orig_platform, platform)
     platform_job['description'] = platform_job['description'].replace(orig_platform, platform)
@@ -199,4 +236,25 @@ def _change_platform_data(platform_job, platform):
     )
     platform_job['worker']['release-properties']['platform'] = platform
 
+    # amend artifactMap entries as well
+    if should_use_artifact_map(backup_platform, config.params['project']):
+        platform_mapping = {
+            'linux64': 'linux-x86_64',
+            'linux': 'linux-i686',
+            'macosx64': 'mac',
+            'win32': 'win32',
+            'win64': 'win64',
+        }
+        orig_platform = platform_mapping.get(orig_platform, orig_platform)
+        platform = platform_mapping.get(platform, platform)
+        platform_job['worker']['artifact-map'] = [
+            {
+                'locale': entry['locale'],
+                'taskId': entry['taskId'],
+                'paths': _change_platform_in_artifact_map_paths(entry['paths'],
+                                                                orig_platform,
+                                                                platform)
+            } for entry in platform_job['worker']['artifact-map']
+        ]
+
     return platform_job

taskcluster/taskgraph/util/scriptworker.py

@@ -432,6 +432,8 @@ def generate_beetmover_upstream_artifacts(config, job, platform, locale=None, de
 
     if not locale:
         locales = map_config['default_locales']
+    elif isinstance(locale, list):
+        locales = locale
     else:
         locales = [locale]
 
@@ -562,7 +564,10 @@ def generate_beetmover_artifact_map(config, job, **kwargs):
     dependencies = job['dependencies'].keys()
 
     if kwargs.get('locale'):
-        locales = [kwargs['locale']]
+        if isinstance(kwargs['locale'], list):
+            locales = kwargs['locale']
+        else:
+            locales = [kwargs['locale']]
     else:
         locales = map_config['default_locales']
 
@@ -780,6 +785,7 @@ def generate_beetmover_partials_artifact_map(config, job, partials_info, **kwarg
                 kwargs.update({
                     'partial': pname,
                     'from_buildid': info['buildid'],
+                    'previous_version': info.get('previousVersion'),
                     'buildid': str(config.params['moz_build_date']),
                     'locale': locale,
                     'version': config.params['version'],
@@ -826,10 +832,15 @@ def should_use_artifact_map(platform, project):
         'fennec'
     ]
     projects = ['mozilla-central', 'mozilla-beta', 'mozilla-release']
-    if any([pl in platform for pl in platforms]) and any([pj in project for pj in projects]):
+    if any([pl in platform for pl in platforms]) and any([pj == project for pj in projects]):
         return True
 
     platforms = [
+        'linux',    # needed for beetmover-langpacks-checksums
+        'linux64',  # which inherit amended platform from their beetmover counterpart
+        'win32',
+        'win64',
+        'macosx64',
         'linux-shippable',
         'linux64-shippable',
         'macosx64-shippable',
@@ -838,9 +849,11 @@ def should_use_artifact_map(platform, project):
         'win64-aarch64-shippable',
         'win64-asan-reporter-nightly',
         'linux64-asan-reporter-nightly',
+        'firefox-source',
+        'firefox-release',
     ]
-    projects = ['mozilla-central']
+    projects = ['try', 'mozilla-central', 'mozilla-beta', 'mozilla-release']
-    if any([pl in platform for pl in platforms]) and any([pj in project for pj in projects]):
+    if any([pl == platform for pl in platforms]) and any([pj == project for pj in projects]):
         return True
 
     return False