From 0cd3337d9068dc98e45b8acef7fba8fb3b9ad521 Mon Sep 17 00:00:00 2001
From: Tom Prince <mozilla@hocat.ca>
Date: Thu, 11 Jul 2019 16:40:13 +0000
Subject: [PATCH] Bug 1458385: Package mar and mbsdiff as a toolchain;
 r=glandium

Differential Revision: https://phabricator.services.mozilla.com/D24229

--HG--
extra : moz-landing-system : lando
---
 build/sparse-profiles/taskgraph               |  7 +++++
 mfbt/moz.build                                |  2 +-
 other-licenses/bsdiff/moz.build               |  1 +
 taskcluster/ci/toolchain/misc.yml             | 17 ++++++++++
 taskcluster/scripts/misc/build-mar-tools.sh   | 31 +++++++++++++++++++
 taskcluster/taskgraph/transforms/repackage.py |  6 ++--
 tools/update-packaging/app.mozbuild           |  9 ++++++
 tools/update-packaging/moz.configure          |  0
 8 files changed, 68 insertions(+), 5 deletions(-)
 create mode 100755 taskcluster/scripts/misc/build-mar-tools.sh
 create mode 100644 tools/update-packaging/app.mozbuild
 create mode 100644 tools/update-packaging/moz.configure

diff --git a/build/sparse-profiles/taskgraph b/build/sparse-profiles/taskgraph
index 9a9d70ac9b88..3df3d0b0b514 100644
--- a/build/sparse-profiles/taskgraph
+++ b/build/sparse-profiles/taskgraph
@@ -56,3 +56,10 @@ path:.cron.yml
 
 # for the wrench-deps toolchain task
 path:gfx/wr/Cargo.lock
+
+# for the mar-tools toolchain task
+path:mfbt/
+path:modules/libmar/
+path:other-licenses/bsdiff/
+path:other-licenses/nsis/Contrib/CityHash/cityhash/
+path:toolkit/mozapps/update/updater
diff --git a/mfbt/moz.build b/mfbt/moz.build
index aa36b32c9a22..0e379493c1ea 100644
--- a/mfbt/moz.build
+++ b/mfbt/moz.build
@@ -155,7 +155,7 @@ UNIFIED_SOURCES += [
     'Utf8.cpp',
 ]
 
-if CONFIG['MOZ_BUILD_APP'] not in ('memory', 'tools/crashreporter'):
+if CONFIG['MOZ_BUILD_APP'] not in ('memory', 'tools/crashreporter', 'tools/update-packaging'):
     # RecordReplay.cpp uses files from js/
     UNIFIED_SOURCES += [
         'RecordReplay.cpp',
diff --git a/other-licenses/bsdiff/moz.build b/other-licenses/bsdiff/moz.build
index 424e096393b4..c7080a6e249f 100644
--- a/other-licenses/bsdiff/moz.build
+++ b/other-licenses/bsdiff/moz.build
@@ -18,4 +18,5 @@ if CONFIG['HOST_OS_ARCH'] == 'WINNT':
 
 LOCAL_INCLUDES += [
     '/toolkit/mozapps/update/updater',
+    '/toolkit/mozapps/update/updater/bspatch',
 ]
diff --git a/taskcluster/ci/toolchain/misc.yml b/taskcluster/ci/toolchain/misc.yml
index 6fc1bb6b60b2..93e481309559 100644
--- a/taskcluster/ci/toolchain/misc.yml
+++ b/taskcluster/ci/toolchain/misc.yml
@@ -70,6 +70,23 @@ linux64-libdmg:
         script: build-libdmg-hfsplus.sh
         toolchain-artifact: public/build/dmg.tar.xz
 
+linux64-mar-tools:
+    description: "mar-tools toolchain build"
+    treeherder:
+        symbol: TL(mar-tools)
+    toolchains:
+        - linux64-clang-7
+        - linux64-binutils
+    run:
+        script: build-mar-tools.sh
+        sparse-profile: null
+        toolchain-artifact: public/build/mar-tools.tar.xz
+        resources:
+            - modules/libmar/
+            - other-licenses/nsis/Contrib/CityHash/cityhash/
+            - other-licenses/bsdiff/
+            - toolkit/mozapps/update/updater/bspatch/
+
 linux64-tup:
     description: "tup toolchain build"
     treeherder:
diff --git a/taskcluster/scripts/misc/build-mar-tools.sh b/taskcluster/scripts/misc/build-mar-tools.sh
new file mode 100755
index 000000000000..2313f3125424
--- /dev/null
+++ b/taskcluster/scripts/misc/build-mar-tools.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+set -x -e -v
+
+# This script is for building mar and mbsdiff
+
+WORKSPACE=$HOME/workspace
+UPLOAD_DIR=$HOME/artifacts
+COMPRESS_EXT=xz
+
+cd $WORKSPACE/build/src
+
+. taskcluster/scripts/misc/tooltool-download.sh
+
+export MOZ_OBJDIR=obj-mar
+
+echo ac_add_options --enable-project=tools/update-packaging > .mozconfig
+
+TOOLCHAINS="binutils clang"
+
+for t in $TOOLCHAINS; do
+    PATH="$WORKSPACE/build/src/$t/bin:$PATH"
+done
+
+./mach build -v
+
+mkdir mar-tools
+cp $MOZ_OBJDIR/dist/host/bin/{mar,mbsdiff} mar-tools/
+
+tar -acf mar-tools.tar.$COMPRESS_EXT mar-tools/
+mkdir -p $UPLOAD_DIR
+cp mar-tools.tar.$COMPRESS_EXT $UPLOAD_DIR
diff --git a/taskcluster/taskgraph/transforms/repackage.py b/taskcluster/taskgraph/transforms/repackage.py
index 9e03337919db..59b35713178e 100644
--- a/taskcluster/taskgraph/transforms/repackage.py
+++ b/taskcluster/taskgraph/transforms/repackage.py
@@ -88,7 +88,7 @@ PACKAGE_FORMATS = {
         ],
         'inputs': {
             'input': 'target{archive_format}',
-            'mar': 'mar{executable_extension}',
+            'mar': 'mar-tools/mar{executable_extension}',
         },
         'output': "target.complete.mar",
     },
@@ -392,9 +392,7 @@ def _generate_download_config(task, build_platform, build_task, signing_task,
                     'extract': False,
                 },
             ],
-            build_task: [
-                'host/bin/mar',
-            ],
+            "toolchain": ["linux64-mar-tools"],
         })
     elif build_platform.startswith('win'):
         fetch.update({
diff --git a/tools/update-packaging/app.mozbuild b/tools/update-packaging/app.mozbuild
new file mode 100644
index 000000000000..9053ec17f1dd
--- /dev/null
+++ b/tools/update-packaging/app.mozbuild
@@ -0,0 +1,9 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+DIRS += [
+    '/modules/libmar/src',
+    '/modules/libmar/tool',
+    '/other-licenses/bsdiff',
+]
diff --git a/tools/update-packaging/moz.configure b/tools/update-packaging/moz.configure
new file mode 100644
index 000000000000..e69de29bb2d1