Bug 1395697 - Add scripts to handle MAR recompression in release automation r=catlee

MozReview-Commit-ID: 5WCy01RFQMX --HG-- extra : rebase_source : 933cb0caf9667e14ad54c2a91a7b9c853a979a19
2017-09-16 08:38:34 -04:00 · 2017-09-16 08:38:34 -04:00 · d56e74be69
--- a/taskcluster/docker/funsize-update-generator/Dockerfile
+++ b/taskcluster/docker/funsize-update-generator/Dockerfile
@ -26,7 +26,8 @@ RUN pip install -r /tmp/requirements.txt
 RUN mkdir /home/worker/bin
 COPY scripts/* /home/worker/bin/
 COPY runme.sh /runme.sh
-RUN chmod 755 /home/worker/bin/* /runme.sh
+COPY recompress.sh /recompress.sh
+RUN chmod 755 /home/worker/bin/* /*.sh
 RUN mkdir /home/worker/keys
 COPY *.pubkey /home/worker/keys/

--- a/taskcluster/docker/funsize-update-generator/recompress.sh
+++ b/taskcluster/docker/funsize-update-generator/recompress.sh
@ -0,0 +1,24 @@
+#!/bin/sh
+
+set -xe
+
+test $TASK_ID
+test $SHA1_SIGNING_CERT
+test $SHA384_SIGNING_CERT
+
+ARTIFACTS_DIR="/home/worker/artifacts"
+mkdir -p "$ARTIFACTS_DIR"
+
+curl --location --retry 10 --retry-delay 10 -o /home/worker/task.json \
+    "https://queue.taskcluster.net/v1/task/$TASK_ID"
+
+if [ ! -z $OUTPUT_FILENAME ]; then
+    EXTRA_PARAMS="--output-filename $OUTPUT_FILENAME $EXTRA_PARAMS"
+fi
+
+/home/worker/bin/recompress.py \
+    --artifacts-dir "$ARTIFACTS_DIR" \
+    --task-definition /home/worker/task.json \
+    --sha1-signing-cert "/home/worker/keys/${SHA1_SIGNING_CERT}.pubkey" \
+    --sha384-signing-cert "/home/worker/keys/${SHA384_SIGNING_CERT}.pubkey" \
+    $EXTRA_PARAMS
--- a/taskcluster/docker/funsize-update-generator/scripts/recompress.py
+++ b/taskcluster/docker/funsize-update-generator/scripts/recompress.py
@ -0,0 +1,208 @@
+#!/usr/bin/env python
+from __future__ import absolute_import, print_function
+
+import argparse
+import functools
+import hashlib
+import json
+import logging
+import os
+import shutil
+import tempfile
+import requests
+import sh
+
+import redo
+from mardor.reader import MarReader
+from mardor.signing import get_keysize
+
+
+log = logging.getLogger(__name__)
+ALLOWED_URL_PREFIXES = [
+    "http://download.cdn.mozilla.net/pub/mozilla.org/firefox/nightly/",
+    "http://download.cdn.mozilla.net/pub/firefox/nightly/",
+    "https://mozilla-nightly-updates.s3.amazonaws.com",
+    "https://queue.taskcluster.net/",
+    "http://ftp.mozilla.org/",
+    "http://download.mozilla.org/",
+    "https://archive.mozilla.org/",
+]
+
+
+def verify_signature(mar, certs):
+    log.info("Checking %s signature", mar)
+    with open(mar, 'rb') as mar_fh:
+        m = MarReader(mar_fh)
+        m.verify(verify_key=certs.get(m.signature_type))
+
+
+def is_lzma_compressed_mar(mar):
+    log.info("Checking %s for lzma compression", mar)
+    result = MarReader(open(mar, 'rb')).compression_type == 'xz'
+    if result:
+        log.info("%s is lzma compressed", mar)
+    else:
+        log.info("%s is not lzma compressed", mar)
+    return result
+
+
+@redo.retriable()
+def download(url, dest, mode=None):
+    log.debug("Downloading %s to %s", url, dest)
+    r = requests.get(url)
+    r.raise_for_status()
+
+    bytes_downloaded = 0
+    with open(dest, 'wb') as fd:
+        for chunk in r.iter_content(4096):
+            fd.write(chunk)
+            bytes_downloaded += len(chunk)
+
+    log.debug('Downloaded %s bytes', bytes_downloaded)
+    if 'content-length' in r.headers:
+        log.debug('Content-Length: %s bytes', r.headers['content-length'])
+        if bytes_downloaded != int(r.headers['content-length']):
+            raise IOError('Unexpected number of bytes downloaded')
+
+    if mode:
+        log.debug("chmod %o %s", mode, dest)
+        os.chmod(dest, mode)
+
+
+def change_mar_compression(work_env, file_path):
+    """Toggles MAR compression format between BZ2 and LZMA"""
+    cmd = sh.Command(os.path.join(work_env.workdir,
+                                  "change_mar_compression.pl"))
+    log.debug("Changing MAR compression of %s", file_path)
+    out = cmd("-r", file_path, _env=work_env.env, _timeout=240,
+              _err_to_out=True)
+    if out:
+        log.debug(out)
+
+
+def unpack(work_env, mar, dest_dir):
+    os.mkdir(dest_dir)
+    unwrap_cmd = sh.Command(os.path.join(work_env.workdir,
+                                         "unwrap_full_update.pl"))
+    log.debug("Unwrapping %s", mar)
+    env = work_env.env
+    if not is_lzma_compressed_mar(mar):
+        env['MAR_OLD_FORMAT'] = '1'
+    elif 'MAR_OLD_FORMAT' in env:
+        del env['MAR_OLD_FORMAT']
+    out = unwrap_cmd(mar, _cwd=dest_dir, _env=env, _timeout=240,
+                     _err_to_out=True)
+    if out:
+        log.debug(out)
+
+
+def get_hash(path, hash_type="sha512"):
+    h = hashlib.new(hash_type)
+    with open(path, "rb") as f:
+        for chunk in iter(functools.partial(f.read, 4096), ''):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+class WorkEnv(object):
+
+    def __init__(self):
+        self.workdir = tempfile.mkdtemp()
+
+    def setup(self):
+        self.download_scripts()
+        self.download_martools()
+
+    def download_scripts(self):
+        # unwrap_full_update.pl is not too sensitive to the revision
+        prefix = "https://hg.mozilla.org/mozilla-central/raw-file/default/" \
+            "tools/update-packaging"
+        for f in ("unwrap_full_update.pl", "change_mar_compression.pl"):
+            url = "{prefix}/{f}".format(prefix=prefix, f=f)
+            download(url, dest=os.path.join(self.workdir, f), mode=0o755)
+
+    def download_martools(self):
+        url = "https://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/" \
+            "latest-mozilla-central/mar-tools/linux64/mar"
+        download(url, dest=os.path.join(self.workdir, "mar"), mode=0o755)
+
+    def cleanup(self):
+        shutil.rmtree(self.workdir)
+
+    @property
+    def env(self):
+        my_env = os.environ.copy()
+        my_env['LC_ALL'] = 'C'
+        my_env['MAR'] = os.path.join(self.workdir, "mar")
+        return my_env
+
+
+def verify_allowed_url(mar):
+    if not any(mar.startswith(prefix) for prefix in ALLOWED_URL_PREFIXES):
+        raise ValueError("{mar} is not in allowed URL prefixes: {p}".format(
+            mar=mar, p=ALLOWED_URL_PREFIXES
+        ))
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--artifacts-dir", required=True)
+    parser.add_argument("--sha1-signing-cert", required=True)
+    parser.add_argument("--sha384-signing-cert", required=True)
+    parser.add_argument("--task-definition", required=True,
+                        type=argparse.FileType('r'))
+    parser.add_argument("--output-filename", required=True)
+    parser.add_argument("-q", "--quiet", dest="log_level",
+                        action="store_const", const=logging.WARNING,
+                        default=logging.DEBUG)
+    args = parser.parse_args()
+
+    logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s",
+                        level=args.log_level)
+    task = json.load(args.task_definition)
+
+    signing_certs = {
+        'sha1': open(args.sha1_signing_cert, 'rb').read(),
+        'sha384': open(args.sha384_signing_cert, 'rb').read(),
+    }
+
+    assert(get_keysize(signing_certs['sha1']) == 2048)
+    assert(get_keysize(signing_certs['sha384']) == 4096)
+
+    manifest = []
+    for e in task["extra"]["funsize"]["completes"]:
+        to_mar = e["to_mar"]
+        locale = e["locale"]
+        output_filename = args.output_filename.format(locale=locale)
+        verify_allowed_url(to_mar)
+
+        work_env = WorkEnv()
+        work_env.setup()
+        complete_mars = {}
+        dest = os.path.join(work_env.workdir, "to.mar")
+        unpack_dir = os.path.join(work_env.workdir, "to")
+        download(to_mar, dest)
+        if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION"):
+            verify_signature(dest, signing_certs)
+        # Changing the compression strips the signature
+        change_mar_compression(work_env, dest)
+        complete_mars["hash"] = get_hash(dest)
+        unpack(work_env, dest, unpack_dir)
+        log.info("AV-scanning %s ...", unpack_dir)
+        sh.clamscan("-r", unpack_dir, _timeout=600, _err_to_out=True)
+        log.info("Done.")
+
+        mar_data = {
+            "file_to_sign": output_filename,
+            "hash": get_hash(dest),
+        }
+        shutil.copy(dest, os.path.join(args.artifacts_dir, output_filename))
+        work_env.cleanup()
+        manifest.append(mar_data)
+    manifest_file = os.path.join(args.artifacts_dir, "manifest.json")
+    with open(manifest_file, "w") as fp:
+        json.dump(manifest, fp, indent=2, sort_keys=True)
+
+
+if __name__ == '__main__':
+    main()
--- a/testing/mozharness/configs/beetmover/recompressed_completes.yml.tmpl
+++ b/testing/mozharness/configs/beetmover/recompressed_completes.yml.tmpl
@ -0,0 +1,16 @@
+---
+metadata:
+    name: "Beet Mover Manifest"
+    description: "Maps artifact locations to s3 key names for recompressed completes"
+    owner: "release@mozilla.com"
+
+mapping:
+{% for locale in locales %}
+  {{ locale }}:
+    complete_mar:
+      artifact: {{ artifact_base_url }}/firefox-{{ version }}.{{ locale }}.{{ platform }}.bz2.complete.mar
+      s3_key: {{ s3_prefix }}update/{{ platform }}/{{ locale }}/firefox-{{ version }}.bz2.complete.mar
+    complete_mar_sig:
+      artifact: {{ artifact_base_url }}/firefox-{{ version }}.{{ locale }}.{{ platform }}.bz2.complete.mar.asc
+      s3_key: {{ s3_prefix }}update/{{ platform }}/{{ locale }}/firefox-{{ version }}.bz2.complete.mar.asc
+{% endfor %}
--- a/testing/mozharness/configs/releases/bouncer_firefox_beta.py
+++ b/testing/mozharness/configs/releases/bouncer_firefox_beta.py
@ -105,6 +105,35 @@ config = {
                },
            },
        },
+        "complete-mar-bz2": {
+            "product-name": "Firefox-%(version)s-Complete-bz2",
+            "check_uptake": True,
+            "ssl-only": False,
+            "add-locales": True,
+            "paths": {
+                "linux": {
+                    "path": "/firefox/releases/%(version)s/update/linux-i686/:lang/firefox-%(version)s.bz2.complete.mar",
+                    "bouncer-platform": "linux",
+                },
+                "linux64": {
+                    "path": "/firefox/releases/%(version)s/update/linux-x86_64/:lang/firefox-%(version)s.bz2.complete.mar",
+                    "bouncer-platform": "linux64",
+                },
+                "macosx64": {
+                    "path": "/firefox/releases/%(version)s/update/mac/:lang/firefox-%(version)s.bz2.complete.mar",
+                    "bouncer-platform": "osx",
+                },
+                "win32": {
+                    "path": "/firefox/releases/%(version)s/update/win32/:lang/firefox-%(version)s.bz2.complete.mar",
+                    "bouncer-platform": "win",
+                },
+                "win64": {
+                    "path": "/firefox/releases/%(version)s/update/win64/:lang/firefox-%(version)s.bz2.complete.mar",
+                    "bouncer-platform": "win64",
+                },
+            },
+        },
+    },
    },
    "partials": {
        "releases-dir": {
--- a/testing/mozharness/configs/releases/bouncer_firefox_devedition.py
+++ b/testing/mozharness/configs/releases/bouncer_firefox_devedition.py
@ -105,6 +105,34 @@ config = {
                },
            },
        },
+        "complete-mar-bz2": {
+            "product-name": "Devedition-%(version)s-Complete-bz2",
+            "check_uptake": True,
+            "ssl-only": False,
+            "add-locales": True,
+            "paths": {
+                "linux": {
+                    "path": "/devedition/releases/%(version)s/update/linux-i686/:lang/firefox-%(version)s.bz2.complete.mar",
+                    "bouncer-platform": "linux",
+                },
+                "linux64": {
+                    "path": "/devedition/releases/%(version)s/update/linux-x86_64/:lang/firefox-%(version)s.bz2.complete.mar",
+                    "bouncer-platform": "linux64",
+                },
+                "macosx64": {
+                    "path": "/devedition/releases/%(version)s/update/mac/:lang/firefox-%(version)s.bz2.complete.mar",
+                    "bouncer-platform": "osx",
+                },
+                "win32": {
+                    "path": "/devedition/releases/%(version)s/update/win32/:lang/firefox-%(version)s.bz2.complete.mar",
+                    "bouncer-platform": "win",
+                },
+                "win64": {
+                    "path": "/devedition/releases/%(version)s/update/win64/:lang/firefox-%(version)s.bz2.complete.mar",
+                    "bouncer-platform": "win64",
+                },
+            },
+        },
    },
    "partials": {
        "releases-dir": {
--- a/testing/mozharness/configs/releases/bouncer_firefox_esr.py
+++ b/testing/mozharness/configs/releases/bouncer_firefox_esr.py
@ -102,6 +102,34 @@ config = {
                },
            },
        },
+        "complete-mar-bz2": {
+            "product-name": "Firefox-%(version)s-Complete-bz2",
+            "check_uptake": True,
+            "ssl-only": False,
+            "add-locales": True,
+            "paths": {
+                "linux": {
+                    "path": "/firefox/releases/%(version)s/update/linux-i686/:lang/firefox-%(version)s.bz2.complete.mar",
+                    "bouncer-platform": "linux",
+                },
+                "linux64": {
+                    "path": "/firefox/releases/%(version)s/update/linux-x86_64/:lang/firefox-%(version)s.bz2.complete.mar",
+                    "bouncer-platform": "linux64",
+                },
+                "macosx64": {
+                    "path": "/firefox/releases/%(version)s/update/mac/:lang/firefox-%(version)s.bz2.complete.mar",
+                    "bouncer-platform": "osx",
+                },
+                "win32": {
+                    "path": "/firefox/releases/%(version)s/update/win32/:lang/firefox-%(version)s.bz2.complete.mar",
+                    "bouncer-platform": "win",
+                },
+                "win64": {
+                    "path": "/firefox/releases/%(version)s/update/win64/:lang/firefox-%(version)s.bz2.complete.mar",
+                    "bouncer-platform": "win64",
+                },
+            },
+        },
    },
    "partials": {
        "releases-dir": {
--- a/testing/mozharness/configs/releases/bouncer_firefox_release.py
+++ b/testing/mozharness/configs/releases/bouncer_firefox_release.py
@ -133,6 +133,61 @@ config = {
                },
            },
        },
+        "complete-mar-bz2": {
+            "product-name": "Firefox-%(version)s-Complete-bz2",
+            "check_uptake": True,
+            "ssl-only": False,
+            "add-locales": True,
+            "paths": {
+                "linux": {
+                    "path": "/firefox/releases/%(version)s/update/linux-i686/:lang/firefox-%(version)s.bz2.complete.mar",
+                    "bouncer-platform": "linux",
+                },
+                "linux64": {
+                    "path": "/firefox/releases/%(version)s/update/linux-x86_64/:lang/firefox-%(version)s.bz2.complete.mar",
+                    "bouncer-platform": "linux64",
+                },
+                "macosx64": {
+                    "path": "/firefox/releases/%(version)s/update/mac/:lang/firefox-%(version)s.bz2.complete.mar",
+                    "bouncer-platform": "osx",
+                },
+                "win32": {
+                    "path": "/firefox/releases/%(version)s/update/win32/:lang/firefox-%(version)s.bz2.complete.mar",
+                    "bouncer-platform": "win",
+                },
+                "win64": {
+                    "path": "/firefox/releases/%(version)s/update/win64/:lang/firefox-%(version)s.bz2.complete.mar",
+                    "bouncer-platform": "win64",
+                },
+            },
+        },
+        "complete-mar-candidates-bz2": {
+            "product-name": "Firefox-%(version)sbuild%(build_number)s-Complete-bz2",
+            "check_uptake": False,
+            "ssl-only": False,
+            "add-locales": True,
+            "paths": {
+                "linux": {
+                    "path": "/firefox/candidates/%(version)s-candidates/build%(build_number)s/update/linux-i686/:lang/firefox-%(version)s.bz2.complete.mar",
+                    "bouncer-platform": "linux",
+                },
+                "linux64": {
+                    "path": "/firefox/candidates/%(version)s-candidates/build%(build_number)s/update/linux-x86_64/:lang/firefox-%(version)s.bz2.complete.mar",
+                    "bouncer-platform": "linux64",
+                },
+                "macosx64": {
+                    "path": "/firefox/candidates/%(version)s-candidates/build%(build_number)s/update/mac/:lang/firefox-%(version)s.bz2.complete.mar",
+                    "bouncer-platform": "osx",
+                },
+                "win32": {
+                    "path": "/firefox/candidates/%(version)s-candidates/build%(build_number)s/update/win32/:lang/firefox-%(version)s.bz2.complete.mar",
+                    "bouncer-platform": "win",
+                },
+                "win64": {
+                    "path": "/firefox/candidates/%(version)s-candidates/build%(build_number)s/update/win64/:lang/firefox-%(version)s.bz2.complete.mar",
+                    "bouncer-platform": "win64",
+                },
+            },
    },
    "partials": {
        "releases-dir": {