bug 1385946 Update funsize to enable both sha1 and sha384 signing formats a=catlee DONTBUILD

2017-08-01 14:59:25 +01:00 · 2017-08-01 14:59:25 +01:00 · be0a8a2ff2
--- a/taskcluster/docker/funsize-balrog-submitter/nightly_sha1.pubkey
+++ b/taskcluster/docker/funsize-balrog-submitter/nightly_sha1.pubkey
--- a/taskcluster/docker/funsize-balrog-submitter/nightly_sha384.pubkey
+++ b/taskcluster/docker/funsize-balrog-submitter/nightly_sha384.pubkey
@ -0,0 +1,14 @@
+-----BEGIN PUBLIC KEY-----
+MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAth151NGY8PBzn0bii9Yc
+AjYHZDwP9Lj1c3owG0zLqW2kPcdp86QTAcoYunHGYFFakNG3tooZhzwkMjZ1OrXc
+ERjD6AuVSGIBdsKtKP4vLtMjDUteFN4K2+rveozcnYFZuTWEajGu8uoYsv4QgdEA
+nTBC39j0J33xlfUR+XKuxzhxNrFX+fRFWuLDJrPziMcVA/mzf0gXlhtEsfV0HYyg
+yWpHdIWww+llysD1QOQAHk94Ss8c/4BFXFxlwlLeNlB1ZqLm1LsNy0jUy9EHeO3C
+H6eqmiFEbpdjlrkJdgR1NcTzeY/Qf/nhWH6BAZrSapQycF7OSLU+rFWMQUElSPLc
+NVl7oNAAfSYLTvRjPGi+mJK3wGFQw1EpwQl+elE1oj4+sHvIVpDrLb6btpxfr1cZ
+pR4Di/hkOIymxEDWvtUhOxUXnYbDKQSDcAHKM/xR3sdIAiVtVuL4hyBwlAqkQc2j
+H+SmnCbazgnq5+dN4y5DjoOgbZQ/koE3s3bUzzMeIxaul9v4gMtGROw3PQ3OZcP0
+lgjPRhY+NeTnWMo2nGb4/eS6Cn2qFLfbEQjsj6pJJBNKfvK/gm1jXb3PgXXdf8+d
+2xTPOX8QNpSK7C0w4vYlvSpYZlsx2cznEOV6LDqP0QHUnmd/k1xWRRGiQ7gtT+BV
+Fn0h7JyTGmEdFu6l4OhS8hMCAwEAAQ==
+-----END PUBLIC KEY-----
--- a/taskcluster/docker/funsize-balrog-submitter/release_sha1.pubkey
+++ b/taskcluster/docker/funsize-balrog-submitter/release_sha1.pubkey
--- a/taskcluster/docker/funsize-balrog-submitter/release_sha384.pubkey
+++ b/taskcluster/docker/funsize-balrog-submitter/release_sha384.pubkey
@ -0,0 +1,14 @@
+-----BEGIN PUBLIC KEY-----
+MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxCHbY+fP3dvaP9XVbmK6
+i4rbqo72INEWgDSYbr/DIYfCSzHC9H8pU8dyjt+Nd8OtoUZtBD1N9fP7SlrvPZSI
+ZSW4k0e9Ky5aV3Uy+ivamSvYszkhqdeP2y7MBu73XHKYONR9PnKa+ovmREwSEI+h
+1e0ebm8zvF7Ndwx0mOeZkDu9SDkDGg4aj2xrJyBBOuGVjuctMZ6l1davANI5xiJ0
+GBEU3tR1gJs1T4vLBis5mEFn9y4kgyw/HrxmRYGnZL4fLb2fTI+pNW0Twu3KWwwi
+LgLkkVrNWiHSk7YWqxjcg5IA3pQETQ17paTHoB5Mnkvuh6MkDXvRG5VgAHZAigr6
+fJMsasOUaBeos/cD1LDQEIObpetlxc0Fiu/lvUts0755otkhI+yv35+wUa6GJrsE
+CsT7c/LaFtQXg06aGXbMLDn0bE/e+nw9KWT/rE1iYXMFkzrqoTeYJ+v7/fD/ywU8
+m8l4CZmXxzd/RogMrM3xl+j4ucAAltDQyL4yLySaIT05w5U8z2zJDEXFvpFDSRfF
+K3kjLwGub7wNwaQDuh/msIUdavu4g+GNikCXAJ8AssLuYatyHoltd2tf+EIIDW3U
+zzLpymnLo3cAz3IPfXyqVB+mcLcpqbHjl3hWms6l1wGtz6S4WqdrWs/KfzS5EyDK
+r63xn1Rg/XFmR57EsFEXAZ8CAwEAAQ==
+-----END PUBLIC KEY-----
--- a/taskcluster/docker/funsize-balrog-submitter/requirements.txt
+++ b/taskcluster/docker/funsize-balrog-submitter/requirements.txt
@ -1 +1 @@
-mar==1.2
+mar==2.1.1
--- a/taskcluster/docker/funsize-balrog-submitter/runme.sh
+++ b/taskcluster/docker/funsize-balrog-submitter/runme.sh
@ -4,7 +4,9 @@ set -xe

 test $PARENT_TASK_ARTIFACTS_URL_PREFIX
 test $BALROG_API_ROOT
-test $SIGNING_CERT
+test $SHA1_SIGNING_CERT
+test $SHA384_SIGNING_CERT
+

 ARTIFACTS_DIR="/home/worker/artifacts"
 mkdir -p "$ARTIFACTS_DIR"
@ -17,6 +19,7 @@ python /home/worker/bin/funsize-balrog-submitter.py \
    --artifacts-url-prefix "$PARENT_TASK_ARTIFACTS_URL_PREFIX" \
    --manifest "$ARTIFACTS_DIR/manifest.json" \
    -a "$BALROG_API_ROOT" \
-    --signing-cert "/home/worker/keys/${SIGNING_CERT}.pubkey" \
+    --sha1-signing-cert "/home/worker/keys/${SHA1_SIGNING_CERT}.pubkey" \
+    --sha384-signing-cert "/home/worker/keys/${SHA384_SIGNING_CERT}.pubkey" \
    --verbose \
    $EXTRA_BALROG_SUBMITTER_PARAMS
--- a/taskcluster/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py
+++ b/taskcluster/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py
@ -8,7 +8,8 @@ import hashlib
 import requests
 import tempfile
 from boto.s3.connection import S3Connection
-from mardor.marfile import MarFile
+from mardor.reader import MarReader
+from mardor.signing import get_keysize

 site.addsitedir("/home/worker/tools/lib/python")

@ -47,14 +48,15 @@ def download(url, dest, mode=None):
        os.chmod(dest, mode)


-def verify_signature(mar, signature):
+def verify_signature(mar, certs):
    log.info("Checking %s signature", mar)
-    m = MarFile(mar, signature_versions=[(1, signature)])
-    m.verify_signatures()
+    with open(mar, 'rb') as mar_fh:
+        m = MarReader(mar_fh)
+        m.verify(verify_key=certs.get(m.signature_type()))


 def verify_copy_to_s3(bucket_name, aws_access_key_id, aws_secret_access_key,
-                      mar_url, mar_dest, signing_cert):
+                      mar_url, mar_dest, signing_certs):
    conn = S3Connection(aws_access_key_id, aws_secret_access_key)
    bucket = conn.get_bucket(bucket_name)
    _, dest = tempfile.mkstemp()
@ -62,7 +64,7 @@ def verify_copy_to_s3(bucket_name, aws_access_key_id, aws_secret_access_key,
    download(mar_url, dest)
    log.info("Verifying the signature...")
    if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION"):
-        verify_signature(dest, signing_cert)
+        verify_signature(dest, signing_certs)
    for name in possible_names(mar_dest, 10):
        log.info("Checking if %s already exists", name)
        key = bucket.get_key(name)
@ -112,7 +114,8 @@ def main():
                        help="Balrog API root")
    parser.add_argument("-d", "--dummy", action="store_true",
                        help="Add '-dummy' suffix to branch name")
-    parser.add_argument("--signing-cert", required=True)
+    parser.add_argument("--sha1-signing-cert", required=True)
+    parser.add_argument("--sha384-signing-cert", required=True)
    parser.add_argument("-v", "--verbose", action="store_const",
                        dest="loglevel", const=logging.DEBUG,
                        default=logging.INFO)
@ -141,6 +144,14 @@ def main():
    manifest = json.load(open(args.manifest))
    auth = (balrog_username, balrog_password)

+    signing_certs = {
+        'sha1': args.sha1_signing_cert,
+        'sha384': args.sha384_signing_cert,
+    }
+
+    assert(get_keysize(open(signing_certs['sha1'], 'rb').read()) == 2048)
+    assert(get_keysize(open(signing_certs['sha384'], 'rb').read()) == 4096)
+
    for e in manifest:
        complete_info = [{
            "hash": e["to_hash"],
@ -186,10 +197,10 @@ def main():
                                               complete_mar_filename)
            partial_info[0]["url"] = verify_copy_to_s3(
                s3_bucket, aws_access_key_id, aws_secret_access_key,
-                partial_mar_url, partial_mar_dest, args.signing_cert)
+                partial_mar_url, partial_mar_dest, signing_certs)
            complete_info[0]["url"] = verify_copy_to_s3(
                s3_bucket, aws_access_key_id, aws_secret_access_key,
-                complete_mar_url, complete_mar_dest, args.signing_cert)
+                complete_mar_url, complete_mar_dest, signing_certs)
            partial_info[0]["from_buildid"] = e["from_buildid"]
            submitter = NightlySubmitterV4(api_root=args.api_root, auth=auth,
                                           dummy=args.dummy)
--- a/taskcluster/docker/funsize-update-generator/nightly_sha1.pubkey
+++ b/taskcluster/docker/funsize-update-generator/nightly_sha1.pubkey
--- a/taskcluster/docker/funsize-update-generator/nightly_sha384.pubkey
+++ b/taskcluster/docker/funsize-update-generator/nightly_sha384.pubkey
@ -0,0 +1,14 @@
+-----BEGIN PUBLIC KEY-----
+MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAth151NGY8PBzn0bii9Yc
+AjYHZDwP9Lj1c3owG0zLqW2kPcdp86QTAcoYunHGYFFakNG3tooZhzwkMjZ1OrXc
+ERjD6AuVSGIBdsKtKP4vLtMjDUteFN4K2+rveozcnYFZuTWEajGu8uoYsv4QgdEA
+nTBC39j0J33xlfUR+XKuxzhxNrFX+fRFWuLDJrPziMcVA/mzf0gXlhtEsfV0HYyg
+yWpHdIWww+llysD1QOQAHk94Ss8c/4BFXFxlwlLeNlB1ZqLm1LsNy0jUy9EHeO3C
+H6eqmiFEbpdjlrkJdgR1NcTzeY/Qf/nhWH6BAZrSapQycF7OSLU+rFWMQUElSPLc
+NVl7oNAAfSYLTvRjPGi+mJK3wGFQw1EpwQl+elE1oj4+sHvIVpDrLb6btpxfr1cZ
+pR4Di/hkOIymxEDWvtUhOxUXnYbDKQSDcAHKM/xR3sdIAiVtVuL4hyBwlAqkQc2j
+H+SmnCbazgnq5+dN4y5DjoOgbZQ/koE3s3bUzzMeIxaul9v4gMtGROw3PQ3OZcP0
+lgjPRhY+NeTnWMo2nGb4/eS6Cn2qFLfbEQjsj6pJJBNKfvK/gm1jXb3PgXXdf8+d
+2xTPOX8QNpSK7C0w4vYlvSpYZlsx2cznEOV6LDqP0QHUnmd/k1xWRRGiQ7gtT+BV
+Fn0h7JyTGmEdFu6l4OhS8hMCAwEAAQ==
+-----END PUBLIC KEY-----
--- a/taskcluster/docker/funsize-update-generator/release_sha1.pubkey
+++ b/taskcluster/docker/funsize-update-generator/release_sha1.pubkey
--- a/taskcluster/docker/funsize-update-generator/release_sha384.pubkey
+++ b/taskcluster/docker/funsize-update-generator/release_sha384.pubkey
@ -0,0 +1,14 @@
+-----BEGIN PUBLIC KEY-----
+MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxCHbY+fP3dvaP9XVbmK6
+i4rbqo72INEWgDSYbr/DIYfCSzHC9H8pU8dyjt+Nd8OtoUZtBD1N9fP7SlrvPZSI
+ZSW4k0e9Ky5aV3Uy+ivamSvYszkhqdeP2y7MBu73XHKYONR9PnKa+ovmREwSEI+h
+1e0ebm8zvF7Ndwx0mOeZkDu9SDkDGg4aj2xrJyBBOuGVjuctMZ6l1davANI5xiJ0
+GBEU3tR1gJs1T4vLBis5mEFn9y4kgyw/HrxmRYGnZL4fLb2fTI+pNW0Twu3KWwwi
+LgLkkVrNWiHSk7YWqxjcg5IA3pQETQ17paTHoB5Mnkvuh6MkDXvRG5VgAHZAigr6
+fJMsasOUaBeos/cD1LDQEIObpetlxc0Fiu/lvUts0755otkhI+yv35+wUa6GJrsE
+CsT7c/LaFtQXg06aGXbMLDn0bE/e+nw9KWT/rE1iYXMFkzrqoTeYJ+v7/fD/ywU8
+m8l4CZmXxzd/RogMrM3xl+j4ucAAltDQyL4yLySaIT05w5U8z2zJDEXFvpFDSRfF
+K3kjLwGub7wNwaQDuh/msIUdavu4g+GNikCXAJ8AssLuYatyHoltd2tf+EIIDW3U
+zzLpymnLo3cAz3IPfXyqVB+mcLcpqbHjl3hWms6l1wGtz6S4WqdrWs/KfzS5EyDK
+r63xn1Rg/XFmR57EsFEXAZ8CAwEAAQ==
+-----END PUBLIC KEY-----
--- a/taskcluster/docker/funsize-update-generator/requirements.txt
+++ b/taskcluster/docker/funsize-update-generator/requirements.txt
@ -1,2 +1,2 @@
-mar==1.2
+mar==2.1.1
 redo
--- a/taskcluster/docker/funsize-update-generator/runme.sh
+++ b/taskcluster/docker/funsize-update-generator/runme.sh
@ -3,7 +3,8 @@
 set -xe

 test $TASK_ID
-test $SIGNING_CERT
+test $SHA1_SIGNING_CERT
+test $SHA384_SIGNING_CERT

 ARTIFACTS_DIR="/home/worker/artifacts"
 mkdir -p "$ARTIFACTS_DIR"
@ -21,5 +22,6 @@ fi
 /home/worker/bin/funsize.py \
    --artifacts-dir "$ARTIFACTS_DIR" \
    --task-definition /home/worker/task.json \
-    --signing-cert "/home/worker/keys/${SIGNING_CERT}.pubkey" \
+    --sha1-signing-cert "/home/worker/keys/${SHA1_SIGNING_CERT}.pubkey" \
+    --sha384-signing-cert "/home/worker/keys/${SHA384_SIGNING_CERT}.pubkey" \
    $EXTRA_PARAMS
--- a/taskcluster/docker/funsize-update-generator/scripts/funsize.py
+++ b/taskcluster/docker/funsize-update-generator/scripts/funsize.py
@ -13,7 +13,9 @@ import requests
 import sh

 import redo
-from mardor.marfile import MarFile
+from mardor.reader import MarReader
+from mardor.signing import get_keysize
+

 log = logging.getLogger(__name__)
 ALLOWED_URL_PREFIXES = [
@ -30,10 +32,16 @@ DEFAULT_FILENAME_TEMPLATE = "{appName}-{branch}-{version}-{platform}-" \
                            "{locale}-{from_buildid}-{to_buildid}.partial.mar"


-def verify_signature(mar, signature):
+def verify_signature(mar, certs):
    log.info("Checking %s signature", mar)
-    m = MarFile(mar, signature_versions=[(1, signature)])
-    m.verify_signatures()
+    with open(mar, 'rb') as mar_fh:
+        m = MarReader(mar_fh)
+        m.verify(verify_key=certs.get(m.signature_type()))
+
+
+def is_lzma_compressed_mar(mar):
+    log.info("Checking %s for lzma compression", mar)
+    return MarReader(open(mar, 'rb')).compression_type() == 'xz'


@redo.retriable()
@ -64,7 +72,12 @@ def unpack(work_env, mar, dest_dir):
    unwrap_cmd = sh.Command(os.path.join(work_env.workdir,
                                         "unwrap_full_update.pl"))
    log.debug("Unwrapping %s", mar)
-    out = unwrap_cmd(mar, _cwd=dest_dir, _env=work_env.env, _timeout=240,
+    env = work_env.env
+    if not is_lzma_compressed_mar(mar):
+        env['MAR_OLD_FORMAT'] = 1
+    elif 'MAR_OLD_FORMAT' in env:
+        del env['MAR_OLD_FORMAT']
+    out = unwrap_cmd(mar, _cwd=dest_dir, _env=env, _timeout=240,
                     _err_to_out=True)
    if out:
        log.debug(out)
@ -96,6 +109,10 @@ def generate_partial(work_env, from_dir, to_dir, dest_mar, channel_ids,
    env = work_env.env
    env["MOZ_PRODUCT_VERSION"] = version
    env["MOZ_CHANNEL_ID"] = channel_ids
+    if not is_lzma_compressed_mar(dest_mar):
+        env['MAR_OLD_FORMAT'] = 1
+    elif 'MAR_OLD_FORMAT' in env:
+        del env['MAR_OLD_FORMAT']
    make_incremental_update = os.path.join(work_env.workdir,
                                           "make_incremental_update.sh")
    out = sh.bash(make_incremental_update, dest_mar, from_dir, to_dir,
@ -166,7 +183,8 @@ def verify_allowed_url(mar):
 def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--artifacts-dir", required=True)
-    parser.add_argument("--signing-cert", required=True)
+    parser.add_argument("--sha1-signing-cert", required=True)
+    parser.add_argument("--sha384-signing-cert", required=True)
    parser.add_argument("--task-definition", required=True,
                        type=argparse.FileType('r'))
    parser.add_argument("--filename-template",
@ -183,6 +201,14 @@ def main():
    task = json.load(args.task_definition)
    # TODO: verify task["extra"]["funsize"]["partials"] with jsonschema

+    signing_certs = {
+        'sha1': args.sha1_signing_cert,
+        'sha384': args.sha384_signing_cert,
+    }
+
+    assert(get_keysize(open(signing_certs['sha1'], 'rb').read()) == 2048)
+    assert(get_keysize(open(signing_certs['sha384'], 'rb').read()) == 4096)
+
    if args.no_freshclam:
        log.info("Skipping freshclam")
    else:
@ -207,7 +233,7 @@ def main():
            unpack_dir = os.path.join(work_env.workdir, mar_type)
            download(f, dest)
            if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION"):
-                verify_signature(dest, args.signing_cert)
+                verify_signature(dest, signing_certs)
            complete_mars["%s_size" % mar_type] = os.path.getsize(dest)
            complete_mars["%s_hash" % mar_type] = get_hash(dest)
            unpack(work_env, dest, unpack_dir)