diff --git a/taskcluster/docker/funsize-balrog-submitter/nightly.pubkey b/taskcluster/docker/funsize-balrog-submitter/nightly_sha1.pubkey similarity index 100% rename from taskcluster/docker/funsize-balrog-submitter/nightly.pubkey rename to taskcluster/docker/funsize-balrog-submitter/nightly_sha1.pubkey diff --git a/taskcluster/docker/funsize-balrog-submitter/nightly_sha384.pubkey b/taskcluster/docker/funsize-balrog-submitter/nightly_sha384.pubkey new file mode 100644 index 000000000000..e51049844c1d --- /dev/null +++ b/taskcluster/docker/funsize-balrog-submitter/nightly_sha384.pubkey @@ -0,0 +1,14 @@ +-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAth151NGY8PBzn0bii9Yc +AjYHZDwP9Lj1c3owG0zLqW2kPcdp86QTAcoYunHGYFFakNG3tooZhzwkMjZ1OrXc +ERjD6AuVSGIBdsKtKP4vLtMjDUteFN4K2+rveozcnYFZuTWEajGu8uoYsv4QgdEA +nTBC39j0J33xlfUR+XKuxzhxNrFX+fRFWuLDJrPziMcVA/mzf0gXlhtEsfV0HYyg +yWpHdIWww+llysD1QOQAHk94Ss8c/4BFXFxlwlLeNlB1ZqLm1LsNy0jUy9EHeO3C +H6eqmiFEbpdjlrkJdgR1NcTzeY/Qf/nhWH6BAZrSapQycF7OSLU+rFWMQUElSPLc +NVl7oNAAfSYLTvRjPGi+mJK3wGFQw1EpwQl+elE1oj4+sHvIVpDrLb6btpxfr1cZ +pR4Di/hkOIymxEDWvtUhOxUXnYbDKQSDcAHKM/xR3sdIAiVtVuL4hyBwlAqkQc2j +H+SmnCbazgnq5+dN4y5DjoOgbZQ/koE3s3bUzzMeIxaul9v4gMtGROw3PQ3OZcP0 +lgjPRhY+NeTnWMo2nGb4/eS6Cn2qFLfbEQjsj6pJJBNKfvK/gm1jXb3PgXXdf8+d +2xTPOX8QNpSK7C0w4vYlvSpYZlsx2cznEOV6LDqP0QHUnmd/k1xWRRGiQ7gtT+BV +Fn0h7JyTGmEdFu6l4OhS8hMCAwEAAQ== +-----END PUBLIC KEY----- diff --git a/taskcluster/docker/funsize-balrog-submitter/release.pubkey b/taskcluster/docker/funsize-balrog-submitter/release_sha1.pubkey similarity index 100% rename from taskcluster/docker/funsize-balrog-submitter/release.pubkey rename to taskcluster/docker/funsize-balrog-submitter/release_sha1.pubkey diff --git a/taskcluster/docker/funsize-balrog-submitter/release_sha384.pubkey b/taskcluster/docker/funsize-balrog-submitter/release_sha384.pubkey new file mode 100644 index 000000000000..ec1103d82847 --- /dev/null +++ b/taskcluster/docker/funsize-balrog-submitter/release_sha384.pubkey @@ -0,0 +1,14 @@ +-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxCHbY+fP3dvaP9XVbmK6 +i4rbqo72INEWgDSYbr/DIYfCSzHC9H8pU8dyjt+Nd8OtoUZtBD1N9fP7SlrvPZSI +ZSW4k0e9Ky5aV3Uy+ivamSvYszkhqdeP2y7MBu73XHKYONR9PnKa+ovmREwSEI+h +1e0ebm8zvF7Ndwx0mOeZkDu9SDkDGg4aj2xrJyBBOuGVjuctMZ6l1davANI5xiJ0 +GBEU3tR1gJs1T4vLBis5mEFn9y4kgyw/HrxmRYGnZL4fLb2fTI+pNW0Twu3KWwwi +LgLkkVrNWiHSk7YWqxjcg5IA3pQETQ17paTHoB5Mnkvuh6MkDXvRG5VgAHZAigr6 +fJMsasOUaBeos/cD1LDQEIObpetlxc0Fiu/lvUts0755otkhI+yv35+wUa6GJrsE +CsT7c/LaFtQXg06aGXbMLDn0bE/e+nw9KWT/rE1iYXMFkzrqoTeYJ+v7/fD/ywU8 +m8l4CZmXxzd/RogMrM3xl+j4ucAAltDQyL4yLySaIT05w5U8z2zJDEXFvpFDSRfF +K3kjLwGub7wNwaQDuh/msIUdavu4g+GNikCXAJ8AssLuYatyHoltd2tf+EIIDW3U +zzLpymnLo3cAz3IPfXyqVB+mcLcpqbHjl3hWms6l1wGtz6S4WqdrWs/KfzS5EyDK +r63xn1Rg/XFmR57EsFEXAZ8CAwEAAQ== +-----END PUBLIC KEY----- diff --git a/taskcluster/docker/funsize-balrog-submitter/requirements.txt b/taskcluster/docker/funsize-balrog-submitter/requirements.txt index aec79b364420..12da8dc14bc7 100644 --- a/taskcluster/docker/funsize-balrog-submitter/requirements.txt +++ b/taskcluster/docker/funsize-balrog-submitter/requirements.txt @@ -1 +1 @@ -mar==1.2 +mar==2.1.1 diff --git a/taskcluster/docker/funsize-balrog-submitter/runme.sh b/taskcluster/docker/funsize-balrog-submitter/runme.sh index ecf22240335e..d8dec5786790 100644 --- a/taskcluster/docker/funsize-balrog-submitter/runme.sh +++ b/taskcluster/docker/funsize-balrog-submitter/runme.sh @@ -4,7 +4,9 @@ set -xe test $PARENT_TASK_ARTIFACTS_URL_PREFIX test $BALROG_API_ROOT -test $SIGNING_CERT +test $SHA1_SIGNING_CERT +test $SHA384_SIGNING_CERT + ARTIFACTS_DIR="/home/worker/artifacts" mkdir -p "$ARTIFACTS_DIR" @@ -17,6 +19,7 @@ python /home/worker/bin/funsize-balrog-submitter.py \ --artifacts-url-prefix "$PARENT_TASK_ARTIFACTS_URL_PREFIX" \ --manifest "$ARTIFACTS_DIR/manifest.json" \ -a "$BALROG_API_ROOT" \ - --signing-cert "/home/worker/keys/${SIGNING_CERT}.pubkey" \ + --sha1-signing-cert "/home/worker/keys/${SHA1_SIGNING_CERT}.pubkey" \ + --sha384-signing-cert "/home/worker/keys/${SHA384_SIGNING_CERT}.pubkey" \ --verbose \ $EXTRA_BALROG_SUBMITTER_PARAMS diff --git a/taskcluster/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py b/taskcluster/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py index b5cd45f7db9b..c60cf1d7cd17 100644 --- a/taskcluster/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py +++ b/taskcluster/docker/funsize-balrog-submitter/scripts/funsize-balrog-submitter.py @@ -8,7 +8,8 @@ import hashlib import requests import tempfile from boto.s3.connection import S3Connection -from mardor.marfile import MarFile +from mardor.reader import MarReader +from mardor.signing import get_keysize site.addsitedir("/home/worker/tools/lib/python") @@ -47,14 +48,15 @@ def download(url, dest, mode=None): os.chmod(dest, mode) -def verify_signature(mar, signature): +def verify_signature(mar, certs): log.info("Checking %s signature", mar) - m = MarFile(mar, signature_versions=[(1, signature)]) - m.verify_signatures() + with open(mar, 'rb') as mar_fh: + m = MarReader(mar_fh) + m.verify(verify_key=certs.get(m.signature_type())) def verify_copy_to_s3(bucket_name, aws_access_key_id, aws_secret_access_key, - mar_url, mar_dest, signing_cert): + mar_url, mar_dest, signing_certs): conn = S3Connection(aws_access_key_id, aws_secret_access_key) bucket = conn.get_bucket(bucket_name) _, dest = tempfile.mkstemp() @@ -62,7 +64,7 @@ def verify_copy_to_s3(bucket_name, aws_access_key_id, aws_secret_access_key, download(mar_url, dest) log.info("Verifying the signature...") if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION"): - verify_signature(dest, signing_cert) + verify_signature(dest, signing_certs) for name in possible_names(mar_dest, 10): log.info("Checking if %s already exists", name) key = bucket.get_key(name) @@ -112,7 +114,8 @@ def main(): help="Balrog API root") parser.add_argument("-d", "--dummy", action="store_true", help="Add '-dummy' suffix to branch name") - parser.add_argument("--signing-cert", required=True) + parser.add_argument("--sha1-signing-cert", required=True) + parser.add_argument("--sha384-signing-cert", required=True) parser.add_argument("-v", "--verbose", action="store_const", dest="loglevel", const=logging.DEBUG, default=logging.INFO) @@ -141,6 +144,14 @@ def main(): manifest = json.load(open(args.manifest)) auth = (balrog_username, balrog_password) + signing_certs = { + 'sha1': args.sha1_signing_cert, + 'sha384': args.sha384_signing_cert, + } + + assert(get_keysize(open(signing_certs['sha1'], 'rb').read()) == 2048) + assert(get_keysize(open(signing_certs['sha384'], 'rb').read()) == 4096) + for e in manifest: complete_info = [{ "hash": e["to_hash"], @@ -186,10 +197,10 @@ def main(): complete_mar_filename) partial_info[0]["url"] = verify_copy_to_s3( s3_bucket, aws_access_key_id, aws_secret_access_key, - partial_mar_url, partial_mar_dest, args.signing_cert) + partial_mar_url, partial_mar_dest, signing_certs) complete_info[0]["url"] = verify_copy_to_s3( s3_bucket, aws_access_key_id, aws_secret_access_key, - complete_mar_url, complete_mar_dest, args.signing_cert) + complete_mar_url, complete_mar_dest, signing_certs) partial_info[0]["from_buildid"] = e["from_buildid"] submitter = NightlySubmitterV4(api_root=args.api_root, auth=auth, dummy=args.dummy) diff --git a/taskcluster/docker/funsize-update-generator/nightly.pubkey b/taskcluster/docker/funsize-update-generator/nightly_sha1.pubkey similarity index 100% rename from taskcluster/docker/funsize-update-generator/nightly.pubkey rename to taskcluster/docker/funsize-update-generator/nightly_sha1.pubkey diff --git a/taskcluster/docker/funsize-update-generator/nightly_sha384.pubkey b/taskcluster/docker/funsize-update-generator/nightly_sha384.pubkey new file mode 100644 index 000000000000..e51049844c1d --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/nightly_sha384.pubkey @@ -0,0 +1,14 @@ +-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAth151NGY8PBzn0bii9Yc +AjYHZDwP9Lj1c3owG0zLqW2kPcdp86QTAcoYunHGYFFakNG3tooZhzwkMjZ1OrXc +ERjD6AuVSGIBdsKtKP4vLtMjDUteFN4K2+rveozcnYFZuTWEajGu8uoYsv4QgdEA +nTBC39j0J33xlfUR+XKuxzhxNrFX+fRFWuLDJrPziMcVA/mzf0gXlhtEsfV0HYyg +yWpHdIWww+llysD1QOQAHk94Ss8c/4BFXFxlwlLeNlB1ZqLm1LsNy0jUy9EHeO3C +H6eqmiFEbpdjlrkJdgR1NcTzeY/Qf/nhWH6BAZrSapQycF7OSLU+rFWMQUElSPLc +NVl7oNAAfSYLTvRjPGi+mJK3wGFQw1EpwQl+elE1oj4+sHvIVpDrLb6btpxfr1cZ +pR4Di/hkOIymxEDWvtUhOxUXnYbDKQSDcAHKM/xR3sdIAiVtVuL4hyBwlAqkQc2j +H+SmnCbazgnq5+dN4y5DjoOgbZQ/koE3s3bUzzMeIxaul9v4gMtGROw3PQ3OZcP0 +lgjPRhY+NeTnWMo2nGb4/eS6Cn2qFLfbEQjsj6pJJBNKfvK/gm1jXb3PgXXdf8+d +2xTPOX8QNpSK7C0w4vYlvSpYZlsx2cznEOV6LDqP0QHUnmd/k1xWRRGiQ7gtT+BV +Fn0h7JyTGmEdFu6l4OhS8hMCAwEAAQ== +-----END PUBLIC KEY----- diff --git a/taskcluster/docker/funsize-update-generator/release.pubkey b/taskcluster/docker/funsize-update-generator/release_sha1.pubkey similarity index 100% rename from taskcluster/docker/funsize-update-generator/release.pubkey rename to taskcluster/docker/funsize-update-generator/release_sha1.pubkey diff --git a/taskcluster/docker/funsize-update-generator/release_sha384.pubkey b/taskcluster/docker/funsize-update-generator/release_sha384.pubkey new file mode 100644 index 000000000000..ec1103d82847 --- /dev/null +++ b/taskcluster/docker/funsize-update-generator/release_sha384.pubkey @@ -0,0 +1,14 @@ +-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxCHbY+fP3dvaP9XVbmK6 +i4rbqo72INEWgDSYbr/DIYfCSzHC9H8pU8dyjt+Nd8OtoUZtBD1N9fP7SlrvPZSI +ZSW4k0e9Ky5aV3Uy+ivamSvYszkhqdeP2y7MBu73XHKYONR9PnKa+ovmREwSEI+h +1e0ebm8zvF7Ndwx0mOeZkDu9SDkDGg4aj2xrJyBBOuGVjuctMZ6l1davANI5xiJ0 +GBEU3tR1gJs1T4vLBis5mEFn9y4kgyw/HrxmRYGnZL4fLb2fTI+pNW0Twu3KWwwi +LgLkkVrNWiHSk7YWqxjcg5IA3pQETQ17paTHoB5Mnkvuh6MkDXvRG5VgAHZAigr6 +fJMsasOUaBeos/cD1LDQEIObpetlxc0Fiu/lvUts0755otkhI+yv35+wUa6GJrsE +CsT7c/LaFtQXg06aGXbMLDn0bE/e+nw9KWT/rE1iYXMFkzrqoTeYJ+v7/fD/ywU8 +m8l4CZmXxzd/RogMrM3xl+j4ucAAltDQyL4yLySaIT05w5U8z2zJDEXFvpFDSRfF +K3kjLwGub7wNwaQDuh/msIUdavu4g+GNikCXAJ8AssLuYatyHoltd2tf+EIIDW3U +zzLpymnLo3cAz3IPfXyqVB+mcLcpqbHjl3hWms6l1wGtz6S4WqdrWs/KfzS5EyDK +r63xn1Rg/XFmR57EsFEXAZ8CAwEAAQ== +-----END PUBLIC KEY----- diff --git a/taskcluster/docker/funsize-update-generator/requirements.txt b/taskcluster/docker/funsize-update-generator/requirements.txt index 58a2d60b7018..986a415b29aa 100644 --- a/taskcluster/docker/funsize-update-generator/requirements.txt +++ b/taskcluster/docker/funsize-update-generator/requirements.txt @@ -1,2 +1,2 @@ -mar==1.2 +mar==2.1.1 redo diff --git a/taskcluster/docker/funsize-update-generator/runme.sh b/taskcluster/docker/funsize-update-generator/runme.sh index 92094a76e1f5..4d9785cc9a14 100644 --- a/taskcluster/docker/funsize-update-generator/runme.sh +++ b/taskcluster/docker/funsize-update-generator/runme.sh @@ -3,7 +3,8 @@ set -xe test $TASK_ID -test $SIGNING_CERT +test $SHA1_SIGNING_CERT +test $SHA384_SIGNING_CERT ARTIFACTS_DIR="/home/worker/artifacts" mkdir -p "$ARTIFACTS_DIR" @@ -21,5 +22,6 @@ fi /home/worker/bin/funsize.py \ --artifacts-dir "$ARTIFACTS_DIR" \ --task-definition /home/worker/task.json \ - --signing-cert "/home/worker/keys/${SIGNING_CERT}.pubkey" \ + --sha1-signing-cert "/home/worker/keys/${SHA1_SIGNING_CERT}.pubkey" \ + --sha384-signing-cert "/home/worker/keys/${SHA384_SIGNING_CERT}.pubkey" \ $EXTRA_PARAMS diff --git a/taskcluster/docker/funsize-update-generator/scripts/funsize.py b/taskcluster/docker/funsize-update-generator/scripts/funsize.py index fd591817c128..48f7bd1a8a69 100755 --- a/taskcluster/docker/funsize-update-generator/scripts/funsize.py +++ b/taskcluster/docker/funsize-update-generator/scripts/funsize.py @@ -13,7 +13,9 @@ import requests import sh import redo -from mardor.marfile import MarFile +from mardor.reader import MarReader +from mardor.signing import get_keysize + log = logging.getLogger(__name__) ALLOWED_URL_PREFIXES = [ @@ -30,10 +32,16 @@ DEFAULT_FILENAME_TEMPLATE = "{appName}-{branch}-{version}-{platform}-" \ "{locale}-{from_buildid}-{to_buildid}.partial.mar" -def verify_signature(mar, signature): +def verify_signature(mar, certs): log.info("Checking %s signature", mar) - m = MarFile(mar, signature_versions=[(1, signature)]) - m.verify_signatures() + with open(mar, 'rb') as mar_fh: + m = MarReader(mar_fh) + m.verify(verify_key=certs.get(m.signature_type())) + + +def is_lzma_compressed_mar(mar): + log.info("Checking %s for lzma compression", mar) + return MarReader(open(mar, 'rb')).compression_type() == 'xz' @redo.retriable() @@ -64,7 +72,12 @@ def unpack(work_env, mar, dest_dir): unwrap_cmd = sh.Command(os.path.join(work_env.workdir, "unwrap_full_update.pl")) log.debug("Unwrapping %s", mar) - out = unwrap_cmd(mar, _cwd=dest_dir, _env=work_env.env, _timeout=240, + env = work_env.env + if not is_lzma_compressed_mar(mar): + env['MAR_OLD_FORMAT'] = 1 + elif 'MAR_OLD_FORMAT' in env: + del env['MAR_OLD_FORMAT'] + out = unwrap_cmd(mar, _cwd=dest_dir, _env=env, _timeout=240, _err_to_out=True) if out: log.debug(out) @@ -96,6 +109,10 @@ def generate_partial(work_env, from_dir, to_dir, dest_mar, channel_ids, env = work_env.env env["MOZ_PRODUCT_VERSION"] = version env["MOZ_CHANNEL_ID"] = channel_ids + if not is_lzma_compressed_mar(dest_mar): + env['MAR_OLD_FORMAT'] = 1 + elif 'MAR_OLD_FORMAT' in env: + del env['MAR_OLD_FORMAT'] make_incremental_update = os.path.join(work_env.workdir, "make_incremental_update.sh") out = sh.bash(make_incremental_update, dest_mar, from_dir, to_dir, @@ -166,7 +183,8 @@ def verify_allowed_url(mar): def main(): parser = argparse.ArgumentParser() parser.add_argument("--artifacts-dir", required=True) - parser.add_argument("--signing-cert", required=True) + parser.add_argument("--sha1-signing-cert", required=True) + parser.add_argument("--sha384-signing-cert", required=True) parser.add_argument("--task-definition", required=True, type=argparse.FileType('r')) parser.add_argument("--filename-template", @@ -183,6 +201,14 @@ def main(): task = json.load(args.task_definition) # TODO: verify task["extra"]["funsize"]["partials"] with jsonschema + signing_certs = { + 'sha1': args.sha1_signing_cert, + 'sha384': args.sha384_signing_cert, + } + + assert(get_keysize(open(signing_certs['sha1'], 'rb').read()) == 2048) + assert(get_keysize(open(signing_certs['sha384'], 'rb').read()) == 4096) + if args.no_freshclam: log.info("Skipping freshclam") else: @@ -207,7 +233,7 @@ def main(): unpack_dir = os.path.join(work_env.workdir, mar_type) download(f, dest) if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION"): - verify_signature(dest, args.signing_cert) + verify_signature(dest, signing_certs) complete_mars["%s_size" % mar_type] = os.path.getsize(dest) complete_mars["%s_hash" % mar_type] = get_hash(dest) unpack(work_env, dest, unpack_dir)