#!/usr/bin/env python2.7 # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. # This script is used to create and manipulate archives containing # files necessary to build Firefox on Windows (referred to as the # "Windows toolchain"). # # When updating behavior of this script, remember to update the docs # in ``build/docs/toolchains.rst``. from __future__ import absolute_import, unicode_literals import hashlib import os import sys import six from mozpack.files import FileFinder from mozpack.mozjar import JarWriter import mozpack.path as mozpath SDK_RELEASE = "10.0.17134.0" PATTERNS = [ { "srcdir": "%(vs_path)s/DIA SDK", "dstdir": "DIA SDK", "files": [ { "pattern": "bin/**", "ignore": ("bin/arm/**",), }, { "pattern": "idl/**", }, { "pattern": "include/**", }, { "pattern": "lib/**", "ignore": ("lib/arm/**",), }, ], }, { "srcdir": "%(vs_path)s/VC/Tools/MSVC/14.16.27023", "dstdir": "VC", "files": [ # ATL is needed by Breakpad. { "pattern": "atlmfc/include/**", }, { "pattern": "atlmfc/lib/arm64/atls.*", }, { "pattern": "atlmfc/lib/x64/atls.*", }, { "pattern": "atlmfc/lib/x86/atls.*", }, # ARM64 PGO-instrumented builds require ARM64 pgort140.dll. { "pattern": "bin/arm64/pgort140.dll", }, { "pattern": "bin/Hostx64/**", }, # 32-bit PGO-instrumented builds require 32-bit pgort140.dll. { "pattern": "bin/Hostx86/x86/pgort140.dll", }, { "pattern": "include/**", }, { "pattern": "lib/**", "ignore": ( "lib/arm64/store/**", "lib/onecore/**", "lib/x64/store/**", "lib/x86/store/**", ), }, ], }, { "srcdir": "%(vs_path)s/VC/Redist/MSVC/14.16.27012", "dstdir": "VC/redist", "files": [ { "pattern": "arm64/Microsoft.VC141.CRT/**", }, { "pattern": "x64/Microsoft.VC141.CRT/**", }, { "pattern": "x86/Microsoft.VC141.CRT/**", }, ], }, { "srcdir": "%(sdk_path)s", "dstdir": "SDK", "files": [ { "pattern": "bin/%s/x64/**" % SDK_RELEASE, }, { "pattern": "Include/%s/**" % SDK_RELEASE, }, { "pattern": "Lib/%s/ucrt/arm64/**" % SDK_RELEASE, }, { "pattern": "Lib/%s/ucrt/x64/**" % SDK_RELEASE, }, { "pattern": "Lib/%s/ucrt/x86/**" % SDK_RELEASE, }, { "pattern": "Lib/%s/um/arm64/**" % SDK_RELEASE, }, { "pattern": "Lib/%s/um/x64/**" % SDK_RELEASE, }, { "pattern": "Lib/%s/um/x86/**" % SDK_RELEASE, }, { "pattern": "Redist/D3D/**", }, { "pattern": "Redist/ucrt/DLLs/x64/**", }, { "pattern": "Redist/ucrt/DLLs/x86/**", }, ], }, ] def find_vs_paths(): """Resolve source locations of files. Returns a 2-tuple of (Visual Studio Path, SDK Path). """ pf = os.environ.get("ProgramFiles(x86)") if not pf: raise Exception( 'No "ProgramFiles(x86)" environment variable. ' "Not running on 64-bit Windows?" ) vs_path = os.path.join(pf, "Microsoft Visual Studio", "2017", "Community") if not os.path.exists(vs_path): raise Exception( "%s does not exist; Visual Studio 2017 not installed?" % vs_path ) sdk_path = os.path.join(pf, "Windows Kits", "10") if not os.path.exists(sdk_path): raise Exception("%s does not exist; Windows 10 SDK not installed?" % sdk_path) sdk_fullver_path = os.path.join(sdk_path, "Include", SDK_RELEASE) if not os.path.exists(sdk_fullver_path): raise Exception( "%s does not exist; Wrong SDK version installed?" % sdk_fullver_path ) return vs_path, sdk_path def resolve_files(): """Resolve the files that constitute a standalone toolchain. This is a generator of (dest path, file) where the destination path is relative and the file instance is a BaseFile from mozpack. """ vs_path, sdk_path = find_vs_paths() for entry in PATTERNS: fullpath = entry["srcdir"] % { "vs_path": vs_path, "sdk_path": sdk_path, } for pattern in entry["files"]: finder = FileFinder(fullpath, ignore=pattern.get("ignore", [])) for p, f in finder.find(pattern["pattern"]): dstpath = "%s/%s" % (entry["dstdir"], p) yield dstpath.encode("utf-8"), f def resolve_files_and_hash(manifest): """Resolve files and hash their data. This is a generator of 3-tuples of (relpath, data, mode). As data is read, the manifest is populated with metadata. Keys are set to the relative file path. Values are 2-tuples of (data length, sha-256). """ assert manifest == {} for p, f in resolve_files(): data = f.read() sha256 = hashlib.sha256() sha256.update(data) manifest[p] = (len(data), sha256.hexdigest().encode("utf-8")) yield p, data, f.mode def format_manifest(manifest): """Return formatted SHA-256 manifests as a byte strings.""" sha256_lines = [] for path, (length, sha256) in sorted(manifest.items()): sha256_lines.append(b"%s\t%d\t%s" % (sha256, length, path)) # Trailing newline. sha256_lines.append(b"") return b"\n".join(sha256_lines) def write_zip(zip_path, prefix=None): """Write toolchain data to a zip file.""" prefix = six.ensure_binary(prefix, encoding="utf-8") with JarWriter(file=zip_path, compress_level=5) as zip: manifest = {} for p, data, mode in resolve_files_and_hash(manifest): print(p) if prefix: p = mozpath.join(prefix, p) zip.add(p, data, mode=mode) sha256_manifest = format_manifest(manifest) sdk_path = b"SDK_VERSION" sha256_path = b"MANIFEST.SHA256" if prefix: sdk_path = mozpath.join(prefix, sdk_path) sha256_path = mozpath.join(prefix, sha256_path) zip.add(sdk_path, SDK_RELEASE.encode("utf-8")) zip.add(sha256_path, sha256_manifest) if __name__ == "__main__": if len(sys.argv) != 3: print("usage: %s create-zip " % sys.argv[0]) sys.exit(1) assert sys.argv[1] == "create-zip" prefix = os.path.basename(sys.argv[2]) destzip = "%s.zip" % sys.argv[2] write_zip(destzip, prefix=prefix) sha1 = hashlib.sha1() sha256 = hashlib.sha256() sha512 = hashlib.sha512() with open(destzip, "rb") as fh: data = fh.read() sha1.update(data) sha256.update(data) sha512.update(data) print("Hashes of %s (size=%d)" % (destzip, len(data))) print("SHA-1: %s" % sha1.hexdigest()) print("SHA-256: %s" % sha256.hexdigest()) print("SHA-512: %s" % sha512.hexdigest())