Bug 1690870 - Re-sign Mach-O binaries on macOS. r=firefox-build-system-reviewers,andi

Differential Revision: https://phabricator.services.mozilla.com/D129791

python/mozbuild/mozbuild/artifacts.py

@@ -52,6 +52,8 @@ import tarfile
 import tempfile
 import six.moves.urllib_parse as urlparse
 import zipfile
+from contextlib import contextmanager
+from io import BufferedReader
 
 import pylru
 from gecko_taskgraph.util.taskcluster import (
@@ -70,6 +72,7 @@ from mozpack.files import JarFinder, TarFinder
 from mozpack.mozjar import JarReader, JarWriter
 from mozpack.packager.unpack import UnpackFinder
 import mozpack.path as mozpath
+from mozpack import executables
 
 # Number of candidate pushheads to cache per parent changeset.
 NUM_PUSHHEADS_TO_QUERY_PER_PARENT = 50
@@ -208,6 +211,11 @@ class ArtifactJob(object):
                 "found none!".format(re=self._maven_zip_re)
             )
 
+    @contextmanager
+    def get_writer(self, **kwargs):
+        with JarWriter(**kwargs) as writer:
+            yield writer
+
     def process_artifact(self, filename, processed_filename):
         if filename.endswith(ArtifactJob._test_zip_archive_suffix) and self._tests_re:
             return self.process_tests_zip_artifact(filename, processed_filename)
@@ -229,7 +237,7 @@ class ArtifactJob(object):
 
         added_entry = False
 
-        with JarWriter(file=processed_filename, compress_level=5) as writer:
+        with self.get_writer(file=processed_filename, compress_level=5) as writer:
             reader = JarReader(filename)
             for filename, entry in six.iteritems(reader.entries):
                 for pattern, (src_prefix, dest_prefix) in self.test_artifact_patterns:
@@ -288,7 +296,7 @@ class ArtifactJob(object):
 
         added_entry = False
 
-        with JarWriter(file=processed_filename, compress_level=5) as writer:
+        with self.get_writer(file=processed_filename, compress_level=5) as writer:
             with tarfile.open(filename) as reader:
                 for filename, entry in TarFinder(filename, reader):
                     for (
@@ -349,7 +357,7 @@ class ArtifactJob(object):
     def process_symbols_archive(
         self, filename, processed_filename, skip_compressed=False
     ):
-        with JarWriter(file=processed_filename, compress_level=5) as writer:
+        with self.get_writer(file=processed_filename, compress_level=5) as writer:
             for filename, entry in self.iter_artifact_archive(filename):
                 if skip_compressed and filename.endswith(".gz"):
                     self.log(
@@ -417,7 +425,7 @@ class AndroidArtifactJob(ArtifactJob):
 
     def process_package_artifact(self, filename, processed_filename):
         # Extract all .so files into the root, which will get copied into dist/bin.
-        with JarWriter(file=processed_filename, compress_level=5) as writer:
+        with self.get_writer(file=processed_filename, compress_level=5) as writer:
             for p, f in UnpackFinder(JarFinder(filename, JarReader(filename))):
                 if not any(
                     mozpath.match(p, pat) for pat in self.package_artifact_patterns
@@ -448,7 +456,7 @@ class AndroidArtifactJob(ArtifactJob):
 
         import gzip
 
-        with JarWriter(file=processed_filename, compress_level=5) as writer:
+        with self.get_writer(file=processed_filename, compress_level=5) as writer:
             for filename, entry in self.iter_artifact_archive(filename):
                 if not filename.endswith(".gz"):
                     continue
@@ -497,7 +505,7 @@ class LinuxArtifactJob(ArtifactJob):
     def process_package_artifact(self, filename, processed_filename):
         added_entry = False
 
-        with JarWriter(file=processed_filename, compress_level=5) as writer:
+        with self.get_writer(file=processed_filename, compress_level=5) as writer:
             with tarfile.open(filename) as reader:
                 for p, f in UnpackFinder(TarFinder(filename, reader)):
                     if not any(
@@ -526,6 +534,41 @@ class LinuxArtifactJob(ArtifactJob):
             )
 
 
+class ResignJarWriter(JarWriter):
+    def __init__(self, job, **kwargs):
+        super().__init__(**kwargs)
+        self._job = job
+
+    def add(self, name, data, mode=None):
+        if self._job._substs["HOST_OS_ARCH"] == "Darwin":
+            # Wrap in a BufferedReader so that executable.get_type can peek at the
+            # data signature without subsequent read() being affected.
+            data = BufferedReader(data)
+            if executables.get_type(data) == executables.MACHO:
+                # If the file is a Mach-O binary, we run `codesign -s - -f` against
+                # it to force a local codesign against the original binary, which is
+                # likely unsigned. As of writing, only arm64 macs require codesigned
+                # binaries, but it doesn't hurt to do it on intel macs as well
+                # preemptively, because they could end up with the same requirement
+                # in future versions of macOS.
+                tmp = tempfile.NamedTemporaryFile(delete=False)
+                try:
+                    shutil.copyfileobj(data, tmp)
+                    tmp.close()
+                    self._job.log(
+                        logging.DEBUG, "artifact", {"path": name}, "Re-signing {path}"
+                    )
+                    subprocess.check_call(
+                        ["codesign", "-s", "-", "-f", tmp.name],
+                        stdout=subprocess.DEVNULL,
+                        stderr=subprocess.DEVNULL,
+                    )
+                    data = open(tmp.name, "rb")
+                finally:
+                    os.unlink(tmp.name)
+        super().add(name, data, mode=mode)
+
+
 class MacArtifactJob(ArtifactJob):
     package_re = r"public/build/target\.dmg"
     product = "firefox"
@@ -552,6 +595,11 @@ class MacArtifactJob(ArtifactJob):
         root, paths = self._paths_no_keep_path
         return (root, [p.format(product=self.product) for p in paths])
 
+    @contextmanager
+    def get_writer(self, **kwargs):
+        with ResignJarWriter(self, **kwargs) as writer:
+            yield writer
+
     def process_package_artifact(self, filename, processed_filename):
         tempdir = tempfile.mkdtemp()
         oldcwd = os.getcwd()
@@ -604,7 +652,7 @@ class MacArtifactJob(ArtifactJob):
                 )
             ]
 
-            with JarWriter(file=processed_filename, compress_level=5) as writer:
+            with self.get_writer(file=processed_filename, compress_level=5) as writer:
                 root, paths = self.paths_no_keep_path
                 finder = UnpackFinder(mozpath.join(source, root))
                 for path in paths:
@@ -616,7 +664,7 @@ class MacArtifactJob(ArtifactJob):
                             "Adding {path} to processed archive",
                         )
                         destpath = mozpath.join("bin", os.path.basename(p))
-                        writer.add(destpath.encode("utf-8"), f, mode=f.mode)
+                        writer.add(destpath.encode("utf-8"), f.open(), mode=f.mode)
 
                 for root, paths in paths_keep_path:
                     finder = UnpackFinder(mozpath.join(source, root))
@@ -683,7 +731,7 @@ class WinArtifactJob(ArtifactJob):
 
     def process_package_artifact(self, filename, processed_filename):
         added_entry = False
-        with JarWriter(file=processed_filename, compress_level=5) as writer:
+        with self.get_writer(file=processed_filename, compress_level=5) as writer:
             for p, f in UnpackFinder(JarFinder(filename, JarReader(filename))):
                 if not any(
                     mozpath.match(p, pat) for pat in self.package_artifact_patterns

python/mozbuild/mozpack/executables.py

@@ -7,6 +7,7 @@ from __future__ import absolute_import, print_function, unicode_literals
 import os
 import struct
 import subprocess
+from io import BytesIO
 from mozpack.errors import errors
 
 MACHO_SIGNATURES = [
@@ -25,36 +26,41 @@ MACHO = 1
 ELF = 2
 
 
-def get_type(path):
+def get_type(path_or_fileobj):
     """
     Check the signature of the give file and returns what kind of executable
     matches.
     """
-    with open(path, "rb") as f:
-        signature = f.read(4)
-        if len(signature) < 4:
-            return UNKNOWN
-        signature = struct.unpack(">L", signature)[0]
-        if signature == ELF_SIGNATURE:
-            return ELF
-        if signature in MACHO_SIGNATURES:
-            return MACHO
-        if signature != FAT_SIGNATURE:
-            return UNKNOWN
-        # We have to sanity check the second four bytes, because Java class
-        # files use the same magic number as Mach-O fat binaries.
-        # This logic is adapted from file(1), which says that Mach-O uses
-        # these bytes to count the number of architectures within, while
-        # Java uses it for a version number. Conveniently, there are only
-        # 18 labelled Mach-O architectures, and Java's first released
-        # class format used the version 43.0.
-        num = f.read(4)
-        if len(num) < 4:
-            return UNKNOWN
-        num = struct.unpack(">L", num)[0]
-        if num < 20:
-            return MACHO
+    if hasattr(path_or_fileobj, "peek"):
+        f = BytesIO(path_or_fileobj.peek(8))
+    elif hasattr(path_or_fileobj, "read"):
+        f = path_or_fileobj
+    else:
+        f = open(path, "rb")
+    signature = f.read(4)
+    if len(signature) < 4:
         return UNKNOWN
+    signature = struct.unpack(">L", signature)[0]
+    if signature == ELF_SIGNATURE:
+        return ELF
+    if signature in MACHO_SIGNATURES:
+        return MACHO
+    if signature != FAT_SIGNATURE:
+        return UNKNOWN
+    # We have to sanity check the second four bytes, because Java class
+    # files use the same magic number as Mach-O fat binaries.
+    # This logic is adapted from file(1), which says that Mach-O uses
+    # these bytes to count the number of architectures within, while
+    # Java uses it for a version number. Conveniently, there are only
+    # 18 labelled Mach-O architectures, and Java's first released
+    # class format used the version 43.0.
+    num = f.read(4)
+    if len(num) < 4:
+        return UNKNOWN
+    num = struct.unpack(">L", num)[0]
+    if num < 20:
+        return MACHO
+    return UNKNOWN
 
 
 def is_executable(path):