зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1675384 - Resurrect mac unification code. r=firefox-build-system-reviewers,mhentges
This adds back part of the code that was removed in bug 1339182, reformats it with black, adjusts it to make flake8 happy, and converts it to python 3. This also adjusts the script after bug 1534003, which changed the about:buildconfig page title. Differential Revision: https://phabricator.services.mozilla.com/D95977
This commit is contained in:
Mike Hommey
Родитель
41a5e5c57f
Коммит
e7bd434ff1
|
|
@ -15,3 +15,5 @@ skip-if = python == 2 && os == "mac"
|
|||
[test_packager_l10n.py]
|
||||
[test_packager_unpack.py]
|
||||
[test_path.py]
|
||||
[test_unify.py]
|
||||
skip-if = python == 2
|
||||
|
|
|
|||
|
|
@ -0,0 +1,251 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from mozbuild.util import ensureParentDir
|
||||
|
||||
from mozpack.unify import (
|
||||
UnifiedFinder,
|
||||
UnifiedBuildFinder,
|
||||
)
|
||||
import mozunit
|
||||
from mozpack.test.test_files import TestWithTmpDir
|
||||
from mozpack.files import FileFinder
|
||||
from mozpack.mozjar import JarWriter
|
||||
from mozpack.test.test_files import MockDest
|
||||
from io import StringIO
|
||||
import os
|
||||
import sys
|
||||
from mozpack.errors import (
|
||||
ErrorMessage,
|
||||
AccumulatedErrors,
|
||||
errors,
|
||||
)
|
||||
|
||||
|
||||
class TestUnified(TestWithTmpDir):
|
||||
def create_one(self, which, path, content):
|
||||
file = self.tmppath(os.path.join(which, path))
|
||||
ensureParentDir(file)
|
||||
if isinstance(content, str):
|
||||
content = content.encode("utf-8")
|
||||
open(file, "wb").write(content)
|
||||
|
||||
def create_both(self, path, content):
|
||||
for p in ["a", "b"]:
|
||||
self.create_one(p, path, content)
|
||||
|
||||
|
||||
class TestUnifiedFinder(TestUnified):
|
||||
def test_unified_finder(self):
|
||||
self.create_both("foo/bar", "foobar")
|
||||
self.create_both("foo/baz", "foobaz")
|
||||
self.create_one("a", "bar", "bar")
|
||||
self.create_one("b", "baz", "baz")
|
||||
self.create_one("a", "qux", "foobar")
|
||||
self.create_one("b", "qux", "baz")
|
||||
self.create_one("a", "test/foo", "a\nb\nc\n")
|
||||
self.create_one("b", "test/foo", "b\nc\na\n")
|
||||
self.create_both("test/bar", "a\nb\nc\n")
|
||||
|
||||
finder = UnifiedFinder(
|
||||
FileFinder(self.tmppath("a")),
|
||||
FileFinder(self.tmppath("b")),
|
||||
sorted=["test"],
|
||||
)
|
||||
self.assertEqual(
|
||||
sorted(
|
||||
[(f, c.open().read().decode("utf-8")) for f, c in finder.find("foo")]
|
||||
),
|
||||
[("foo/bar", "foobar"), ("foo/baz", "foobaz")],
|
||||
)
|
||||
self.assertRaises(ErrorMessage, any, finder.find("bar"))
|
||||
self.assertRaises(ErrorMessage, any, finder.find("baz"))
|
||||
self.assertRaises(ErrorMessage, any, finder.find("qux"))
|
||||
self.assertEqual(
|
||||
sorted(
|
||||
[(f, c.open().read().decode("utf-8")) for f, c in finder.find("test")]
|
||||
),
|
||||
[("test/bar", "a\nb\nc\n"), ("test/foo", "a\nb\nc\n")],
|
||||
)
|
||||
|
||||
|
||||
class TestUnifiedBuildFinder(TestUnified):
|
||||
def test_unified_build_finder(self):
|
||||
finder = UnifiedBuildFinder(
|
||||
FileFinder(self.tmppath("a")), FileFinder(self.tmppath("b"))
|
||||
)
|
||||
|
||||
# Test chrome.manifest unification
|
||||
self.create_both("chrome.manifest", "a\nb\nc\n")
|
||||
self.create_one("a", "chrome/chrome.manifest", "a\nb\nc\n")
|
||||
self.create_one("b", "chrome/chrome.manifest", "b\nc\na\n")
|
||||
self.assertEqual(
|
||||
sorted(
|
||||
[
|
||||
(f, c.open().read().decode("utf-8"))
|
||||
for f, c in finder.find("**/chrome.manifest")
|
||||
]
|
||||
),
|
||||
[("chrome.manifest", "a\nb\nc\n"), ("chrome/chrome.manifest", "a\nb\nc\n")],
|
||||
)
|
||||
|
||||
# Test buildconfig.html unification
|
||||
self.create_one(
|
||||
"a",
|
||||
"chrome/browser/foo/buildconfig.html",
|
||||
"\n".join(
|
||||
[
|
||||
"<html>",
|
||||
"<body>",
|
||||
"<h1>Build Configuration</h1>",
|
||||
"<div>foo</div>",
|
||||
"</body>",
|
||||
"</html>",
|
||||
]
|
||||
),
|
||||
)
|
||||
self.create_one(
|
||||
"b",
|
||||
"chrome/browser/foo/buildconfig.html",
|
||||
"\n".join(
|
||||
[
|
||||
"<html>",
|
||||
"<body>",
|
||||
"<h1>Build Configuration</h1>",
|
||||
"<div>bar</div>",
|
||||
"</body>",
|
||||
"</html>",
|
||||
]
|
||||
),
|
||||
)
|
||||
self.assertEqual(
|
||||
sorted(
|
||||
[
|
||||
(f, c.open().read().decode("utf-8"))
|
||||
for f, c in finder.find("**/buildconfig.html")
|
||||
]
|
||||
),
|
||||
[
|
||||
(
|
||||
"chrome/browser/foo/buildconfig.html",
|
||||
"\n".join(
|
||||
[
|
||||
"<html>",
|
||||
"<body>",
|
||||
"<h1>Build Configuration</h1>",
|
||||
"<div>foo</div>",
|
||||
"<hr> </hr>",
|
||||
"<div>bar</div>",
|
||||
"</body>",
|
||||
"</html>",
|
||||
]
|
||||
),
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
# Test xpi file unification
|
||||
xpi = MockDest()
|
||||
with JarWriter(fileobj=xpi, compress=True) as jar:
|
||||
jar.add("foo", "foo")
|
||||
jar.add("bar", "bar")
|
||||
foo_xpi = xpi.read()
|
||||
self.create_both("foo.xpi", foo_xpi)
|
||||
|
||||
with JarWriter(fileobj=xpi, compress=True) as jar:
|
||||
jar.add("foo", "bar")
|
||||
self.create_one("a", "bar.xpi", foo_xpi)
|
||||
self.create_one("b", "bar.xpi", xpi.read())
|
||||
|
||||
errors.out = StringIO()
|
||||
with self.assertRaises(AccumulatedErrors), errors.accumulate():
|
||||
self.assertEqual(
|
||||
[(f, c.open().read()) for f, c in finder.find("*.xpi")],
|
||||
[("foo.xpi", foo_xpi)],
|
||||
)
|
||||
errors.out = sys.stderr
|
||||
|
||||
# Test install.rdf unification
|
||||
x86_64 = "Darwin_x86_64-gcc3"
|
||||
x86 = "Darwin_x86-gcc3"
|
||||
target_tag = "<{em}targetPlatform>{platform}</{em}targetPlatform>"
|
||||
target_attr = '{em}targetPlatform="{platform}" '
|
||||
|
||||
rdf_tag = "".join(
|
||||
[
|
||||
'<{RDF}Description {em}bar="bar" {em}qux="qux">',
|
||||
"<{em}foo>foo</{em}foo>",
|
||||
"{targets}",
|
||||
"<{em}baz>baz</{em}baz>",
|
||||
"</{RDF}Description>",
|
||||
]
|
||||
)
|
||||
rdf_attr = "".join(
|
||||
[
|
||||
'<{RDF}Description {em}bar="bar" {attr}{em}qux="qux">',
|
||||
"{targets}",
|
||||
"<{em}foo>foo</{em}foo><{em}baz>baz</{em}baz>",
|
||||
"</{RDF}Description>",
|
||||
]
|
||||
)
|
||||
|
||||
for descr_ns, target_ns in (("RDF:", ""), ("", "em:"), ("RDF:", "em:")):
|
||||
# First we need to infuse the above strings with our namespaces and
|
||||
# platform values.
|
||||
ns = {"RDF": descr_ns, "em": target_ns}
|
||||
target_tag_x86_64 = target_tag.format(platform=x86_64, **ns)
|
||||
target_tag_x86 = target_tag.format(platform=x86, **ns)
|
||||
target_attr_x86_64 = target_attr.format(platform=x86_64, **ns)
|
||||
target_attr_x86 = target_attr.format(platform=x86, **ns)
|
||||
|
||||
tag_x86_64 = rdf_tag.format(targets=target_tag_x86_64, **ns)
|
||||
tag_x86 = rdf_tag.format(targets=target_tag_x86, **ns)
|
||||
tag_merged = rdf_tag.format(
|
||||
targets=target_tag_x86_64 + target_tag_x86, **ns
|
||||
)
|
||||
tag_empty = rdf_tag.format(targets="", **ns)
|
||||
|
||||
attr_x86_64 = rdf_attr.format(attr=target_attr_x86_64, targets="", **ns)
|
||||
attr_x86 = rdf_attr.format(attr=target_attr_x86, targets="", **ns)
|
||||
attr_merged = rdf_attr.format(
|
||||
attr="", targets=target_tag_x86_64 + target_tag_x86, **ns
|
||||
)
|
||||
|
||||
# This table defines the test cases, columns "a" and "b" being the
|
||||
# contents of the install.rdf of the respective platform and
|
||||
# "result" the exepected merged content after unification.
|
||||
testcases = (
|
||||
# _____a_____ _____b_____ ___result___#
|
||||
(tag_x86_64, tag_x86, tag_merged),
|
||||
(tag_x86_64, tag_empty, tag_empty),
|
||||
(tag_empty, tag_x86, tag_empty),
|
||||
(tag_empty, tag_empty, tag_empty),
|
||||
(attr_x86_64, attr_x86, attr_merged),
|
||||
(tag_x86_64, attr_x86, tag_merged),
|
||||
(attr_x86_64, tag_x86, attr_merged),
|
||||
(attr_x86_64, tag_empty, tag_empty),
|
||||
(tag_empty, attr_x86, tag_empty),
|
||||
)
|
||||
|
||||
# Now create the files from the above table and compare
|
||||
results = []
|
||||
for emid, (rdf_a, rdf_b, result) in enumerate(testcases):
|
||||
filename = "ext/id{0}/install.rdf".format(emid)
|
||||
self.create_one("a", filename, rdf_a)
|
||||
self.create_one("b", filename, rdf_b)
|
||||
results.append((filename, result))
|
||||
|
||||
self.assertEqual(
|
||||
sorted(
|
||||
[
|
||||
(f, c.open().read().decode("utf-8"))
|
||||
for f, c in finder.find("**/install.rdf")
|
||||
]
|
||||
),
|
||||
results,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
mozunit.main()
|
||||
|
|
@ -0,0 +1,253 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from mozpack.files import (
|
||||
BaseFinder,
|
||||
JarFinder,
|
||||
ExecutableFile,
|
||||
BaseFile,
|
||||
GeneratedFile,
|
||||
)
|
||||
from mozpack.executables import (
|
||||
MACHO_SIGNATURES,
|
||||
)
|
||||
from mozpack.mozjar import JarReader
|
||||
from mozpack.errors import errors
|
||||
from tempfile import mkstemp
|
||||
import mozpack.path as mozpath
|
||||
import struct
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import buildconfig
|
||||
from collections import OrderedDict
|
||||
|
||||
# Regular expressions for unifying install.rdf
|
||||
FIND_TARGET_PLATFORM = re.compile(
|
||||
r"""
|
||||
<(?P<ns>[-._0-9A-Za-z]+:)?targetPlatform> # The targetPlatform tag, with any namespace
|
||||
(?P<platform>[^<]*) # The actual platform value
|
||||
</(?P=ns)?targetPlatform> # The closing tag
|
||||
""",
|
||||
re.X,
|
||||
)
|
||||
FIND_TARGET_PLATFORM_ATTR = re.compile(
|
||||
r"""
|
||||
(?P<tag><(?:[-._0-9A-Za-z]+:)?Description) # The opening part of the <Description> tag
|
||||
(?P<attrs>[^>]*?)\s+ # The initial attributes
|
||||
(?P<ns>[-._0-9A-Za-z]+:)?targetPlatform= # The targetPlatform attribute, with any namespace
|
||||
[\'"](?P<platform>[^\'"]+)[\'"] # The actual platform value
|
||||
(?P<otherattrs>[^>]*?>) # The remaining attributes and closing angle bracket
|
||||
""",
|
||||
re.X,
|
||||
)
|
||||
|
||||
|
||||
def may_unify_binary(file):
|
||||
"""
|
||||
Return whether the given BaseFile instance is an ExecutableFile that
|
||||
may be unified. Only non-fat Mach-O binaries are to be unified.
|
||||
"""
|
||||
if isinstance(file, ExecutableFile):
|
||||
signature = file.open().read(4)
|
||||
if len(signature) < 4:
|
||||
return False
|
||||
signature = struct.unpack(">L", signature)[0]
|
||||
if signature in MACHO_SIGNATURES:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class UnifiedExecutableFile(BaseFile):
|
||||
"""
|
||||
File class for executable and library files that to be unified with 'lipo'.
|
||||
"""
|
||||
|
||||
def __init__(self, executable1, executable2):
|
||||
"""
|
||||
Initialize a UnifiedExecutableFile with a pair of ExecutableFiles to
|
||||
be unified. They are expected to be non-fat Mach-O executables.
|
||||
"""
|
||||
assert isinstance(executable1, ExecutableFile)
|
||||
assert isinstance(executable2, ExecutableFile)
|
||||
self._executables = (executable1, executable2)
|
||||
|
||||
def copy(self, dest, skip_if_older=True):
|
||||
"""
|
||||
Create a fat executable from the two Mach-O executable given when
|
||||
creating the instance.
|
||||
skip_if_older is ignored.
|
||||
"""
|
||||
assert isinstance(dest, str)
|
||||
tmpfiles = []
|
||||
try:
|
||||
for e in self._executables:
|
||||
fd, f = mkstemp()
|
||||
os.close(fd)
|
||||
tmpfiles.append(f)
|
||||
e.copy(f, skip_if_older=False)
|
||||
lipo = buildconfig.substs.get("LIPO") or "lipo"
|
||||
subprocess.call([lipo, "-create"] + tmpfiles + ["-output", dest])
|
||||
finally:
|
||||
for f in tmpfiles:
|
||||
os.unlink(f)
|
||||
|
||||
|
||||
class UnifiedFinder(BaseFinder):
|
||||
"""
|
||||
Helper to get unified BaseFile instances from two distinct trees on the
|
||||
file system.
|
||||
"""
|
||||
|
||||
def __init__(self, finder1, finder2, sorted=[], **kargs):
|
||||
"""
|
||||
Initialize a UnifiedFinder. finder1 and finder2 are BaseFinder
|
||||
instances from which files are picked. UnifiedFinder.find() will act as
|
||||
FileFinder.find() but will error out when matches can only be found in
|
||||
one of the two trees and not the other. It will also error out if
|
||||
matches can be found on both ends but their contents are not identical.
|
||||
|
||||
The sorted argument gives a list of mozpath.match patterns. File
|
||||
paths matching one of these patterns will have their contents compared
|
||||
with their lines sorted.
|
||||
"""
|
||||
assert isinstance(finder1, BaseFinder)
|
||||
assert isinstance(finder2, BaseFinder)
|
||||
self._finder1 = finder1
|
||||
self._finder2 = finder2
|
||||
self._sorted = sorted
|
||||
BaseFinder.__init__(self, finder1.base, **kargs)
|
||||
|
||||
def _find(self, path):
|
||||
"""
|
||||
UnifiedFinder.find() implementation.
|
||||
"""
|
||||
files1 = OrderedDict()
|
||||
for p, f in self._finder1.find(path):
|
||||
files1[p] = f
|
||||
files2 = set()
|
||||
for p, f in self._finder2.find(path):
|
||||
files2.add(p)
|
||||
if p in files1:
|
||||
if may_unify_binary(files1[p]) and may_unify_binary(f):
|
||||
yield p, UnifiedExecutableFile(files1[p], f)
|
||||
else:
|
||||
err = errors.count
|
||||
unified = self.unify_file(p, files1[p], f)
|
||||
if unified:
|
||||
yield p, unified
|
||||
elif err == errors.count:
|
||||
self._report_difference(p, files1[p], f)
|
||||
else:
|
||||
errors.error("File missing in %s: %s" % (self._finder1.base, p))
|
||||
for p in [p for p in files1 if p not in files2]:
|
||||
errors.error("File missing in %s: %s" % (self._finder2.base, p))
|
||||
|
||||
def _report_difference(self, path, file1, file2):
|
||||
"""
|
||||
Report differences between files in both trees.
|
||||
"""
|
||||
errors.error(
|
||||
"Can't unify %s: file differs between %s and %s"
|
||||
% (path, self._finder1.base, self._finder2.base)
|
||||
)
|
||||
if not isinstance(file1, ExecutableFile) and not isinstance(
|
||||
file2, ExecutableFile
|
||||
):
|
||||
from difflib import unified_diff
|
||||
|
||||
for line in unified_diff(
|
||||
[l.decode("utf-8") for l in file1.open().readlines()],
|
||||
[l.decode("utf-8") for l in file2.open().readlines()],
|
||||
os.path.join(self._finder1.base, path),
|
||||
os.path.join(self._finder2.base, path),
|
||||
):
|
||||
errors.out.write(line)
|
||||
|
||||
def unify_file(self, path, file1, file2):
|
||||
"""
|
||||
Given two BaseFiles and the path they were found at, check whether
|
||||
their content match and return the first BaseFile if they do.
|
||||
"""
|
||||
content1 = file1.open().readlines()
|
||||
content2 = file2.open().readlines()
|
||||
if content1 == content2:
|
||||
return file1
|
||||
for pattern in self._sorted:
|
||||
if mozpath.match(path, pattern):
|
||||
if sorted(content1) == sorted(content2):
|
||||
return file1
|
||||
break
|
||||
return None
|
||||
|
||||
|
||||
class UnifiedBuildFinder(UnifiedFinder):
|
||||
"""
|
||||
Specialized UnifiedFinder for Mozilla applications packaging. It allows
|
||||
"*.manifest" files to differ in their order, and unifies "buildconfig.html"
|
||||
files by merging their content.
|
||||
"""
|
||||
|
||||
def __init__(self, finder1, finder2, **kargs):
|
||||
UnifiedFinder.__init__(
|
||||
self, finder1, finder2, sorted=["**/*.manifest"], **kargs
|
||||
)
|
||||
|
||||
def unify_file(self, path, file1, file2):
|
||||
"""
|
||||
Unify files taking Mozilla application special cases into account.
|
||||
Otherwise defer to UnifiedFinder.unify_file.
|
||||
"""
|
||||
basename = mozpath.basename(path)
|
||||
if basename == "buildconfig.html":
|
||||
content1 = file1.open().readlines()
|
||||
content2 = file2.open().readlines()
|
||||
# Copy everything from the first file up to the end of its <body>,
|
||||
# insert a <hr> between the two files and copy the second file's
|
||||
# content beginning after its leading <h1>.
|
||||
return GeneratedFile(
|
||||
b"".join(
|
||||
content1[: content1.index(b"</body>\n")]
|
||||
+ [b"<hr> </hr>\n"]
|
||||
+ content2[content2.index(b"<h1>Build Configuration</h1>\n") + 1 :]
|
||||
)
|
||||
)
|
||||
elif basename == "install.rdf":
|
||||
# install.rdf files often have em:targetPlatform (either as
|
||||
# attribute or as tag) that will differ between platforms. The
|
||||
# unified install.rdf should contain both em:targetPlatforms if
|
||||
# they exist, or strip them if only one file has a target platform.
|
||||
content1, content2 = (
|
||||
FIND_TARGET_PLATFORM_ATTR.sub(
|
||||
lambda m: m.group("tag")
|
||||
+ m.group("attrs")
|
||||
+ m.group("otherattrs")
|
||||
+ "<%stargetPlatform>%s</%stargetPlatform>"
|
||||
% (m.group("ns") or "", m.group("platform"), m.group("ns") or ""),
|
||||
f.open().read().decode("utf-8"),
|
||||
)
|
||||
for f in (file1, file2)
|
||||
)
|
||||
|
||||
platform2 = FIND_TARGET_PLATFORM.search(content2)
|
||||
return GeneratedFile(
|
||||
FIND_TARGET_PLATFORM.sub(
|
||||
lambda m: m.group(0) + platform2.group(0) if platform2 else "",
|
||||
content1,
|
||||
)
|
||||
)
|
||||
elif path.endswith(".xpi"):
|
||||
finder1 = JarFinder(
|
||||
os.path.join(self._finder1.base, path), JarReader(fileobj=file1.open())
|
||||
)
|
||||
finder2 = JarFinder(
|
||||
os.path.join(self._finder2.base, path), JarReader(fileobj=file2.open())
|
||||
)
|
||||
unifier = UnifiedFinder(finder1, finder2, sorted=self._sorted)
|
||||
err = errors.count
|
||||
all(unifier.find(""))
|
||||
if err == errors.count:
|
||||
return file1
|
||||
return None
|
||||
return UnifiedFinder.unify_file(self, path, file1, file2)
|
||||
|
|
@ -36,6 +36,8 @@ py2:
|
|||
- config/create_res.py
|
||||
- config/printconfigsetting.py
|
||||
- python/mozbuild/mozbuild/html_build_viewer.py
|
||||
- python/mozbuild/mozpack/unify.py
|
||||
- python/mozbuild/mozpack/test/test_unify.py
|
||||
- python/mozlint
|
||||
- python/mozperftest
|
||||
- python/mozrelease/mozrelease/partner_repack.py
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче