mozilla
/
gecko-dev
зеркало из https://github.com/mozilla/gecko-dev.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 1643625 Clear up references to MAR_OLD_FORMAT in packaging tools r=nthomas 

I think we don't need these checks any more.

Differential Revision: https://phabricator.services.mozilla.com/D78480
This commit is contained in:
Simon Fraser 2020-06-11 22:23:10 +00:00
Родитель f8424202b2
Коммит 65cbd9e05b
8 изменённых файлов: 36 добавлений и 334 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

6
python/mozbuild/mozbuild/mach_commands.py
Просмотреть файл

@ -1418,21 +1418,17 @@ class Repackage(MachCommandBase):
help='Mar binary path')
@CommandArgument('--output', '-o', type=str, required=True,
help='Output filename')
@CommandArgument('--format', type=str, default='lzma',
choices=('lzma', 'bz2'),
help='Mar format')
@CommandArgument('--arch', type=str, required=True,
help='The archtecture you are building.')
@CommandArgument('--mar-channel-id', type=str,
help='Mar channel id')
def repackage_mar(self, input, mar, output, format, arch, mar_channel_id):
def repackage_mar(self, input, mar, output, arch, mar_channel_id):
from mozbuild.repackaging.mar import repackage_mar
repackage_mar(
self.topsrcdir,
input,
mar,
output,
format,
arch=arch,
mar_channel_id=mar_channel_id,
)

4
python/mozbuild/mozbuild/repackaging/mar.py
Просмотреть файл

@ -27,7 +27,7 @@ _BCJ_OPTIONS = {
def repackage_mar(
topsrcdir, package, mar, output, mar_format="lzma", arch=None, mar_channel_id=None
topsrcdir, package, mar, output, arch=None, mar_channel_id=None
):
if not zipfile.is_zipfile(package) and not tarfile.is_tarfile(package):
raise Exception("Package file %s is not a valid .zip or .tar file." % package)
@ -66,8 +66,6 @@ def repackage_mar(
env['MAR'] = mozpath.normpath(mar)
if arch:
env['BCJ_OPTIONS'] = ' '.join(_BCJ_OPTIONS[arch])
if mar_format == 'bz2':
env['MAR_OLD_FORMAT'] = '1'
if mar_channel_id:
env['MAR_CHANNEL_ID'] = mar_channel_id
# The Windows build systems have xz installed but it isn't in the path

2
taskcluster/docker/funsize-update-generator/scripts/funsize.py
Просмотреть файл

@ -319,8 +319,6 @@ async def generate_partial(from_dir, to_dir, dest_mar, mar_data, tools_dir, arch
env["MOZ_PRODUCT_VERSION"] = mar_data["version"]
env["MAR_CHANNEL_ID"] = mar_data["MAR_CHANNEL_ID"]
env["BRANCH"] = mar_data["branch"]
if "MAR_OLD_FORMAT" in env:
del env["MAR_OLD_FORMAT"]
make_incremental_update = tools_dir / "make_incremental_update.sh"
cmd = f"{make_incremental_update} {dest_mar} {from_dir} {to_dir}"

208
tools/update-packaging/change_mar_compression.pl
Просмотреть файл

@ -1,208 +0,0 @@
#!/usr/bin/perl -w
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# This tool extracts a mar file, changes the compression of the files contained
# by the mar file either from bzip2 to lzma or lzma to bzip2, and then recreates
# the mar file. The script determines whether the files are compressed with
# bzip2 or lzma to determine what compression should be used. The permissions
# of the files will be retained as long as the script is run on a system such
# as Linux or Mac OS X but not on Windows. The mar file will be created with the
# same MAR channel and Product version information as the original mar file.
# If a mar signature is required the converted mar files will need to be
# re-signed afterwards.
# Author: Robert Strong
#
# -----------------------------------------------------------------------------
# By default just assume that these tools exist in our path
use Getopt::Std;
use Cwd 'abs_path';
use File::Basename;
$|++;
my ($MAR, $XZ, $BZIP2, $MAR_OLD_FORMAT, $FILES, $CHANNEL, $VERSION, $REPLACE, $archive, $tmparchive, @marentries, @marfiles);
if (defined($ENV{"MAR"})) {
$MAR = $ENV{"MAR"};
}
else {
$MAR = "mar";
}
if (defined($ENV{"BZIP2"})) {
$BZIP2 = $ENV{"BZIP2"};
}
else {
$BZIP2 = "bzip2";
}
if (defined($ENV{"XZ"})) {
$XZ = $ENV{"XZ"};
}
else {
$XZ = "xz";
}
sub print_usage
{
print "Usage: change_mar_compression.pl [OPTIONS] ARCHIVE\n";
print "\n";
print "The ARCHIVE will be recreated using either bzip2 for mar file contents that\n";
print "are compressed with lzma or lzma for mar file contents that are compressed\n";
print "with bzip2. The new mar file that is created will not be signed but all other\n";
print "attributes that should be retained will be retained.\n";
print "\n";
print "Options:\n";
print " -h show this help text\n";
print " -r replace the original mar file with the new mar file\n";
}
my %opts;
getopts("hr", \%opts);
if (defined($opts{'h'}) || scalar(@ARGV) != 1) {
print_usage();
exit 1;
}
if ($opts{'r'}) {
$REPLACE = 1;
}
$archive = $ARGV[0];
@marentries = `"$MAR" -T "$archive"`;
$? && die("Couldn't run \"$MAR\" -t");
system("$MAR -x \"$archive\"") == 0 ||
die "Couldn't run $MAR -x";
open(my $testfilename, "updatev3.manifest") or die $!;
binmode($testfilename);
read($testfilename, my $bytes, 3);
if ($bytes eq "BZh") {
$MAR_OLD_TO_NEW = 1;
print "Converting mar file from bzip2 to lzma compression\n";
} else {
undef $MAR_OLD_TO_NEW;
print "Converting mar file from lzma to bzip2 compression\n";
}
close $testfilename;
print "\n";
# The channel is the 4th line of the output
shift @marentries;
shift @marentries;
shift @marentries;
$CHANNEL = substr($marentries[0], 24, -1);
print "MAR channel name: " . $CHANNEL . "\n";
# The version is the 5th line of the output
shift @marentries;
$VERSION = substr($marentries[0], 23, -1);
print "Product version: " . $VERSION . "\n";
# The file entries start on the 8th line of the output
shift @marentries;
shift @marentries;
shift @marentries;
print "\n";
# Decompress the extracted files
foreach (@marentries) {
tr/\n\r//d;
my @splits = split(/\t/,$_);
my $file = $splits[2];
print "Decompressing: " . $file . "\n";
if ($MAR_OLD_TO_NEW) {
system("mv \"$file\" \"$file.bz2\"") == 0 ||
print "\n" && die "Couldn't mv \"$file\"";
system("\"$BZIP2\" -d \"$file.bz2\"") == 0 ||
print "\n" && die "Couldn't decompress \"$file\"";
}
else {
system("mv \"$file\" \"$file.xz\"") == 0 ||
print "\n" && die "Couldn't mv \"$file\"";
system("\"$XZ\" -d \"$file.xz\"") == 0 ||
print "\n" && die "Couldn't decompress \"$file\"";
}
}
print "All files decompressed\n";
print "\n";
# Compress the files in the requested format
$FILES = "";
foreach (@marentries) {
tr/\n\r//d;
my @splits = split(/\t/,$_);
my $mod = $splits[1];
my $file = $splits[2];
print "Compressing: " . $file . "\n";
if ($MAR_OLD_TO_NEW) {
system("\"$XZ\" --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout \"$file\" > \"$file.xz\"") == 0 ||
die "Couldn't compress \"$file\"";
system("mv \"$file.xz\" \"$file\"") == 0 ||
die "Couldn't mv \"$file.xz\"";
}
else {
system("\"$BZIP2\" -z9 \"$file\"") == 0 ||
die "Couldn't compress \"$file\"";
system("mv \"$file.bz2\" \"$file\"") == 0 ||
die "Couldn't mv \"$file.bz2\"";
}
$FILES = $FILES . "\"$file\" ";
chmod oct($mod), $file;
}
print "All files compressed\n";
my $filesuffix = ".bz";
if ($MAR_OLD_TO_NEW) {
$filesuffix = ".xz";
}
$tmparchive = $archive . $filesuffix;
system("$MAR -H $CHANNEL -V $VERSION -c \"$tmparchive\" $FILES") == 0 ||
die "Couldn't run $MAR -c";
if ($REPLACE) {
print "\n";
print "Replacing mar file with the converted mar file\n";
unlink $archive;
system("mv \"$tmparchive\" \"$archive\"") == 0 ||
die "Couldn't mv \"$tmparchive\"";
}
print "\n";
print "Removing extracted files\n";
foreach (@marentries) {
tr/\n\r//d;
my @splits = split(/\t/,$_);
my $file = $splits[2];
unlink $file;
my $dirpath = $file;
while (1) {
if (index($dirpath, '/') < 0) {
last;
}
$dirpath = substr($dirpath, 0, rindex($dirpath, '/'));
rmdir($dirpath);
if (-d $dirpath) {
last;
}
}
}
print "\n";
if ($MAR_OLD_TO_NEW) {
print "Finished converting mar file from bzip2 to lzma compression\n";
} else {
print "Finished converting mar file from lzma to bzip2 compression\n";
}

29
tools/update-packaging/common.sh
Просмотреть файл

@ -14,29 +14,24 @@ QUIET=0
# By default just assume that these tools exist on our path
MAR=${MAR:-mar}
MBSDIFF=${MBSDIFF:-mbsdiff}
if [[ -z "${MAR_OLD_FORMAT}" ]]; then
XZ=${XZ:-xz}
XZ=${XZ:-xz}
$XZ --version > /dev/null 2>&1
if [ $? -ne 0 ]; then
# If $XZ is not set and not found on the path then this is probably
# running on a windows buildbot. Some of the Windows build systems have
# xz.exe in topsrcdir/xz/. Look in the places this would be in both a
# mozilla-central and comm-central build.
XZ="$(dirname "$(dirname "$(dirname "$0")")")/xz/xz.exe"
$XZ --version > /dev/null 2>&1
if [ $? -ne 0 ]; then
# If $XZ is not set and not found on the path then this is probably
# running on a windows buildbot. Some of the Windows build systems have
# xz.exe in topsrcdir/xz/. Look in the places this would be in both a
# mozilla-central and comm-central build.
XZ="$(dirname "$(dirname "$(dirname "$0")")")/xz/xz.exe"
XZ="$(dirname "$(dirname "$(dirname "$(dirname "$0")")")")/xz/xz.exe"
$XZ --version > /dev/null 2>&1
if [ $? -ne 0 ]; then
XZ="$(dirname "$(dirname "$(dirname "$(dirname "$0")")")")/xz/xz.exe"
$XZ --version > /dev/null 2>&1
if [ $? -ne 0 ]; then
echo "xz was not found on this system!"
echo "exiting"
exit 1
fi
echo "xz was not found on this system!"
echo "exiting"
exit 1
fi
fi
else
MAR_OLD_FORMAT=1
BZIP2=${BZIP2:-bzip2}
fi
# -----------------------------------------------------------------------------

15
tools/update-packaging/make_full_update.sh
Просмотреть файл

@ -101,11 +101,7 @@ for ((i=0; $i<$num_files; i=$i+1)); do
dir=$(dirname "$f")
mkdir -p "$workdir/$dir"
if [[ -n $MAR_OLD_FORMAT ]]; then
$BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
else
$XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$targetdir/$f" > "$workdir/$f"
fi
$XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$targetdir/$f" > "$workdir/$f"
copy_perm "$targetdir/$f" "$workdir/$f"
targetfiles="$targetfiles \"$f\""
@ -116,13 +112,8 @@ notice ""
notice "Adding file and directory remove instructions from file 'removed-files'"
append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3"
if [[ -n $MAR_OLD_FORMAT ]]; then
$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
else
$XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv2" && mv -f "$updatemanifestv2.xz" "$updatemanifestv2"
$XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3"
fi
$XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv2" && mv -f "$updatemanifestv2.xz" "$updatemanifestv2"
$XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3"
mar_command="$mar_command -C \"$workdir\" -c output.mar"
eval "$mar_command $targetfiles"

68
tools/update-packaging/make_incremental_update.sh
Просмотреть файл

@ -170,11 +170,7 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do
if check_for_add_if_not_update "$f"; then
# The full workdir may not exist yet, so create it if necessary.
mkdir -p `dirname "$workdir/$f"`
if [[ -n $MAR_OLD_FORMAT ]]; then
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
else
$XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
fi
$XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
copy_perm "$newdir/$f" "$workdir/$f"
make_add_if_not_instruction "$f" "$updatemanifestv3"
archivefiles="$archivefiles \"$f\""
@ -184,11 +180,7 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do
if check_for_forced_update "$requested_forced_updates" "$f"; then
# The full workdir may not exist yet, so create it if necessary.
mkdir -p `dirname "$workdir/$f"`
if [[ -n $MAR_OLD_FORMAT ]]; then
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
else
$XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
fi
$XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
copy_perm "$newdir/$f" "$workdir/$f"
make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 1
archivefiles="$archivefiles \"$f\""
@ -216,44 +208,21 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do
# if service is not enabled then default to old behavior
if [ -z "$MBSDIFF_HOOK" ]; then
$MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
if [[ -n $MAR_OLD_FORMAT ]]; then
$BZIP2 -z9 "$workdir/$f.patch"
else
$XZ --compress --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch"
fi
$XZ --compress --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch"
else
# if service enabled then check patch existence for retrieval
if [[ -n $MAR_OLD_FORMAT ]]; then
if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then
verbose_notice "file \"$f\" found in funsize, diffing skipped"
else
# if not found already - compute it and cache it for future use
$MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
$BZIP2 -z9 "$workdir/$f.patch"
$MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"
fi
if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"; then
verbose_notice "file \"$f\" found in funsize, diffing skipped"
else
if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"; then
verbose_notice "file \"$f\" found in funsize, diffing skipped"
else
# if not found already - compute it and cache it for future use
$MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
$XZ --compress --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch"
$MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"
fi
# if not found already - compute it and cache it for future use
$MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
$XZ --compress --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch"
$MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"
fi
fi
if [[ -n $MAR_OLD_FORMAT ]]; then
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
else
$XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
fi
$XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
copy_perm "$newdir/$f" "$workdir/$f"
if [[ -n $MAR_OLD_FORMAT ]]; then
patchfile="$workdir/$f.patch.bz2"
else
patchfile="$workdir/$f.patch.xz"
fi
patchfile="$workdir/$f.patch.xz"
patchsize=$(get_file_size "$patchfile")
fullsize=$(get_file_size "$workdir/$f")
@ -294,11 +263,7 @@ for ((i=0; $i<$num_newfiles; i=$i+1)); do
dir=$(dirname "$workdir/$f")
mkdir -p "$dir"
if [[ -n $MAR_OLD_FORMAT ]]; then
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
else
$XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
fi
$XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
copy_perm "$newdir/$f" "$workdir/$f"
if check_for_add_if_not_update "$f"; then
@ -339,13 +304,8 @@ for ((i=0; $i<$num_olddirs; i=$i+1)); do
fi
done
if [[ -n $MAR_OLD_FORMAT ]]; then
$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
else
$XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv2" && mv -f "$updatemanifestv2.xz" "$updatemanifestv2"
$XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3"
fi
$XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv2" && mv -f "$updatemanifestv2.xz" "$updatemanifestv2"
$XZ --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3"
mar_command="$mar_command -C \"$workdir\" -c output.mar"
eval "$mar_command $archivefiles"

38
tools/update-packaging/unwrap_full_update.pl
Просмотреть файл

@ -13,7 +13,7 @@
use Getopt::Std;
my ($MAR, $XZ, $BZIP2, $MAR_OLD_FORMAT, $archive, @marentries, @marfiles);
my ($MAR, $XZ, $archive, @marentries, @marfiles);
if (defined($ENV{"MAR"})) {
$MAR = $ENV{"MAR"};
@ -22,13 +22,6 @@ else {
$MAR = "mar";
}
if (defined($ENV{"BZIP2"})) {
$BZIP2 = $ENV{"BZIP2"};
}
else {
$BZIP2 = "bzip2";
}
if (defined($ENV{"XZ"})) {
$XZ = $ENV{"XZ"};
}
@ -84,19 +77,6 @@ $? && die("Couldn't run \"$MAR\" -t");
system($MAR, "-x", $archive) == 0 ||
die "Couldn't run $MAR -x";
# Try to determine if the mar file contains bzip2 compressed files and if not
# assume that the mar file contains lzma compressed files. The updatev3.manifest
# file is checked since a valid mar file must have this file in the root path.
open(my $testfilename, "updatev3.manifest") or die $!;
binmode($testfilename);
read($testfilename, my $bytes, 3);
if ($bytes eq "BZh") {
$MAR_OLD_FORMAT = 1;
} else {
undef $MAR_OLD_FORMAT;
}
close $testfilename;
shift @marentries;
foreach (@marentries) {
@ -105,18 +85,10 @@ foreach (@marentries) {
my $file = $splits[2];
print "Decompressing: " . $file . "\n";
if ($MAR_OLD_FORMAT) {
system("mv", $file, "$file.bz2") == 0 ||
die "Couldn't mv \"$file\"";
system($BZIP2, "-d", "$file.bz2") == 0 ||
die "Couldn't decompress \"$file\"";
}
else {
system("mv", $file, "$file.xz") == 0 ||
die "Couldn't mv \"$file\"";
system($XZ, "-d", "$file.xz") == 0 ||
die "Couldn't decompress \"$file\"";
}
system("mv", $file, "$file.xz") == 0 ||
die "Couldn't mv \"$file\"";
system($XZ, "-d", "$file.xz") == 0 ||
die "Couldn't decompress \"$file\"";
}
print "Finished\n";