From cca6da95a51dfb1d1b3b96ff0589f624d963bbb1 Mon Sep 17 00:00:00 2001 From: Ted Mielczarek Date: Thu, 5 Nov 2009 09:04:49 -0500 Subject: [PATCH] bug 526668 - add option to 'unify' to allow files to match if their sorted contents match. r=bsmedberg --- build/Makefile.in | 22 +++++ build/macosx/universal/flight.mk | 2 + build/macosx/universal/unify | 140 +++++++++++++++++++++++++++++-- 3 files changed, 159 insertions(+), 5 deletions(-) diff --git a/build/Makefile.in b/build/Makefile.in index 026cd7adb308..f3888bd2b5d5 100644 --- a/build/Makefile.in +++ b/build/Makefile.in @@ -120,6 +120,8 @@ check:: @if ! file -b ./unify-test-universal | head -n1 | grep -q "^Mach-O universal binary"; then \ echo "TEST-UNEXPECTED-FAIL | build/ | unify failed to produce a universal binary!"; \ false; \ + else \ + echo "TEST-PASS | build/ | unify produced a universal binary!"; \ fi # try unifying two identical Java class files rm -f unifytesta.class unifytestb.class unifytestc.class @@ -137,6 +139,26 @@ check:: @if ! diff -q ./unifytesta.class ./unifytestc.class; then \ echo "TEST-UNEXPECTED-FAIL | build/ | unify failed to unify a Java class file!"; \ false; \ + else \ + echo "TEST-PASS | build/ | unify unified a Java class file!"; \ + fi +# try unifying some files that differ only in line ordering + rm -rf unify-sort-test + mkdir unify-sort-test unify-sort-test/a unify-sort-test/b + printf "lmn\nabc\nxyz\n" > unify-sort-test/a/file.foo + printf "xyz\nlmn\nabc\n" > unify-sort-test/b/file.foo + printf "abc\nlmn\nxyz\n" > unify-sort-test/expected-result + @if ! $(srcdir)/macosx/universal/unify --unify-with-sort "\.foo$$" \ + ./unify-sort-test/a ./unify-sort-test/b \ + ./unify-sort-test/c; then \ + echo "TEST-UNEXPECTED-FAIL | build/ | unify failed to unify files with differing line ordering!"; \ + false; \ + fi + @if ! diff -q ./unify-sort-test/expected-result ./unify-sort-test/c/file.foo; then \ + echo "TEST-UNEXPECTED-FAIL | build/ | unify failed to unify files with differing line ordering!"; \ + false; \ + else \ + echo "TEST-PASS | build/ | unify unified files with differing line ordering!"; \ fi endif diff --git a/build/macosx/universal/flight.mk b/build/macosx/universal/flight.mk index 40db72c15d98..947d30675743 100644 --- a/build/macosx/universal/flight.mk +++ b/build/macosx/universal/flight.mk @@ -105,6 +105,7 @@ postflight_all: ln -s $(DIST_UNI) $(DIST_X86)/universal rm -rf $(DIST_UNI)/$(MOZ_PKG_APPNAME)/$(APPNAME) $(TOPSRCDIR)/build/macosx/universal/unify \ + --unify-with-sort "\.manifest$$" \ $(DIST_PPC)/$(MOZ_PKG_APPNAME)/$(APPNAME) \ $(DIST_X86)/$(MOZ_PKG_APPNAME)/$(APPNAME) \ $(DIST_UNI)/$(MOZ_PKG_APPNAME)/$(APPNAME) @@ -123,6 +124,7 @@ postflight_all: cp $(DIST_PPC)/test-package-stage/reftest/automation.py \ $(DIST_X86)/test-package-stage/reftest/; \ $(TOPSRCDIR)/build/macosx/universal/unify \ + --unify-with-sort "all-test-dirs\.list$$" \ $(DIST_PPC)/test-package-stage \ $(DIST_X86)/test-package-stage \ $(DIST_UNI)/test-package-stage; fi diff --git a/build/macosx/universal/unify b/build/macosx/universal/unify index adbdeca44054..4730e6cc1bdf 100755 --- a/build/macosx/universal/unify +++ b/build/macosx/universal/unify @@ -53,6 +53,7 @@ I [B<--dry-run>] [B<--only-one> I] [B<--verbosity> I] +[B<--unify-with-sort> I] =head1 DESCRIPTION @@ -134,6 +135,13 @@ I are: The default I is 2. +=item B<--unify-with-sort> I + +Allows merging files matching I that differ only by the ordering +of the lines contained within them. The unified file will have its contents +sorted. This option may be given multiple times to specify multiple +regexes for matching files. + =back =head1 EXAMPLES @@ -183,13 +191,17 @@ use File::Compare; use File::Copy; use Getopt::Long; -my (%gConfig, $gDryRun, $gOnlyOne, $gVerbosity); +my (%gConfig, $gDryRun, $gOnlyOne, $gVerbosity, @gSortMatches); sub argumentEscape(@); sub command(@); sub compareZipArchives($$); sub complain($$@); sub copyIfIdentical($$$); +sub slurp($); +sub compare_sorted($$); +sub copy_sorted($$); +sub copyIfIdenticalWhenSorted($$$); sub createUniqueFile($$); sub makeUniversal($$$); sub makeUniversalDirectory($$$); @@ -228,12 +240,14 @@ sub readZipCRCs($); $gDryRun = 0; $gOnlyOne = 'copy'; $gVerbosity = 2; +@gSortMatches = (); Getopt::Long::Configure('pass_through'); -GetOptions('dry-run' => \$gDryRun, - 'only-one=s' => \$gOnlyOne, - 'verbosity=i' => \$gVerbosity, - 'config=s' => \%gConfig); # "hidden" option not in usage() +GetOptions('dry-run' => \$gDryRun, + 'only-one=s' => \$gOnlyOne, + 'verbosity=i' => \$gVerbosity, + 'unify-with-sort=s' => \@gSortMatches, + 'config=s' => \%gConfig); # "hidden" option not in usage() if (scalar(@ARGV) != 3 || $gVerbosity < 0 || $gVerbosity > 3 || ($gOnlyOne ne 'skip' && $gOnlyOne ne 'copy' && $gOnlyOne ne 'fail')) { @@ -481,6 +495,116 @@ sub copyIfIdentical($$$) { return 1; } +# slurp($file) +# +# Read the contents of $file into an array and return it. +# Returns undef on error. +sub slurp($) { + my $file = $_[0]; + open FILE, $file or return undef; + my @lines = ; + close FILE; + return @lines; +} + +# compare_sorted($file1, $file2) +# +# Read the contents of both files into arrays, sort the arrays, +# and then compare the two arrays for equality. +# +# Returns 0 if the sorted array contents are equal, or 1 if not. +# Returns undef on error. +sub compare_sorted($$) { + my ($file1, $file2) = @_; + my @lines1 = sort(slurp($file1)); + my @lines2 = sort(slurp($file2)); + + return undef if !@lines1 || !@lines2; + return 1 unless scalar @lines1 == scalar @lines2; + + for (my $i = 0; $i < scalar @lines1; $i++) { + return 1 if $lines1[$i] ne $lines2[$i]; + } + return 0; +} + +# copy_sorted($source, $destination) +# +# $source and $destination are filenames. Read the contents of $source +# into an array, sort it, and then write the sorted contents to $destination. +# Returns 1 on success, and undef on failure. +sub copy_sorted($$) { + my ($src, $dest) = @_; + my @lines = sort(slurp($src)); + return undef unless @lines; + open FILE, "> $dest" or return undef; + print FILE @lines; + close FILE; + return 1; +} + +# copyIfIdenticalWhenSorted($source1, $source2, $target) +# +# $source1 and $source2 are FileAttrCache objects that are compared, and if +# identical, copied to path string $target. The comparison is done by +# sorting the individual lines within the two files and comparing the results. +# +# Returns true on success, false for files that are not equivalent, +# and undef if an error occurs. +sub copyIfIdenticalWhenSorted($$$) { + my ($source1, $source2, $target); + ($source1, $source2, $target) = @_; + + if ($gVerbosity >= 3 || $gDryRun) { + print('cmp -s '. + join(' ',argumentEscape($source1->path(), $source2->path()))."\n"); + } + my ($comparison); + if (!defined($comparison = compare_sorted($source1->path(), + $source2->path())) || + $comparison == -1) { + return complain(1, 'copyIfIdenticalWhenSorted: compare: '.$! + .' while comparing:', + $source1->path(), + $source2->path()); + } + if ($comparison != 0) { + return complain(1, 'copyIfIdenticalWhenSorted: files differ:', + $source1->path(), + $source2->path()); + } + + if ($gVerbosity >= 3 || $gDryRun) { + print('cp '. + join(' ',argumentEscape($source1->path(), $target))."\n"); + } + + if (!$gDryRun) { + my ($isExecutable); + + # Set the execute bits (as allowed by the umask) on the new file if any + # execute bit is set on either old file. + $isExecutable = $source1->lIsExecutable() || + (defined($source2) && $source2->lIsExecutable()); + + if (!createUniqueFile($target, $isExecutable ? 0777 : 0666)) { + # createUniqueFile printed an error. + return 0; + } + + if (!copy_sorted($source1->path(), $target)) { + complain(1, 'copyIfIdenticalWhenSorted: copy_sorted: '.$! + .' while copying', + $source1->path(), + $target); + unlink($target); + return 0; + } + } + + return 1; +} + # createUniqueFile($path, $mode) # # Creates a new plain empty file at pathname $path, provided it does not @@ -968,6 +1092,12 @@ sub makeUniversalInternal($$$$) { return makeUniversalFile($filePPC, $fileX86, $fileTargetPath); } + if (grep { $filePPC->path() =~ m/$_/; } @gSortMatches) { + # Regular files, but should be compared with sorting first. + # copyIfIdenticalWhenSorted will print an error if it fails. + return copyIfIdenticalWhenSorted($filePPC, $fileX86, $fileTargetPath); + } + # Regular file. copyIfIdentical will print an error if it fails. return copyIfIdentical($filePPC, $fileX86, $fileTargetPath); }