зеркало из https://github.com/mozilla/pjs.git
1. mar packaging and build changes - Bug 386760 - directory removal with software update. r=nthomas, r=khuey
This commit is contained in:
Robert Strong
Родитель
a69fc5c197
Коммит
615703f25c
|
|
@ -0,0 +1,64 @@
|
|||
# Any copyright is dedicated to the Public Domain.
|
||||
# http://creativecommons.org/publicdomain/zero/1.0/
|
||||
|
||||
# Creates the precomplete file containing the remove, remove-cc, and rmdir
|
||||
# application update instructions which is used to remove files and directories
|
||||
# that are no longer present in a complete update. The current working directory
|
||||
# is used for the location to enumerate and to create the precomplete file.
|
||||
|
||||
import sys
|
||||
import os
|
||||
|
||||
def get_build_entries(root_path):
|
||||
""" Iterates through the root_path, creating a list for each file and
|
||||
directory. Excludes any path starting with extensions or distribution.
|
||||
"""
|
||||
rel_file_path_set = set()
|
||||
rel_dir_path_set = set()
|
||||
for root, dirs, files in os.walk(root_path):
|
||||
for file_name in files:
|
||||
parent_dir_rel_path = root[len(root_path)+1:]
|
||||
rel_path_file = os.path.join(parent_dir_rel_path, file_name)
|
||||
rel_path_file = rel_path_file.replace("\\", "/")
|
||||
if not (rel_path_file.startswith("distribution/") or
|
||||
rel_path_file.startswith("extensions/")):
|
||||
rel_file_path_set.add(rel_path_file)
|
||||
|
||||
for dir_name in dirs:
|
||||
parent_dir_rel_path = root[len(root_path)+1:]
|
||||
rel_path_dir = os.path.join(parent_dir_rel_path, dir_name)
|
||||
rel_path_dir = rel_path_dir.replace("\\", "/")+"/"
|
||||
if not (rel_path_dir.startswith("distribution/") or
|
||||
rel_path_dir.startswith("extensions/")):
|
||||
rel_dir_path_set.add(rel_path_dir)
|
||||
|
||||
rel_file_path_list = list(rel_file_path_set)
|
||||
rel_file_path_list.sort(reverse=True)
|
||||
rel_dir_path_list = list(rel_dir_path_set)
|
||||
rel_dir_path_list.sort(reverse=True)
|
||||
|
||||
return rel_file_path_list, rel_dir_path_list
|
||||
|
||||
def generate_precomplete():
|
||||
""" Creates the precomplete file containing the remove, remove-cc, and rmdir
|
||||
application update instructions. The current working directory is used
|
||||
for the location to enumerate and to create the precomplete file.
|
||||
"""
|
||||
root_path = os.getcwd()
|
||||
rel_file_path_list, rel_dir_path_list = get_build_entries(root_path)
|
||||
precomplete_file_path = os.path.join(root_path,"precomplete")
|
||||
# open in binary mode to prevent OS specific line endings.
|
||||
precomplete_file = open(precomplete_file_path, "wb")
|
||||
for rel_file_path in rel_file_path_list:
|
||||
if rel_file_path.endswith("channel-prefs.js"):
|
||||
precomplete_file.writelines("remove-cc \""+rel_file_path+"\"\n")
|
||||
else:
|
||||
precomplete_file.writelines("remove \""+rel_file_path+"\"\n")
|
||||
|
||||
for rel_dir_path in rel_dir_path_list:
|
||||
precomplete_file.writelines("rmdir \""+rel_dir_path+"\"\n")
|
||||
|
||||
precomplete_file.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
generate_precomplete()
|
||||
|
|
@ -23,6 +23,7 @@
|
|||
# Benjamin Smedberg <bsmedberg@covad.net>
|
||||
# Arthur Wiebe <artooro@gmail.com>
|
||||
# Mark Mentovai <mark@moxienet.com>
|
||||
# Robert Strong <robert.bugzilla@gmail.com>
|
||||
#
|
||||
# Alternatively, the contents of this file may be used under the terms of
|
||||
# either of the GNU General Public License Version 2 or later (the "GPL"),
|
||||
|
|
@ -378,6 +379,8 @@ endif
|
|||
MAKE_SDK = $(CREATE_FINAL_TAR) - $(MOZ_APP_NAME)-sdk | bzip2 -vf > $(SDK)
|
||||
endif
|
||||
|
||||
CREATE_PRECOMPLETE = $(PYTHON) $(MOZILLA_DIR)/config/createprecomplete.py
|
||||
|
||||
ifdef MOZ_OMNIJAR
|
||||
GENERATE_CACHE ?= true
|
||||
|
||||
|
|
@ -419,10 +422,11 @@ UNPACK_OMNIJAR = \
|
|||
sed -e 's/^\#binary-component/binary-component/' components/components.manifest > components.manifest && \
|
||||
mv components.manifest components
|
||||
|
||||
MAKE_PACKAGE = (cd $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(PACK_OMNIJAR)) && $(INNER_MAKE_PACKAGE)
|
||||
MAKE_PACKAGE = (cd $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(PACK_OMNIJAR)) && \
|
||||
(cd $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(CREATE_PRECOMPLETE)) && $(INNER_MAKE_PACKAGE)
|
||||
UNMAKE_PACKAGE = $(INNER_UNMAKE_PACKAGE) && (cd $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(UNPACK_OMNIJAR))
|
||||
else
|
||||
MAKE_PACKAGE = $(INNER_MAKE_PACKAGE)
|
||||
MAKE_PACKAGE = (cd $(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(CREATE_PRECOMPLETE)) && $(INNER_MAKE_PACKAGE)
|
||||
UNMAKE_PACKAGE = $(INNER_UNMAKE_PACKAGE)
|
||||
endif
|
||||
|
||||
|
|
@ -565,6 +569,7 @@ ifdef MOZ_OMNIJAR
|
|||
@(cd $(DIST)/$(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH) && $(PACK_OMNIJAR))
|
||||
endif
|
||||
@cp -av $(DIST)/$(STAGEPATH)$(MOZ_PKG_DIR)$(_BINPATH)/. $(DEPTH)/installer-stage/core
|
||||
@(cd $(DEPTH)/installer-stage/core && $(CREATE_PRECOMPLETE))
|
||||
ifdef MOZ_OPTIONAL_PKG_LIST
|
||||
@$(NSINSTALL) -D $(DEPTH)/installer-stage/optional
|
||||
$(call PACKAGER_COPY, "$(call core_abspath,$(DIST))",\
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ MBSDIFF=${MBSDIFF:-mbsdiff}
|
|||
# Helper routines
|
||||
|
||||
notice() {
|
||||
echo $* 1>&2
|
||||
echo "$*" 1>&2
|
||||
}
|
||||
|
||||
get_file_size() {
|
||||
|
|
@ -35,13 +35,23 @@ copy_perm() {
|
|||
|
||||
make_add_instruction() {
|
||||
f="$1"
|
||||
|
||||
# Used to log to the console
|
||||
if [ $2 ]; then
|
||||
forced=" (forced)"
|
||||
else
|
||||
forced=
|
||||
fi
|
||||
|
||||
is_extension=$(echo "$f" | grep -c 'extensions/.*/')
|
||||
if [ $is_extension = "1" ]; then
|
||||
# Use the subdirectory of the extensions folder as the file to test
|
||||
# before performing this add instruction.
|
||||
testdir=$(echo "$f" | sed 's/\(extensions\/[^\/]*\)\/.*/\1/')
|
||||
notice " add-if: $f$forced"
|
||||
echo "add-if \"$testdir\" \"$f\""
|
||||
else
|
||||
notice " add: $f$forced"
|
||||
echo "add \"$f\""
|
||||
fi
|
||||
}
|
||||
|
|
@ -54,16 +64,21 @@ make_patch_instruction() {
|
|||
# Use the subdirectory of the extensions folder as the file to test
|
||||
# before performing this add instruction.
|
||||
testdir=$(echo "$f" | sed 's/\(extensions\/[^\/]*\)\/.*/\1/')
|
||||
notice " patch-if: $f"
|
||||
echo "patch-if \"$testdir\" \"$f.patch\" \"$f\""
|
||||
elif [ $is_search_plugin = "1" ]; then
|
||||
notice " patch-if: $f"
|
||||
echo "patch-if \"$f\" \"$f.patch\" \"$f\""
|
||||
else
|
||||
notice " patch: $f"
|
||||
echo "patch \"$f.patch\" \"$f\""
|
||||
fi
|
||||
}
|
||||
|
||||
append_remove_instructions() {
|
||||
dir="$1"
|
||||
filev1="$2"
|
||||
filev2="$3"
|
||||
if [ -f "$dir/removed-files" ]; then
|
||||
prefix=
|
||||
listfile="$dir/removed-files"
|
||||
|
|
@ -73,18 +88,43 @@ append_remove_instructions() {
|
|||
fi
|
||||
if [ -n "$listfile" ]; then
|
||||
# Map spaces to pipes so that we correctly handle filenames with spaces.
|
||||
files=($(cat "$listfile" | tr " " "|"))
|
||||
files=($(cat "$listfile" | tr " " "|" | sort -r))
|
||||
num_files=${#files[*]}
|
||||
for ((i=0; $i<$num_files; i=$i+1)); do
|
||||
# Trim whitespace (including trailing carriage returns)
|
||||
f=$(echo ${files[$i]} | tr "|" " " | sed 's/^ *\(.*\) *$/\1/' | tr -d '\r')
|
||||
# Exclude any blank lines or any lines ending with a slash, which indicate
|
||||
# directories. The updater doesn't know how to remove entire directories.
|
||||
# Map pipes back to whitespace and remove carriage returns
|
||||
f=$(echo ${files[$i]} | tr "|" " " | tr -d '\r')
|
||||
# Trim whitespace
|
||||
f=$(echo $f)
|
||||
# Exclude blank lines.
|
||||
if [ -n "$f" ]; then
|
||||
if [ $(echo "$f" | grep -c '\/$') = 0 ]; then
|
||||
echo "remove \"$prefix$f\""
|
||||
else
|
||||
notice "ignoring remove instruction for directory: $f"
|
||||
# Exclude comments
|
||||
if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then
|
||||
# Normalize the path to the root of the Mac OS X bundle if necessary
|
||||
fixedprefix="$prefix"
|
||||
if [ $prefix ]; then
|
||||
if [ $(echo "$f" | grep -c '^\.\./') = 1 ]; then
|
||||
if [ $(echo "$f" | grep -c '^\.\./\.\./') = 1 ]; then
|
||||
f=$(echo $f | sed -e 's:^\.\.\/\.\.\/::')
|
||||
fixedprefix=""
|
||||
else
|
||||
f=$(echo $f | sed -e 's:^\.\.\/::')
|
||||
fixedprefix=$(echo "$prefix" | sed -e 's:^[^\/]*\/::')
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
if [ $(echo "$f" | grep -c '\/$') = 1 ]; then
|
||||
notice " rmdir: $fixedprefix$f"
|
||||
echo "rmdir \"$fixedprefix$f\"" >> $filev2
|
||||
elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then
|
||||
# Remove the *
|
||||
f=$(echo "$f" | sed -e 's:\*$::')
|
||||
notice " rmrfdir: $fixedprefix$f"
|
||||
echo "rmrfdir \"$fixedprefix$f\"" >> $filev2
|
||||
else
|
||||
notice " remove: $fixedprefix$f"
|
||||
echo "remove \"$fixedprefix$f\"" >> $filev1
|
||||
echo "remove \"$fixedprefix$f\"" >> $filev2
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
|
@ -101,12 +141,30 @@ list_files() {
|
|||
find . -type f \
|
||||
! -name "channel-prefs.js" \
|
||||
! -name "update.manifest" \
|
||||
! -name "updatev2.manifest" \
|
||||
! -name "temp-dirlist" \
|
||||
! -name "temp-filelist" \
|
||||
| sed 's/\.\/\(.*\)/\1/' \
|
||||
| sort > "temp-filelist"
|
||||
| sort -r > "temp-filelist"
|
||||
while read file; do
|
||||
eval "${1}[$count]=\"$file\""
|
||||
(( count++ ))
|
||||
done < "temp-filelist"
|
||||
rm "temp-filelist"
|
||||
}
|
||||
|
||||
# List all directories in the current directory, stripping leading "./"
|
||||
list_dirs() {
|
||||
count=0
|
||||
|
||||
find . -type d \
|
||||
! -name "." \
|
||||
! -name ".." \
|
||||
| sed 's/\.\/\(.*\)/\1/' \
|
||||
| sort -r > "temp-dirlist"
|
||||
while read dir; do
|
||||
eval "${1}[$count]=\"$dir\""
|
||||
(( count++ ))
|
||||
done < "temp-dirlist"
|
||||
rm "temp-dirlist"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,38 +32,50 @@ fi
|
|||
|
||||
archive="$1"
|
||||
targetdir="$2"
|
||||
# Prevent the workdir from being inside the targetdir so it isn't included in
|
||||
# the update mar.
|
||||
if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then
|
||||
# Remove the /
|
||||
targetdir=$(echo "$targetdir" | sed -e 's:\/$::')
|
||||
fi
|
||||
workdir="$targetdir.work"
|
||||
manifest="$workdir/update.manifest"
|
||||
targetfiles="update.manifest"
|
||||
updatemanifestv1="$workdir/update.manifest"
|
||||
updatemanifestv2="$workdir/updatev2.manifest"
|
||||
targetfiles="update.manifest updatev2.manifest"
|
||||
|
||||
mkdir -p "$workdir"
|
||||
|
||||
# On Mac, the precomplete file added by Bug 386760 will cause OS X to reload the
|
||||
# Info.plist so it launches the right architecture, bug 600098
|
||||
|
||||
# Generate a list of all files in the target directory.
|
||||
pushd "$targetdir"
|
||||
if test $? -ne 0 ; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# On Mac, force a top-level file so that OS X reloads the Info.plist
|
||||
# and launches the right architecture for the OS version, bug 600098
|
||||
if [[ -d Contents ]]; then
|
||||
touch force_plist_reload
|
||||
if [ ! -f "precomplete" ]; then
|
||||
notice "precomplete file is missing!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
list_files files
|
||||
|
||||
# Files that should be added on channel change
|
||||
ccfiles=$(find . -type f -name "channel-prefs.js" | sed 's/\.\/\(.*\)/\1/')
|
||||
|
||||
popd
|
||||
|
||||
> $manifest
|
||||
notice ""
|
||||
notice "Adding file add instructions to file 'update.manifest'"
|
||||
> $updatemanifestv1
|
||||
|
||||
num_files=${#files[*]}
|
||||
|
||||
for ((i=0; $i<$num_files; i=$i+1)); do
|
||||
f="${files[$i]}"
|
||||
|
||||
notice "processing $f"
|
||||
|
||||
make_add_instruction "$f" >> $manifest
|
||||
make_add_instruction "$f" >> $updatemanifestv1
|
||||
|
||||
dir=$(dirname "$f")
|
||||
mkdir -p "$workdir/$dir"
|
||||
|
|
@ -73,13 +85,44 @@ for ((i=0; $i<$num_files; i=$i+1)); do
|
|||
targetfiles="$targetfiles \"$f\""
|
||||
done
|
||||
|
||||
# Append remove instructions for any dead files.
|
||||
append_remove_instructions "$targetdir" >> $manifest
|
||||
# Add the type of update to the beginning of and cat the contents of the version
|
||||
# 1 update manifest to the version 2 update manifest.
|
||||
> $updatemanifestv2
|
||||
notice ""
|
||||
notice "Adding type instruction to file 'updatev2.manifest'"
|
||||
notice " type: complete"
|
||||
echo "type \"complete\"" >> $updatemanifestv2
|
||||
|
||||
$BZIP2 -z9 "$manifest" && mv -f "$manifest.bz2" "$manifest"
|
||||
notice ""
|
||||
notice "Adding file ADD instructions for channel change to file 'updatev2.manifest'"
|
||||
for f in $ccfiles; do
|
||||
notice " add-cc: $f"
|
||||
echo "add-cc \"$f\"" >> $updatemanifestv2
|
||||
dir=$(dirname "$f")
|
||||
mkdir -p "$workdir/$dir"
|
||||
$BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
|
||||
copy_perm "$targetdir/$f" "$workdir/$f"
|
||||
|
||||
targetfiles="$targetfiles \"$f\""
|
||||
done
|
||||
|
||||
notice ""
|
||||
notice "Concatenating file 'update.manifest' to file 'updatev2.manifest'"
|
||||
cat $updatemanifestv1 >> $updatemanifestv2
|
||||
|
||||
# Append remove instructions for any dead files.
|
||||
notice ""
|
||||
notice "Adding file and directory remove instructions from file 'removed-files'"
|
||||
append_remove_instructions "$targetdir" "$updatemanifestv1" "$updatemanifestv2"
|
||||
|
||||
$BZIP2 -z9 "$updatemanifestv1" && mv -f "$updatemanifestv1.bz2" "$updatemanifestv1"
|
||||
$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
|
||||
|
||||
eval "$MAR -C \"$workdir\" -c output.mar $targetfiles"
|
||||
mv -f "$workdir/output.mar" "$archive"
|
||||
|
||||
# cleanup
|
||||
rm -fr "$workdir"
|
||||
|
||||
notice ""
|
||||
notice "Finished"
|
||||
|
|
|
|||
|
|
@ -26,6 +26,11 @@ check_for_forced_update() {
|
|||
|
||||
local f
|
||||
|
||||
if [ "$forced_file_chk" = "precomplete" ]; then
|
||||
## "true" *giggle*
|
||||
return 0;
|
||||
fi
|
||||
|
||||
if [ "${forced_file_chk##*.}" = "chk" ]
|
||||
then
|
||||
## "true" *giggle*
|
||||
|
|
@ -70,12 +75,22 @@ shift $arg_start
|
|||
archive="$1"
|
||||
olddir="$2"
|
||||
newdir="$3"
|
||||
# Prevent the workdir from being inside the targetdir so it isn't included in
|
||||
# the update mar.
|
||||
if [ $(echo "$newdir" | grep -c '\/$') = 1 ]; then
|
||||
# Remove the /
|
||||
newdir=$(echo "$newdir" | sed -e 's:\/$::')
|
||||
fi
|
||||
workdir="$newdir.work"
|
||||
manifest="$workdir/update.manifest"
|
||||
archivefiles="update.manifest"
|
||||
updatemanifestv1="$workdir/update.manifest"
|
||||
updatemanifestv2="$workdir/updatev2.manifest"
|
||||
archivefiles="update.manifest updatev2.manifest"
|
||||
|
||||
mkdir -p "$workdir"
|
||||
|
||||
# On Mac, the precomplete file added by Bug 386760 will cause OS X to reload the
|
||||
# Info.plist so it launches the right architecture, bug 600098
|
||||
|
||||
# Generate a list of all files in the target directory.
|
||||
pushd "$olddir"
|
||||
if test $? -ne 0 ; then
|
||||
|
|
@ -83,6 +98,7 @@ if test $? -ne 0 ; then
|
|||
fi
|
||||
|
||||
list_files oldfiles
|
||||
list_dirs olddirs
|
||||
|
||||
popd
|
||||
|
||||
|
|
@ -91,13 +107,26 @@ if test $? -ne 0 ; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "precomplete" ]; then
|
||||
notice "precomplete file is missing!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
list_dirs newdirs
|
||||
list_files newfiles
|
||||
|
||||
# Files that should be added on channel change
|
||||
ccfiles=$(find . -type f -name "channel-prefs.js" | sed 's/\.\/\(.*\)/\1/')
|
||||
|
||||
popd
|
||||
|
||||
> $manifest
|
||||
notice ""
|
||||
notice "Adding file patch and add instructions to file 'update.manifest'"
|
||||
> $updatemanifestv1
|
||||
|
||||
num_oldfiles=${#oldfiles[*]}
|
||||
remove_array=
|
||||
num_removes=0
|
||||
|
||||
for ((i=0; $i<$num_oldfiles; i=$i+1)); do
|
||||
f="${oldfiles[$i]}"
|
||||
|
|
@ -107,16 +136,21 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do
|
|||
continue 1
|
||||
fi
|
||||
|
||||
# removed-files is excluded by make_incremental_updates.py so it is excluded
|
||||
# here for consistency.
|
||||
if [ `basename $f` = "removed-files" ]; then
|
||||
continue 1
|
||||
fi
|
||||
|
||||
# If this file exists in the new directory as well, then check if it differs.
|
||||
if [ -f "$newdir/$f" ]; then
|
||||
|
||||
if check_for_forced_update "$requested_forced_updates" "$f"; then
|
||||
echo 1>&2 " FORCING UPDATE for file '$f'..."
|
||||
# The full workdir may not exist yet, so create it if necessary.
|
||||
# The full workdir may not exist yet, so create it if necessary.
|
||||
mkdir -p `dirname "$workdir/$f"`
|
||||
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
|
||||
copy_perm "$newdir/$f" "$workdir/$f"
|
||||
make_add_instruction "$f" >> $manifest
|
||||
make_add_instruction "$f" 1 >> $updatemanifestv1
|
||||
archivefiles="$archivefiles \"$f\""
|
||||
continue 1
|
||||
fi
|
||||
|
|
@ -124,7 +158,6 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do
|
|||
if ! diff "$olddir/$f" "$newdir/$f" > /dev/null; then
|
||||
# Compute both the compressed binary diff and the compressed file, and
|
||||
# compare the sizes. Then choose the smaller of the two to package.
|
||||
echo " diffing $f"
|
||||
dir=$(dirname "$workdir/$f")
|
||||
mkdir -p "$dir"
|
||||
$MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
|
||||
|
|
@ -135,52 +168,102 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do
|
|||
patchsize=$(get_file_size "$patchfile")
|
||||
fullsize=$(get_file_size "$workdir/$f")
|
||||
|
||||
if [ $patchsize -lt $fullsize -a "$f" != "removed-files" ]; then
|
||||
make_patch_instruction "$f" >> $manifest
|
||||
if [ $patchsize -lt $fullsize ]; then
|
||||
make_patch_instruction "$f" >> $updatemanifestv1
|
||||
mv -f "$patchfile" "$workdir/$f.patch"
|
||||
rm -f "$workdir/$f"
|
||||
archivefiles="$archivefiles \"$f.patch\""
|
||||
else
|
||||
make_add_instruction "$f" >> $manifest
|
||||
make_add_instruction "$f" >> $updatemanifestv1
|
||||
rm -f "$patchfile"
|
||||
archivefiles="$archivefiles \"$f\""
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo "remove \"$f\"" >> $manifest
|
||||
# remove instructions are added after add / patch instructions for
|
||||
# consistency with make_incremental_updates.py
|
||||
remove_array[$num_removes]=$f
|
||||
(( num_removes++ ))
|
||||
fi
|
||||
done
|
||||
|
||||
# Now, we just need to worry about newly added files
|
||||
# Newly added files
|
||||
notice ""
|
||||
notice "Adding file add instructions to file 'update.manifest'"
|
||||
num_newfiles=${#newfiles[*]}
|
||||
|
||||
for ((i=0; $i<$num_newfiles; i=$i+1)); do
|
||||
f="${newfiles[$i]}"
|
||||
|
||||
# removed-files is excluded by make_incremental_updates.py so it is excluded
|
||||
# here for consistency.
|
||||
if [ `basename $f` = "removed-files" ]; then
|
||||
continue 1
|
||||
fi
|
||||
|
||||
# If we've already tested this file, then skip it
|
||||
for ((j=0; $j<$num_oldfiles; j=$j+1)); do
|
||||
if [ "$f" = "${oldfiles[j]}" ]; then
|
||||
continue 2
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
dir=$(dirname "$workdir/$f")
|
||||
mkdir -p "$dir"
|
||||
|
||||
$BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
|
||||
copy_perm "$newdir/$f" "$workdir/$f"
|
||||
|
||||
make_add_instruction "$f" >> "$manifest"
|
||||
make_add_instruction "$f" >> "$updatemanifestv1"
|
||||
archivefiles="$archivefiles \"$f\""
|
||||
done
|
||||
|
||||
# Append remove instructions for any dead files.
|
||||
append_remove_instructions "$newdir" >> $manifest
|
||||
notice ""
|
||||
notice "Adding file remove instructions to file 'update.manifest'"
|
||||
for ((i=0; $i<$num_removes; i=$i+1)); do
|
||||
f="${remove_array[$i]}"
|
||||
notice " remove: $f"
|
||||
echo "remove \"$f\"" >> $updatemanifestv1
|
||||
done
|
||||
|
||||
$BZIP2 -z9 "$manifest" && mv -f "$manifest.bz2" "$manifest"
|
||||
# Add the type of update to the beginning of and cat the contents of the version
|
||||
# 1 update manifest to the version 2 update manifest.
|
||||
notice ""
|
||||
notice "Adding type instruction to file 'updatev2.manifest'"
|
||||
> $updatemanifestv2
|
||||
notice " type: partial"
|
||||
echo "type \"partial\"" >> $updatemanifestv2
|
||||
|
||||
notice ""
|
||||
notice "Concatenating file 'update.manifest' to file 'updatev2.manifest'"
|
||||
cat $updatemanifestv1 >> $updatemanifestv2
|
||||
|
||||
# Append remove instructions for any dead files.
|
||||
notice ""
|
||||
notice "Adding file and directory remove instructions from file 'removed-files'"
|
||||
append_remove_instructions "$newdir" "$updatemanifestv1" "$updatemanifestv2"
|
||||
|
||||
notice ""
|
||||
notice "Adding directory remove instructions for directories that no longer exist"
|
||||
num_olddirs=${#olddirs[*]}
|
||||
|
||||
for ((i=0; $i<$num_olddirs; i=$i+1)); do
|
||||
f="${olddirs[$i]}"
|
||||
# If this dir doesn't exist in the new directory remove it.
|
||||
if [ ! -d "$newdir/$f" ]; then
|
||||
notice " rmdir: $f/"
|
||||
echo "rmdir \"$f/\"" >> $updatemanifestv2
|
||||
fi
|
||||
done
|
||||
|
||||
$BZIP2 -z9 "$updatemanifestv1" && mv -f "$updatemanifestv1.bz2" "$updatemanifestv1"
|
||||
$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
|
||||
|
||||
eval "$MAR -C \"$workdir\" -c output.mar $archivefiles"
|
||||
mv -f "$workdir/output.mar" "$archive"
|
||||
|
||||
# cleanup
|
||||
rm -fr "$workdir"
|
||||
|
||||
notice ""
|
||||
notice "Finished"
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
import os
|
||||
import shutil
|
||||
import sha
|
||||
import sha
|
||||
from os.path import join, getsize
|
||||
from stat import *
|
||||
import re
|
||||
|
|
@ -17,7 +16,8 @@ class PatchInfo:
|
|||
""" Represents the meta-data associated with a patch
|
||||
work_dir = working dir where files are stored for this patch
|
||||
archive_files = list of files to include in this patch
|
||||
manifest = set of patch instructions
|
||||
manifestv1 = set of manifest version 1 patch instructions
|
||||
manifestv2 = set of manifest version 2 patch instructions
|
||||
file_exclusion_list =
|
||||
files to exclude from this patch. names without slashes will be
|
||||
excluded anywhere in the directory hiearchy. names with slashes
|
||||
|
|
@ -26,7 +26,8 @@ class PatchInfo:
|
|||
def __init__(self, work_dir, file_exclusion_list, path_exclusion_list):
|
||||
self.work_dir=work_dir
|
||||
self.archive_files=[]
|
||||
self.manifest=[]
|
||||
self.manifestv1=[]
|
||||
self.manifestv2=[]
|
||||
self.file_exclusion_list=file_exclusion_list
|
||||
self.path_exclusion_list=path_exclusion_list
|
||||
|
||||
|
|
@ -39,12 +40,15 @@ class PatchInfo:
|
|||
if filename.startswith("extensions/"):
|
||||
# Directory immediately following extensions is used for the test
|
||||
testdir = "extensions/"+filename.split("/")[1]
|
||||
self.manifest.append('add-if "'+testdir+'" "'+filename+'"')
|
||||
self.manifestv1.append('add-if "'+testdir+'" "'+filename+'"')
|
||||
self.manifestv2.append('add-if "'+testdir+'" "'+filename+'"')
|
||||
elif filename.startswith("Contents/MacOS/extensions/"):
|
||||
testdir = "Contents/MacOS/extensions/"+filename.split("/")[3]
|
||||
self.manifest.append('add-if "'+testdir+'" "'+filename+'"')
|
||||
self.manifestv1.append('add-if "'+testdir+'" "'+filename+'"')
|
||||
self.manifestv2.append('add-if "'+testdir+'" "'+filename+'"')
|
||||
else:
|
||||
self.manifest.append('add "'+filename+'"')
|
||||
self.manifestv1.append('add "'+filename+'"')
|
||||
self.manifestv2.append('add "'+filename+'"')
|
||||
|
||||
def append_patch_instruction(self, filename, patchname):
|
||||
""" Appends an patch instruction for this patch.
|
||||
|
|
@ -60,51 +64,84 @@ class PatchInfo:
|
|||
"""
|
||||
if filename.startswith("extensions/"):
|
||||
testdir = "extensions/"+filename.split("/")[1]
|
||||
self.manifest.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
|
||||
self.manifestv1.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
|
||||
self.manifestv2.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
|
||||
elif filename.startswith("Contents/MacOS/extensions/"):
|
||||
testdir = "Contents/MacOS/extensions/"+filename.split("/")[3]
|
||||
self.manifest.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
|
||||
self.manifestv1.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
|
||||
self.manifestv2.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
|
||||
elif (filename.startswith("searchplugins/") or
|
||||
filename.startswith("Contents/MacOS/searchplugins/")):
|
||||
self.manifest.append('patch-if "'+filename+'" "'+patchname+'" "'+filename+'"')
|
||||
self.manifestv1.append('patch-if "'+filename+'" "'+patchname+'" "'+filename+'"')
|
||||
self.manifestv2.append('patch-if "'+filename+'" "'+patchname+'" "'+filename+'"')
|
||||
else:
|
||||
self.manifest.append('patch "'+patchname+'" "'+filename+'"')
|
||||
self.manifestv1.append('patch "'+patchname+'" "'+filename+'"')
|
||||
self.manifestv2.append('patch "'+patchname+'" "'+filename+'"')
|
||||
|
||||
def append_remove_instruction(self, filename):
|
||||
""" Appends an remove instruction for this patch.
|
||||
This was ported from
|
||||
mozilla/tools/update-packaging/common.sh/make_remove_instruction
|
||||
"""
|
||||
self.manifest.append('remove "'+filename+'"')
|
||||
if filename.endswith("/"):
|
||||
self.manifestv2.append('rmdir "'+filename+'"')
|
||||
elif filename.endswith("/*"):
|
||||
filename = filename[:-1]
|
||||
self.manifestv2.append('rmrfdir "'+filename+'"')
|
||||
else:
|
||||
self.manifestv1.append('remove "'+filename+'"')
|
||||
self.manifestv2.append('remove "'+filename+'"')
|
||||
|
||||
def create_manifest_file(self):
|
||||
""" Createst the manifest file into to the root of the work_dir """
|
||||
def create_manifest_files(self):
|
||||
""" Createst the v1 manifest file in the root of the work_dir """
|
||||
manifest_file_path = os.path.join(self.work_dir,"update.manifest")
|
||||
manifest_file = open(manifest_file_path, "w")
|
||||
manifest_file.writelines(string.join(self.manifest, '\n'))
|
||||
manifest_file = open(manifest_file_path, "wb")
|
||||
manifest_file.writelines(string.join(self.manifestv1, '\n'))
|
||||
manifest_file.writelines("\n")
|
||||
manifest_file.close()
|
||||
|
||||
bzip_file(manifest_file_path)
|
||||
self.archive_files.append('"update.manifest"')
|
||||
|
||||
""" Createst the v2 manifest file in the root of the work_dir """
|
||||
manifest_file_path = os.path.join(self.work_dir,"updatev2.manifest")
|
||||
manifest_file = open(manifest_file_path, "wb")
|
||||
manifest_file.writelines("type \"partial\"\n")
|
||||
manifest_file.writelines(string.join(self.manifestv2, '\n'))
|
||||
manifest_file.writelines("\n")
|
||||
manifest_file.close()
|
||||
|
||||
bzip_file(manifest_file_path)
|
||||
self.archive_files.append('"updatev2.manifest"')
|
||||
|
||||
def build_marfile_entry_hash(self, root_path):
|
||||
""" Iterates through the root_path, creating a MarFileEntry for each file
|
||||
in that path. Excluseds any filenames in the file_exclusion_list
|
||||
"""
|
||||
mar_entry_hash = {}
|
||||
filename_set = set()
|
||||
for root, dirs, files in os.walk(root_path):
|
||||
for name in files:
|
||||
# filename is relative path from root directory
|
||||
partial_path = root[len(root_path)+1:]
|
||||
if name not in self.file_exclusion_list:
|
||||
filename = os.path.join(partial_path, name)
|
||||
if "/"+filename not in self.path_exclusion_list:
|
||||
mar_entry_hash[filename]=MarFileEntry(root_path, filename)
|
||||
filename_set.add(filename)
|
||||
return mar_entry_hash, filename_set
|
||||
""" Iterates through the root_path, creating a MarFileEntry for each file
|
||||
and directory in that path. Excludes any filenames in the file_exclusion_list
|
||||
"""
|
||||
mar_entry_hash = {}
|
||||
filename_set = set()
|
||||
dirname_set = set()
|
||||
for root, dirs, files in os.walk(root_path):
|
||||
for name in files:
|
||||
# filename is the relative path from root directory
|
||||
partial_path = root[len(root_path)+1:]
|
||||
if name not in self.file_exclusion_list:
|
||||
filename = os.path.join(partial_path, name)
|
||||
if "/"+filename not in self.path_exclusion_list:
|
||||
mar_entry_hash[filename]=MarFileEntry(root_path, filename)
|
||||
filename_set.add(filename)
|
||||
|
||||
for name in dirs:
|
||||
# dirname is the relative path from root directory
|
||||
partial_path = root[len(root_path)+1:]
|
||||
if name not in self.file_exclusion_list:
|
||||
dirname = os.path.join(partial_path, name)
|
||||
if "/"+dirname not in self.path_exclusion_list:
|
||||
dirname = dirname+"/"
|
||||
mar_entry_hash[dirname]=MarFileEntry(root_path, dirname)
|
||||
dirname_set.add(dirname)
|
||||
|
||||
return mar_entry_hash, filename_set, dirname_set
|
||||
|
||||
|
||||
class MarFileEntry:
|
||||
|
|
@ -118,7 +155,7 @@ class MarFileEntry:
|
|||
def __init__(self, root, name):
|
||||
"""root = path the the top of the mar
|
||||
name = relative path within the mar"""
|
||||
self.name=name
|
||||
self.name=name.replace("\\", "/")
|
||||
self.abs_path=os.path.join(root,name)
|
||||
self.sha_cache=None
|
||||
|
||||
|
|
@ -219,10 +256,10 @@ def create_partial_patch_for_file(from_marfile_entry, to_marfile_entry, shas, pa
|
|||
shas[from_marfile_entry.sha(),to_marfile_entry.sha()] = (file_in_manifest_name,file_in_manifest_abspath)
|
||||
patch_info.archive_files.append('"'+file_in_manifest_name+'"')
|
||||
else:
|
||||
print "skipping diff: " + from_marfile_entry.name
|
||||
filename, src_file_abs_path = shas[from_marfile_entry.sha(),to_marfile_entry.sha()]
|
||||
# We've already calculated the patch for this pair of files.
|
||||
if (filename.endswith(".patch")):
|
||||
print "skipping diff: " + from_marfile_entry.name
|
||||
# Patch was smaller than file - add patch instruction to manifest
|
||||
file_in_manifest_name = to_marfile_entry.name+'.patch';
|
||||
patch_info.append_patch_instruction(to_marfile_entry.name, file_in_manifest_name)
|
||||
|
|
@ -250,62 +287,78 @@ def process_explicit_remove_files(dir_path, patch_info):
|
|||
list_file_path = os.path.join(dir_path, "removed-files")
|
||||
prefix=""
|
||||
if not os.path.exists(list_file_path):
|
||||
# Mac has is in Contents/MacOS/
|
||||
# On Mac removed-files contains relative paths from Contents/MacOS/
|
||||
prefix= "Contents/MacOS"
|
||||
list_file_path = os.path.join(dir_path, prefix+"/removed-files")
|
||||
|
||||
if (os.path.exists(list_file_path)):
|
||||
list_file = bz2.BZ2File(list_file_path,"r") # throws if doesn't exist
|
||||
|
||||
lines = []
|
||||
for line in list_file:
|
||||
line = line.strip()
|
||||
# Exclude any blank lines or any lines ending with a slash, which indicate
|
||||
# directories. The updater doesn't know how to remove entire directories.
|
||||
if line and not line.endswith("/"):
|
||||
patch_info.append_remove_instruction(os.path.join(prefix,line))
|
||||
lines.append(line.strip())
|
||||
|
||||
lines.sort(reverse=True)
|
||||
for line in lines:
|
||||
# Exclude any blank and comment lines.
|
||||
if line and not line.startswith("#"):
|
||||
# Python on windows uses \ for path separators and the update
|
||||
# manifests expects / for path separators on all platforms.
|
||||
line=os.path.join(prefix,line).replace("\\", "/")
|
||||
patch_info.append_remove_instruction(line)
|
||||
|
||||
def create_partial_patch(from_dir_path, to_dir_path, patch_filename, shas, patch_info, forced_updates):
|
||||
""" Builds a partial patch by comparing the files in from_dir_path to thoes of to_dir_path"""
|
||||
# Cannocolize the paths for safey
|
||||
from_dir_path = os.path.abspath(from_dir_path)
|
||||
to_dir_path = os.path.abspath(to_dir_path)
|
||||
# First create a hashtable of the from and to directories
|
||||
from_dir_hash,from_dir_set = patch_info.build_marfile_entry_hash(from_dir_path)
|
||||
to_dir_hash,to_dir_set = patch_info.build_marfile_entry_hash(to_dir_path)
|
||||
# Create a hashtable of the from and to directories
|
||||
from_dir_hash,from_file_set,from_dir_set = patch_info.build_marfile_entry_hash(from_dir_path)
|
||||
to_dir_hash,to_file_set,to_dir_set = patch_info.build_marfile_entry_hash(to_dir_path)
|
||||
# Require that the precomplete file is included in the to complete update
|
||||
if "precomplete" not in to_file_set:
|
||||
raise Exception, "missing precomplete file in: "+to_dir_path
|
||||
# Create a list of the forced updates
|
||||
forced_list = forced_updates.strip().split('|')
|
||||
|
||||
forced_list.append("precomplete")
|
||||
|
||||
# Files which exist in both sets need to be patched
|
||||
patch_filenames = list(from_dir_set.intersection(to_dir_set))
|
||||
patch_filenames.sort()
|
||||
patch_filenames = list(from_file_set.intersection(to_file_set))
|
||||
patch_filenames.sort(reverse=True)
|
||||
for filename in patch_filenames:
|
||||
from_marfile_entry = from_dir_hash[filename]
|
||||
to_marfile_entry = to_dir_hash[filename]
|
||||
if filename in forced_list:
|
||||
print "Forcing "+ filename
|
||||
# This filename is in the forced list, explicitly add
|
||||
create_add_patch_for_file(to_dir_hash[filename], patch_info)
|
||||
create_add_patch_for_file(to_dir_hash[filename], patch_info)
|
||||
else:
|
||||
if from_marfile_entry.sha() != to_marfile_entry.sha():
|
||||
# Not the same - calculate a patch
|
||||
create_partial_patch_for_file(from_marfile_entry, to_marfile_entry, shas, patch_info)
|
||||
|
||||
# files in from_dir not in to_dir need to be removed
|
||||
remove_filenames = list(from_dir_set - to_dir_set)
|
||||
remove_filenames.sort()
|
||||
for filename in remove_filenames:
|
||||
patch_info.append_remove_instruction(from_dir_hash[filename].name)
|
||||
|
||||
# files in to_dir not in from_dir need to added
|
||||
add_filenames = list(to_dir_set - from_dir_set)
|
||||
add_filenames.sort()
|
||||
add_filenames = list(to_file_set - from_file_set)
|
||||
add_filenames.sort(reverse=True)
|
||||
for filename in add_filenames:
|
||||
create_add_patch_for_file(to_dir_hash[filename], patch_info)
|
||||
|
||||
# files in from_dir not in to_dir need to be removed
|
||||
remove_filenames = list(from_file_set - to_file_set)
|
||||
remove_filenames.sort(reverse=True)
|
||||
for filename in remove_filenames:
|
||||
patch_info.append_remove_instruction(from_dir_hash[filename].name)
|
||||
|
||||
process_explicit_remove_files(to_dir_path, patch_info)
|
||||
|
||||
# Construct Manifest file
|
||||
patch_info.create_manifest_file()
|
||||
# directories in from_dir not in to_dir need to be removed
|
||||
remove_dirnames = list(from_dir_set - to_dir_set)
|
||||
remove_dirnames.sort(reverse=True)
|
||||
for dirname in remove_dirnames:
|
||||
patch_info.append_remove_instruction(from_dir_hash[dirname].name)
|
||||
|
||||
# Construct the Manifest files
|
||||
patch_info.create_manifest_files()
|
||||
|
||||
# And construct the mar
|
||||
mar_cmd = 'mar -C '+patch_info.work_dir+' -c output.mar '+string.join(patch_info.archive_files, ' ')
|
||||
|
|
@ -402,7 +455,7 @@ def create_partial_patches(patches):
|
|||
|
||||
mar_extract_time = time.time()
|
||||
|
||||
partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename, shas, PatchInfo(work_dir, ['channel-prefs.js','update.manifest','removed-files'],['/readme.txt']),forced_updates)
|
||||
partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename, shas, PatchInfo(work_dir, ['channel-prefs.js','update.manifest','updatev2.manifest','removed-files'],['/readme.txt']),forced_updates)
|
||||
partial_buildid = to_buildid
|
||||
partial_shasum = sha.sha(open(partial_filename).read()).hexdigest()
|
||||
partial_size = str(os.path.getsize(partial_filename))
|
||||
|
|
|
|||
|
|
@ -1,16 +1,26 @@
|
|||
#!/bin/bash
|
||||
# Builds all the reference mars
|
||||
|
||||
rm ref.mar
|
||||
rm ref-mac.mar
|
||||
if [ -f "ref.mar" ]; then
|
||||
rm "ref.mar"
|
||||
fi
|
||||
if [ -f "ref-mac.mar" ]; then
|
||||
rm "ref-mac.mar"
|
||||
fi
|
||||
|
||||
../make_incremental_update.sh ref.mar `pwd`/from `pwd`/to
|
||||
../make_incremental_update.sh ref-mac.mar `pwd`/from `pwd`/to-mac
|
||||
|
||||
rm product-1.0.lang.platform.complete.mar
|
||||
rm product-2.0.lang.platform.complete.mar
|
||||
rm product-2.0.lang.mac.complete.mar
|
||||
if [ -f "product-1.0.lang.platform.complete.mar" ]; then
|
||||
rm "product-1.0.lang.platform.complete.mar"
|
||||
fi
|
||||
if [ -f "product-2.0.lang.platform.complete.mar" ]; then
|
||||
rm "product-2.0.lang.platform.complete.mar"
|
||||
fi
|
||||
if [ -f "product-2.0.lang.mac.complete.mar" ]; then
|
||||
rm "product-2.0.lang.mac.complete.mar"
|
||||
fi
|
||||
|
||||
./make_full_update.sh product-1.0.lang.platform.complete.mar `pwd`/from
|
||||
./make_full_update.sh product-2.0.lang.platform.complete.mar `pwd`/to
|
||||
./make_full_update.sh product-2.0.lang.mac.complete.mar `pwd`/to-mac
|
||||
./make_full_update.sh product-1.0.lang.platform.complete.mar "`pwd`/from"
|
||||
./make_full_update.sh product-2.0.lang.platform.complete.mar "`pwd`/to"
|
||||
./make_full_update.sh product-2.0.lang.mac.complete.mar "`pwd`/to-mac"
|
||||
|
|
|
|||
|
|
@ -11,4 +11,4 @@ cd "$workdir"
|
|||
mar -x "$1"
|
||||
mv update.manifest update.manifest.bz2
|
||||
bzip2 -d update.manifest.bz2
|
||||
cat update.manifest
|
||||
cat update.manifest
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ MBSDIFF=${MBSDIFF:-mbsdiff}
|
|||
# Helper routines
|
||||
|
||||
notice() {
|
||||
echo $* 1>&2
|
||||
echo "$*" 1>&2
|
||||
}
|
||||
|
||||
get_file_size() {
|
||||
|
|
@ -36,13 +36,23 @@ copy_perm() {
|
|||
|
||||
make_add_instruction() {
|
||||
f="$1"
|
||||
|
||||
# Used to log to the console
|
||||
if [ $2 ]; then
|
||||
forced=" (forced)"
|
||||
else
|
||||
forced=
|
||||
fi
|
||||
|
||||
is_extension=$(echo "$f" | grep -c 'extensions/.*/')
|
||||
if [ $is_extension = "1" ]; then
|
||||
# Use the subdirectory of the extensions folder as the file to test
|
||||
# before performing this add instruction.
|
||||
testdir=$(echo "$f" | sed 's/\(extensions\/[^\/]*\)\/.*/\1/')
|
||||
notice " add-if: $f$forced"
|
||||
echo "add-if \"$testdir\" \"$f\""
|
||||
else
|
||||
notice " add: $f$forced"
|
||||
echo "add \"$f\""
|
||||
fi
|
||||
}
|
||||
|
|
@ -55,16 +65,21 @@ make_patch_instruction() {
|
|||
# Use the subdirectory of the extensions folder as the file to test
|
||||
# before performing this add instruction.
|
||||
testdir=$(echo "$f" | sed 's/\(extensions\/[^\/]*\)\/.*/\1/')
|
||||
notice " patch-if: $f"
|
||||
echo "patch-if \"$testdir\" \"$f.patch\" \"$f\""
|
||||
elif [ $is_search_plugin = "1" ]; then
|
||||
notice " patch-if: $f"
|
||||
echo "patch-if \"$f\" \"$f.patch\" \"$f\""
|
||||
else
|
||||
notice " patch: $f"
|
||||
echo "patch \"$f.patch\" \"$f\""
|
||||
fi
|
||||
}
|
||||
|
||||
append_remove_instructions() {
|
||||
dir="$1"
|
||||
filev1="$2"
|
||||
filev2="$3"
|
||||
if [ -f "$dir/removed-files" ]; then
|
||||
prefix=
|
||||
listfile="$dir/removed-files"
|
||||
|
|
@ -74,18 +89,43 @@ append_remove_instructions() {
|
|||
fi
|
||||
if [ -n "$listfile" ]; then
|
||||
# Map spaces to pipes so that we correctly handle filenames with spaces.
|
||||
files=($(cat "$listfile" | tr " " "|"))
|
||||
files=($(cat "$listfile" | tr " " "|" | sort -r))
|
||||
num_files=${#files[*]}
|
||||
for ((i=0; $i<$num_files; i=$i+1)); do
|
||||
# Trim whitespace (including trailing carriage returns)
|
||||
f=$(echo ${files[$i]} | tr "|" " " | sed 's/^ *\(.*\) *$/\1/' | tr -d '\r')
|
||||
# Exclude any blank lines or any lines ending with a slash, which indicate
|
||||
# directories. The updater doesn't know how to remove entire directories.
|
||||
# Map pipes back to whitespace and remove carriage returns
|
||||
f=$(echo ${files[$i]} | tr "|" " " | tr -d '\r')
|
||||
# Trim whitespace
|
||||
f=$(echo $f)
|
||||
# Exclude blank lines.
|
||||
if [ -n "$f" ]; then
|
||||
if [ $(echo "$f" | grep -c '\/$') = 0 ]; then
|
||||
echo "remove \"$prefix$f\""
|
||||
else
|
||||
notice "ignoring remove instruction for directory: $f"
|
||||
# Exclude comments
|
||||
if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then
|
||||
# Normalize the path to the root of the Mac OS X bundle if necessary
|
||||
fixedprefix="$prefix"
|
||||
if [ $prefix ]; then
|
||||
if [ $(echo "$f" | grep -c '^\.\./') = 1 ]; then
|
||||
if [ $(echo "$f" | grep -c '^\.\./\.\./') = 1 ]; then
|
||||
f=$(echo $f | sed -e 's:^\.\.\/\.\.\/::')
|
||||
fixedprefix=""
|
||||
else
|
||||
f=$(echo $f | sed -e 's:^\.\.\/::')
|
||||
fixedprefix=$(echo "$prefix" | sed -e 's:^[^\/]*\/::')
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
if [ $(echo "$f" | grep -c '\/$') = 1 ]; then
|
||||
notice " rmdir: $fixedprefix$f"
|
||||
echo "rmdir \"$fixedprefix$f\"" >> $filev2
|
||||
elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then
|
||||
# Remove the *
|
||||
f=$(echo "$f" | sed -e 's:\*$::')
|
||||
notice " rmrfdir: $fixedprefix$f"
|
||||
echo "rmrfdir \"$fixedprefix$f\"" >> $filev2
|
||||
else
|
||||
notice " remove: $fixedprefix$f"
|
||||
echo "remove \"$fixedprefix$f\"" >> $filev1
|
||||
echo "remove \"$fixedprefix$f\"" >> $filev2
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
|
@ -99,14 +139,29 @@ append_remove_instructions() {
|
|||
list_files() {
|
||||
count=0
|
||||
|
||||
# Schrep - removed the exclusion cases here to allow for generation
|
||||
# of testing mars
|
||||
# Removed the exclusion cases here to allow for generation of testing mars
|
||||
find . -type f \
|
||||
| sed 's/\.\/\(.*\)/\1/' \
|
||||
| sort > "$workdir/temp-filelist"
|
||||
| sort -r > "$workdir/temp-filelist"
|
||||
while read file; do
|
||||
eval "${1}[$count]=\"$file\""
|
||||
(( count++ ))
|
||||
done < "$workdir/temp-filelist"
|
||||
rm "$workdir/temp-filelist"
|
||||
}
|
||||
|
||||
# List all directories in the current directory, stripping leading "./"
|
||||
list_dirs() {
|
||||
count=0
|
||||
|
||||
find . -type d \
|
||||
! -name "." \
|
||||
! -name ".." \
|
||||
| sed 's/\.\/\(.*\)/\1/' \
|
||||
| sort -r > "$workdir/temp-dirlist"
|
||||
while read dir; do
|
||||
eval "${1}[$count]=\"$dir\""
|
||||
(( count++ ))
|
||||
done < "$workdir/temp-dirlist"
|
||||
rm "$workdir/temp-dirlist"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,18 @@ workdir="/tmp/diffmar"
|
|||
fromdir="$workdir/0"
|
||||
todir="$workdir/1"
|
||||
|
||||
# On Windows, creation time can be off by a second or more between the files in
|
||||
# the fromdir and todir due to them being extracted synchronously so use
|
||||
# time-style and exclude seconds from the creation time.
|
||||
lsargs="-algR"
|
||||
unamestr=`uname`
|
||||
if [ ! "$unamestr" = 'Darwin' ]; then
|
||||
unamestr=`uname -o`
|
||||
if [ "$unamestr" = 'Msys' -o "$unamestr" = "Cygwin" ]; then
|
||||
lsargs="-algR --time-style=+%Y-%m-%d-%H:%M"
|
||||
fi
|
||||
fi
|
||||
|
||||
rm -rf "$workdir"
|
||||
mkdir -p "$fromdir"
|
||||
mkdir -p "$todir"
|
||||
|
|
@ -19,17 +31,20 @@ mar -x "$1"
|
|||
rm "$1"
|
||||
mv update.manifest update.manifest.bz2
|
||||
bzip2 -d update.manifest.bz2
|
||||
ls -algR > files.txt
|
||||
# Sort the manifest so we don't get any diffs for ordering
|
||||
#cat update.manifest | sort > update.manifest
|
||||
mv updatev2.manifest updatev2.manifest.bz2
|
||||
bzip2 -d updatev2.manifest.bz2
|
||||
ls $lsargs > files.txt
|
||||
|
||||
cd "$todir"
|
||||
mar -x "$2"
|
||||
rm "$2"
|
||||
mv update.manifest update.manifest.bz2
|
||||
bzip2 -d update.manifest.bz2
|
||||
# Sort the manifest so we don't get any diffs for ordering
|
||||
#cat update.manifest | sort > update.manifest
|
||||
ls -algR > files.txt
|
||||
mv updatev2.manifest updatev2.manifest.bz2
|
||||
bzip2 -d updatev2.manifest.bz2
|
||||
ls $lsargs > files.txt
|
||||
|
||||
diff -r "$fromdir" "$todir"
|
||||
echo "diffing $fromdir and $todir"
|
||||
echo "on linux shell sort and python sort return different results"
|
||||
echo "which cause differences in the manifest files"
|
||||
diff -ru "$fromdir" "$todir"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,21 @@
|
|||
remove {foodir/update.manifest
|
||||
remove {foodir/same.txt
|
||||
remove {foodir/same.bin
|
||||
remove {foodir/removed.txt
|
||||
remove {foodir/readme.txt
|
||||
remove {foodir/force.txt
|
||||
remove {foodir/diff-patch-larger-than-file.txt
|
||||
remove-cc {foodir/channel-prefs.js
|
||||
remove update.manifest
|
||||
remove searchplugins/diff/diff-patch-larger-than-file.txt
|
||||
remove same.txt
|
||||
remove same.bin
|
||||
remove removed.txt
|
||||
remove readme.txt
|
||||
remove precomplete
|
||||
remove force.txt
|
||||
remove diff-patch-larger-than-file.txt
|
||||
remove-cc channel-prefs.js
|
||||
rmdir {foodir/
|
||||
rmdir searchplugins/diff/
|
||||
rmdir searchplugins/
|
||||
|
|
@ -33,32 +33,50 @@ fi
|
|||
|
||||
archive="$1"
|
||||
targetdir="$2"
|
||||
# Prevent the workdir from being inside the targetdir so it isn't included in
|
||||
# the update mar.
|
||||
if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then
|
||||
# Remove the /
|
||||
targetdir=$(echo "$targetdir" | sed -e 's:\/$::')
|
||||
fi
|
||||
workdir="$targetdir.work"
|
||||
manifest="$workdir/update.manifest"
|
||||
targetfiles="update.manifest"
|
||||
updatemanifestv1="$workdir/update.manifest"
|
||||
updatemanifestv2="$workdir/updatev2.manifest"
|
||||
targetfiles="update.manifest updatev2.manifest"
|
||||
|
||||
mkdir -p "$workdir"
|
||||
|
||||
# On Mac, the precomplete file added by Bug 386760 will cause OS X to reload the
|
||||
# Info.plist so it launches the right architecture, bug 600098
|
||||
|
||||
# Generate a list of all files in the target directory.
|
||||
pushd "$targetdir"
|
||||
if test $? -ne 0 ; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "precomplete" ]; then
|
||||
notice "precomplete file is missing!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
list_files files
|
||||
|
||||
# Files that should be added on channel change
|
||||
ccfiles=$(find . -type f -name "channel-prefs.js" | sed 's/\.\/\(.*\)/\1/')
|
||||
|
||||
popd
|
||||
|
||||
> $manifest
|
||||
notice ""
|
||||
notice "Adding file add instructions to file 'update.manifest'"
|
||||
> $updatemanifestv1
|
||||
|
||||
num_files=${#files[*]}
|
||||
|
||||
for ((i=0; $i<$num_files; i=$i+1)); do
|
||||
f="${files[$i]}"
|
||||
|
||||
notice "processing $f"
|
||||
|
||||
make_add_instruction "$f" >> $manifest
|
||||
make_add_instruction "$f" >> $updatemanifestv1
|
||||
|
||||
dir=$(dirname "$f")
|
||||
mkdir -p "$workdir/$dir"
|
||||
|
|
@ -68,13 +86,44 @@ for ((i=0; $i<$num_files; i=$i+1)); do
|
|||
targetfiles="$targetfiles \"$f\""
|
||||
done
|
||||
|
||||
# Append remove instructions for any dead files.
|
||||
append_remove_instructions "$targetdir" >> $manifest
|
||||
# Add the type of update to the beginning of and cat the contents of the version
|
||||
# 1 update manifest to the version 2 update manifest.
|
||||
> $updatemanifestv2
|
||||
notice ""
|
||||
notice "Adding type instruction to file 'updatev2.manifest'"
|
||||
notice " type: complete"
|
||||
echo "type \"complete\"" >> $updatemanifestv2
|
||||
|
||||
$BZIP2 -z9 "$manifest" && mv -f "$manifest.bz2" "$manifest"
|
||||
notice ""
|
||||
notice "Adding file add on channel change instructions to file 'updatev2.manifest'"
|
||||
for f in $ccfiles; do
|
||||
notice " add-cc: $f"
|
||||
echo "add-cc \"$f\"" >> $updatemanifestv2
|
||||
dir=$(dirname "$f")
|
||||
mkdir -p "$workdir/$dir"
|
||||
$BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
|
||||
copy_perm "$targetdir/$f" "$workdir/$f"
|
||||
|
||||
targetfiles="$targetfiles \"$f\""
|
||||
done
|
||||
|
||||
notice ""
|
||||
notice "Concatenating file 'update.manifest' to file 'updatev2.manifest'"
|
||||
cat $updatemanifestv1 >> $updatemanifestv2
|
||||
|
||||
# Append remove instructions for any dead files.
|
||||
notice ""
|
||||
notice "Adding file and directory remove instructions from file 'removed-files'"
|
||||
append_remove_instructions "$targetdir" "$updatemanifestv1" "$updatemanifestv2"
|
||||
|
||||
$BZIP2 -z9 "$updatemanifestv1" && mv -f "$updatemanifestv1.bz2" "$updatemanifestv1"
|
||||
$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
|
||||
|
||||
eval "$MAR -C \"$workdir\" -c output.mar $targetfiles"
|
||||
mv -f "$workdir/output.mar" "$archive"
|
||||
|
||||
# cleanup
|
||||
rm -fr "$workdir"
|
||||
|
||||
notice ""
|
||||
notice "Finished"
|
||||
|
|
|
|||
|
|
@ -7,13 +7,3 @@ echo "diffing ref.mar and test.mar"
|
|||
./diffmar.sh ref.mar test.mar
|
||||
echo "diffing ref-mac.mar and test-mac.mar"
|
||||
./diffmar.sh ref-mac.mar test-mac.mar
|
||||
|
||||
|
||||
echo "testing make_incremental_updates_mar.py"
|
||||
python ../make_incremental_updates_mar.py -f testpatchfile.txt
|
||||
|
||||
echo "diffing ref.mar and test.mar"
|
||||
./diffmar.sh ref.mar test.mar
|
||||
echo "diffing ref-mac.mar and test-mac.mar"
|
||||
./diffmar.sh ref-mac.mar test-mac.mar
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,4 @@
|
|||
remove precomplete
|
||||
remove Contents/MacOS/removed-files
|
||||
rmdir Contents/MacOS/
|
||||
rmdir Contents/
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
remove {foodir/update.manifest
|
||||
remove {foodir/same.txt
|
||||
remove {foodir/same.bin
|
||||
remove {foodir/readme.txt
|
||||
remove {foodir/force.txt
|
||||
remove {foodir/diff-patch-larger-than-file.txt
|
||||
remove-cc {foodir/channel-prefs.js
|
||||
remove update.manifest
|
||||
remove searchplugins/diff/diff-patch-larger-than-file.txt
|
||||
remove same.txt
|
||||
remove same.bin
|
||||
remove removed-files
|
||||
remove readme.txt
|
||||
remove precomplete
|
||||
remove force.txt
|
||||
remove diff-patch-larger-than-file.txt
|
||||
remove-cc channel-prefs.js
|
||||
rmdir {foodir/
|
||||
rmdir searchplugins/diff/
|
||||
rmdir searchplugins/added/
|
||||
rmdir searchplugins/
|
||||
|
|
@ -7,36 +7,37 @@ from make_incremental_updates import PatchInfo, MarFileEntry
|
|||
class TestPatchInfo(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.work_dir = 'work_dir'
|
||||
self.file_exclusion_list = ['channel-prefs.js','update.manifest','removed-files']
|
||||
self.file_exclusion_list = ['channel-prefs.js','update.manifest','updatev2.manifest','removed-files']
|
||||
self.path_exclusion_list = ['/readme.txt']
|
||||
self.patch_info = PatchInfo(self.work_dir, self.file_exclusion_list, self.path_exclusion_list)
|
||||
|
||||
def testPatchInfo(self):
|
||||
self.assertEquals(self.work_dir, self.patch_info.work_dir)
|
||||
self.assertEquals([], self.patch_info.archive_files)
|
||||
self.assertEquals([], self.patch_info.manifest)
|
||||
self.assertEquals([], self.patch_info.manifestv1)
|
||||
self.assertEquals([], self.patch_info.manifestv2)
|
||||
self.assertEquals(self.file_exclusion_list, self.patch_info.file_exclusion_list)
|
||||
self.assertEquals(self.path_exclusion_list, self.patch_info.path_exclusion_list)
|
||||
|
||||
def test_append_add_instruction(self):
|
||||
self.patch_info.append_add_instruction('file.test')
|
||||
self.assertEquals(['add "file.test"'], self.patch_info.manifest)
|
||||
self.assertEquals(['add "file.test"'], self.patch_info.manifestv1)
|
||||
|
||||
def test_append_patch_instruction(self):
|
||||
self.patch_info.append_patch_instruction('file.test', 'patchname')
|
||||
self.assertEquals(['patch "patchname" "file.test"'], self.patch_info.manifest)
|
||||
self.assertEquals(['patch "patchname" "file.test"'], self.patch_info.manifestv1)
|
||||
|
||||
""" FIXME touches the filesystem, need refactoring
|
||||
def test_append_remove_instruction(self):
|
||||
self.patch_info.append_remove_instruction('file.test')
|
||||
self.assertEquals(['remove "file.test"'], self.patch_info.manifest)
|
||||
self.assertEquals(['remove "file.test"'], self.patch_info.manifestv1)
|
||||
|
||||
def test_create_manifest_file(self):
|
||||
self.patch_info.create_manifest_file()
|
||||
"""
|
||||
|
||||
def test_build_marfile_entry_hash(self):
|
||||
self.assertEquals(({}, set([])), self.patch_info.build_marfile_entry_hash('root_path'))
|
||||
self.assertEquals(({}, set([]), set([])), self.patch_info.build_marfile_entry_hash('root_path'))
|
||||
|
||||
""" FIXME touches the filesystem, need refactoring
|
||||
class TestMarFileEntry(unittest.TestCase):
|
||||
|
|
@ -66,7 +67,7 @@ class TestMarFileEntry(unittest.TestCase):
|
|||
class TestMakeIncrementalUpdates(unittest.TestCase):
|
||||
def setUp(self):
|
||||
work_dir = '.'
|
||||
self.patch_info = PatchInfo(work_dir, ['channel-prefs.js','update.manifest','removed-files'],['/readme.txt'])
|
||||
self.patch_info = PatchInfo(work_dir, ['channel-prefs.js','update.manifest','updatev2.manifest','removed-files'],['/readme.txt'])
|
||||
root_path = '/'
|
||||
filename = 'test.file'
|
||||
self.mar_file_entry = MarFileEntry(root_path, filename)
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче