From 62fd669dc722d16b099048f8bc9a5383070fe104 Mon Sep 17 00:00:00 2001
From: Taras Glek <tglek@mozilla.com>
Date: Fri, 10 Sep 2010 12:30:07 -0400
Subject: [PATCH] Bug 589368 - Locale repacking support for jar reordering;
 r=ted a=blocking-betaN+

---
 config/optimizejars.py                | 170 ++++++++++++++++++--------
 modules/libjar/nsZipArchive.cpp       |   2 +-
 toolkit/locales/l10n.mk               |   2 +
 toolkit/mozapps/installer/packager.mk |   7 +-
 4 files changed, 124 insertions(+), 57 deletions(-)

diff --git a/config/optimizejars.py b/config/optimizejars.py
index 2f26637a874..234d82bf039 100644
--- a/config/optimizejars.py
+++ b/config/optimizejars.py
@@ -183,14 +183,24 @@ class BinaryBlob:
         assert_true(out_data == data, "Serialization fail %d !=%d"% (len(out_data), len(data)))
         return retstruct
 
-def optimizejar(log, jar, outjar):
-    entries = open(log).read().rstrip().split("\n")
+def optimizejar(jar, outjar, inlog = None):
+    if inlog is not None:
+        inlog = open(inlog).read().rstrip()
+        # in the case of an empty log still move the index forward
+        if len(inlog) == 0:
+            inlog = []
+        else:
+            inlog = inlog.split("\n")
+    outlog = []
     jarblob = BinaryBlob(jar)
     dirend = jarblob.read_struct(cdir_end, jarblob.length - size_of(cdir_end))
     assert_true(dirend.signature == ENDSIG, "no signature in the end");
-
     cdir_offset = dirend.cdir_offset
-   
+    readahead = 0
+    if inlog is None and cdir_offset == 4:
+        readahead = struct.unpack("<I", jarblob.readAt(0, 4))[0]
+        print("%s: startup data ends at byte %d" % (outjar, readahead));
+
     jarblob.offset = cdir_offset
     central_directory = []
     for i  in range(0, dirend.cdir_entries):
@@ -198,40 +208,46 @@ def optimizejar(log, jar, outjar):
         central_directory.append(entry)
         
     reordered_count = 0
-    dup_guard = set()
-    for ordered_name in entries:
-        if ordered_name in dup_guard:
-            continue
-        else:
-            dup_guard.add(ordered_name)
-        found = False
-        for i in range(reordered_count, len(central_directory)):
-            if central_directory[i].filename == ordered_name:
-                # swap the cdir entries
-                tmp = central_directory[i]
-                central_directory[i] = central_directory[reordered_count]
-                central_directory[reordered_count] = tmp
-                reordered_count = reordered_count + 1
-                found = True
-                break
-        assert_true(found, "Can't find %s in %s\n" %(ordered_name, jar))
+    if inlog is not None:
+        dup_guard = set()
+        for ordered_name in inlog:
+            if ordered_name in dup_guard:
+                continue
+            else:
+                dup_guard.add(ordered_name)
+            found = False
+            for i in range(reordered_count, len(central_directory)):
+                if central_directory[i].filename == ordered_name:
+                    # swap the cdir entries
+                    tmp = central_directory[i]
+                    central_directory[i] = central_directory[reordered_count]
+                    central_directory[reordered_count] = tmp
+                    reordered_count = reordered_count + 1
+                    found = True
+                    break
+            if not found:
+                print( "Can't find '%s' in %s" % (ordered_name, jar))
 
-    # have to put central directory at offset 1 cos 0 confuses some tools
-    dirend.cdir_offset = 1
-    dirend_data = dirend.pack()
     outfd = open(outjar, "wb")
-    # make room for central dir + end of dir + 1 extra byte at front
-    out_offset = dirend.cdir_offset + dirend.cdir_size + len(dirend_data)
-    outfd.seek(out_offset)
+    out_offset = 0
+    if inlog is not None:
+        # have to put central directory at offset 4 cos 0 confuses some tools.
+        # This also lets us specify how many entries should be preread
+        dirend.cdir_offset = 4
+        # make room for central dir + end of dir + 4 extra bytes at front
+        out_offset = dirend.cdir_offset + dirend.cdir_size + size_of(cdir_end) 
+        outfd.seek(out_offset)
 
     cdir_data = ""
     written_count = 0
+    # store number of bytes suggested for readahead
     for entry in central_directory:
         # read in the header twice..first for comparison, second time for convenience when writing out
         jarfile = jarblob.read_struct(local_file_header, entry.offset)
         assert_true(jarfile.filename == entry.filename, "Directory/Localheader mismatch")
         data = jarfile.pack()
         outfd.write(data)
+        old_entry_offset = entry.offset
         entry.offset = out_offset
         out_offset = out_offset + len(data)
         entry_data = entry.pack()
@@ -240,40 +256,88 @@ def optimizejar(log, jar, outjar):
         assert_true(len(entry_data) != expected_len,
                     "%s entry size - expected:%d got:%d" % (entry.filename, len(entry_data), expected_len))
         written_count += 1
-        if written_count == reordered_count:
-            print("%s: startup data ends at byte %d"%( outjar, out_offset));
-        elif written_count < reordered_count:
-            print("%s @ %d" % (entry.filename, entry.offset))
+        if inlog is not None:
+            if written_count == reordered_count:
+                readahead = out_offset
+                print("%s: startup data ends at byte %d"%( outjar, readahead));
+            elif written_count < reordered_count:
+                pass
+                #print("%s @ %d" % (entry.filename, out_offset))
+        elif readahead >= old_entry_offset + len(data):
+            outlog.append(entry.filename)
+            reordered_count += 1
 
+    if inlog is None:
+        dirend.cdir_offset = out_offset
+
+    dirend_data = dirend.pack()
     assert_true(size_of(cdir_end) == len(dirend_data), "Failed to serialize directory end correctly. Serialized size;%d, expected:%d"%(len(dirend_data), size_of(cdir_end)));
-    outfd.write(dirend_data)
+
     outfd.seek(dirend.cdir_offset)
     assert_true(len(cdir_data) == dirend.cdir_size, "Failed to serialize central directory correctly. Serialized size;%d, expected:%d expected-size:%d" % (len(cdir_data), dirend.cdir_size, dirend.cdir_size - len(cdir_data)));
     outfd.write(cdir_data)
     outfd.write(dirend_data)
+
+    # for ordered jars the central directory is written in the begining of the file, so a second central-directory
+    # entry has to be written in the end of the file
+    if inlog is not None:
+        outfd.seek(0)
+        outfd.write(struct.pack("<I", readahead));
+        outfd.seek(out_offset)
+        outfd.write(dirend_data)
+
+    print "%s %d/%d in %s" % (("Ordered" if inlog is not None else "Deoptimized"),
+                              reordered_count, len(central_directory), outjar)
     outfd.close()
-    print "Ordered %d/%d in %s" % (reordered_count, len(central_directory), outjar)
-    
-if len(sys.argv) != 4:
-    print "Usage: %s JAR_LOG_DIR IN_JAR_DIR OUT_JAR_DIR" % sys.argv[0]
+    return outlog
+        
+if len(sys.argv) != 5:
+    print "Usage: --optimize|--deoptimize %s JAR_LOG_DIR IN_JAR_DIR OUT_JAR_DIR" % sys.argv[0]
     exit(1)
 
-JAR_LOG_DIR = sys.argv[1]
-IN_JAR_DIR = sys.argv[2]
-OUT_JAR_DIR = sys.argv[3]
+def optimize(JAR_LOG_DIR, IN_JAR_DIR, OUT_JAR_DIR):
+    if not os.path.exists(JAR_LOG_DIR):
+        print("No jar logs found in %s. No jars to optimize." % JAR_LOG_DIR)
+        exit(0)
 
-if not os.path.exists(JAR_LOG_DIR):
-    print "No jar logs found. No jars to optimize."
-    exit(0)
+    ls = os.listdir(JAR_LOG_DIR)
+    for logfile in ls:
+        if not logfile.endswith(".jar.log"):
+            continue
+        injarfile = os.path.join(IN_JAR_DIR, logfile[:-4])
+        outjarfile = os.path.join(OUT_JAR_DIR, logfile[:-4]) 
+        if not os.path.exists(injarfile):
+            print "Warning: Skipping %s, %s doesn't exist" % (logfile, injarfile)
+            continue
+        logfile = os.path.join(JAR_LOG_DIR, logfile)
+        optimizejar(injarfile, outjarfile, logfile)
 
-ls = os.listdir(JAR_LOG_DIR)
-for logfile in ls:
-    if logfile[-8:] != ".jar.log":
-        continue
-    injarfile = os.path.join(IN_JAR_DIR, logfile[:-4])
-    outjarfile = os.path.join(OUT_JAR_DIR, logfile[:-4]) 
-    if not os.path.exists(injarfile):
-        print "Warning: Skipping %s, %s doesn't exist" % (logfile, injarfile)
-        continue
-    logfile = os.path.join(JAR_LOG_DIR, logfile)
-    optimizejar(logfile, injarfile, outjarfile)
+def deoptimize(JAR_LOG_DIR, IN_JAR_DIR, OUT_JAR_DIR):
+    if not os.path.exists(JAR_LOG_DIR):
+        os.makedirs(JAR_LOG_DIR)
+
+    ls = os.listdir(IN_JAR_DIR)
+    for jarfile in ls:
+        if not jarfile.endswith(".jar"):
+            continue
+        injarfile = os.path.join(IN_JAR_DIR, jarfile)
+        outjarfile = os.path.join(OUT_JAR_DIR, jarfile) 
+        logfile = os.path.join(JAR_LOG_DIR, jarfile + ".log")
+        log = optimizejar(injarfile, outjarfile, None)
+        open(logfile, "wb").write("\n".join(log))
+
+def main():        
+    MODE = sys.argv[1]
+    JAR_LOG_DIR = sys.argv[2]
+    IN_JAR_DIR = sys.argv[3]
+    OUT_JAR_DIR = sys.argv[4]
+    if MODE == "--optimize":
+        optimize(JAR_LOG_DIR, IN_JAR_DIR, OUT_JAR_DIR)
+    elif MODE == "--deoptimize":
+        deoptimize(JAR_LOG_DIR, IN_JAR_DIR, OUT_JAR_DIR)
+    else:
+        print("Unknown mode %s" % MODE)
+        exit(1)
+
+if __name__ == '__main__':
+    main()
diff --git a/modules/libjar/nsZipArchive.cpp b/modules/libjar/nsZipArchive.cpp
index a40080914ab..b9fde94bc5b 100644
--- a/modules/libjar/nsZipArchive.cpp
+++ b/modules/libjar/nsZipArchive.cpp
@@ -594,7 +594,7 @@ nsresult nsZipArchive::BuildFileList()
   const PRUint8* startp = mFd->mFileData;
   const PRUint8* endp = startp + mFd->mLen;
   
-  PRUint32 centralOffset = 1;
+  PRUint32 centralOffset = 4;
   if (mFd->mLen > ZIPCENTRAL_SIZE && *(PRUint32*)(startp + centralOffset) == CENTRALSIG) {
     // Success means optimized jar layout from bug 559961 is in effect
   } else {
diff --git a/toolkit/locales/l10n.mk b/toolkit/locales/l10n.mk
index 29625d786ff..5e43dc3e935 100644
--- a/toolkit/locales/l10n.mk
+++ b/toolkit/locales/l10n.mk
@@ -132,6 +132,8 @@ MOZDEPTH ?= $(DEPTH)
 
 repackage-zip: UNPACKAGE="$(ZIP_IN)"
 repackage-zip:
+# Adjust jar logs with the new locale (can't use sed -i because of bug 373784)
+	-$(PERL) -pi -e "s/en-US/$(AB_CD)/g" $(_ABS_DIST)/jarlog/*.jar.log
 # call a hook for apps to put their uninstall helper.exe into the package
 	$(UNINSTALLER_PACKAGE_HOOK)
 # copy xpi-stage over, but not install.rdf and chrome.manifest,
diff --git a/toolkit/mozapps/installer/packager.mk b/toolkit/mozapps/installer/packager.mk
index 01814fbcc57..e9f2ab4b42d 100644
--- a/toolkit/mozapps/installer/packager.mk
+++ b/toolkit/mozapps/installer/packager.mk
@@ -87,6 +87,7 @@ SDK           = $(SDK_PATH)$(PKG_BASENAME).sdk$(SDK_SUFFIX)
 
 MAKE_PACKAGE	= $(error What is a $(MOZ_PKG_FORMAT) package format?);
 MAKE_CAB	= $(error Don't know how to make a CAB!);
+_ABS_DIST = $(call core_abspath,$(DIST))
 
 ifdef WINCE
 ifndef WINCE_WINDOWS_MOBILE
@@ -184,7 +185,6 @@ endif
 INNER_MAKE_PACKAGE	= $(_ABS_MOZSRCDIR)/build/package/mac_osx/pkg-dmg \
   --source "$(PKG_DMG_SOURCE)" --target "$(PACKAGE)" \
   --volname "$(MOZ_APP_DISPLAYNAME)" $(PKG_DMG_FLAGS)
-_ABS_DIST = $(call core_abspath,$(DIST))
 INNER_UNMAKE_PACKAGE	= \
   set -ex; \
   rm -rf $(_ABS_DIST)/unpack.tmp; \
@@ -241,10 +241,11 @@ PACK_OMNIJAR	= \
   mv components.manifest components && \
   find . | xargs touch -t 201001010000 && \
   zip -r9mX omni.jar $(OMNIJAR_FILES) -x $(NON_OMNIJAR_FILES) && \
-  $(OPTIMIZE_JARS_CMD) $(DIST)/jarlog/ ./ ./ && \
+  $(OPTIMIZE_JARS_CMD) --optimize $(_ABS_DIST)/jarlog/ ./ ./ && \
   mv binary.manifest components && \
   printf "manifest components/binary.manifest\n" > chrome.manifest
 UNPACK_OMNIJAR	= \
+  $(OPTIMIZE_JARS_CMD) --deoptimize $(_ABS_DIST)/jarlog/ ./ ./ && \
   unzip -o omni.jar && \
   rm -f components/binary.manifest && \
   sed -e 's/^\#binary-component/binary-component/' components/components.manifest > components.manifest && \
@@ -456,7 +457,7 @@ else
 endif # DMG
 endif # MOZ_PKG_MANIFEST
 endif # UNIVERSAL_BINARY
-	$(OPTIMIZE_JARS_CMD) $(DIST)/jarlog/ $(DIST)/bin/chrome $(DIST)/$(STAGEPATH)$(MOZ_PKG_DIR)/chrome
+	$(OPTIMIZE_JARS_CMD) --optimize $(DIST)/jarlog/ $(DIST)/bin/chrome $(DIST)/$(STAGEPATH)$(MOZ_PKG_DIR)/chrome
 ifndef PKG_SKIP_STRIP
 	@echo "Stripping package directory..."
 	@cd $(DIST)/$(STAGEPATH)$(MOZ_PKG_DIR); find . ! -type d \