Backed out 4 changesets (bug 1529894) for xperf failures on a CLOSED TREE.

Backed out changeset d31f88692394 (bug 1529894)
Backed out changeset e2d7b59776a2 (bug 1529894)
Backed out changeset 3c37fd9d30d5 (bug 1529894)
Backed out changeset e93f4871731b (bug 1529894)

--HG--
extra : rebase_source : ae317f856e08d09ec655cb7b7a87c120f3c626a5

modules/libjar/nsZipArchive.cpp

@@ -22,7 +22,6 @@
 #include "mozilla/Logging.h"
 #include "mozilla/UniquePtrExtensions.h"
 #include "stdlib.h"
-#include "nsDirectoryService.h"
 #include "nsWildCard.h"
 #include "nsXULAppAPI.h"
 #include "nsZipArchive.h"
@@ -79,8 +78,14 @@ static uint32_t HashName(const char *aName, uint16_t nameLen);
 
 class ZipArchiveLogger {
  public:
-  void Init(const char *env) {
+  void Write(const nsACString &zip, const char *entry) const {
+    if (!XRE_IsParentProcess()) {
+      return;
+    }
     if (!fd) {
+      char *env = PR_GetEnv("MOZ_JAR_LOG_FILE");
+      if (!env) return;
+
       nsCOMPtr<nsIFile> logFile;
       nsresult rv = NS_NewLocalFile(NS_ConvertUTF8toUTF16(env), false,
                                     getter_AddRefs(logFile));
@@ -111,16 +116,11 @@ class ZipArchiveLogger {
 #endif
       fd = file;
     }
-  }
-
-  void Write(const nsACString &zip, const char *entry) const {
-    if (fd) {
-      nsCString buf(zip);
-      buf.Append(' ');
-      buf.Append(entry);
-      buf.Append('\n');
-      PR_Write(fd, buf.get(), buf.Length());
-    }
+    nsCString buf(zip);
+    buf.Append(' ');
+    buf.Append(entry);
+    buf.Append('\n');
+    PR_Write(fd, buf.get(), buf.Length());
   }
 
   void AddRef() {
@@ -138,7 +138,7 @@ class ZipArchiveLogger {
 
  private:
   int refCnt;
-  PRFileDesc *fd;
+  mutable PRFileDesc *fd;
 };
 
 static ZipArchiveLogger zipLog;
@@ -336,52 +336,7 @@ nsresult nsZipArchive::OpenArchive(nsZipHandle *aZipHandle, PRFileDesc *aFd) {
   //-- get table of contents for archive
   nsresult rv = BuildFileList(aFd);
   if (NS_SUCCEEDED(rv)) {
-    if (aZipHandle->mFile && XRE_IsParentProcess()) {
-      static char *env = PR_GetEnv("MOZ_JAR_LOG_FILE");
-      if (env) {
-        zipLog.Init(env);
-        // We only log accesses in jar/zip archives within the NS_GRE_DIR
-        // and/or the APK on Android. For the former, we log the archive path
-        // relative to NS_GRE_DIR, and for the latter, the nested-archive
-        // path within the APK. This makes the path match the path of the
-        // archives relative to the packaged dist/$APP_NAME directory in a
-        // build.
-        if (aZipHandle->mFile.IsZip()) {
-          // Nested archive, likely omni.ja in APK.
-          aZipHandle->mFile.GetPath(mURI);
-        } else if (nsDirectoryService::gService) {
-          // We can reach here through the initialization of Omnijar from
-          // XRE_InitCommandLine, which happens before the directory service
-          // is initialized. When that happens, it means the opened archive is
-          // the APK, and we don't care to log that one, so we just skip
-          // when the directory service is not initialized.
-          nsCOMPtr<nsIFile> dir = aZipHandle->mFile.GetBaseFile();
-          nsCOMPtr<nsIFile> gre_dir;
-          nsAutoCString path;
-          if (NS_SUCCEEDED(nsDirectoryService::gService->Get(
-                  NS_GRE_DIR, NS_GET_IID(nsIFile), getter_AddRefs(gre_dir)))) {
-            nsAutoCString leaf;
-            nsCOMPtr<nsIFile> parent;
-            while (NS_SUCCEEDED(dir->GetNativeLeafName(leaf)) &&
-                   NS_SUCCEEDED(dir->GetParent(getter_AddRefs(parent)))) {
-              if (!parent) {
-                break;
-              }
-              dir = parent;
-              if (path.Length()) {
-                path.Insert('/', 0);
-              }
-              path.Insert(leaf, 0);
-              bool equals;
-              if (NS_SUCCEEDED(dir->Equals(gre_dir, &equals)) && equals) {
-                mURI.Assign(path);
-                break;
-              }
-            }
-          }
-        }
-      }
-    }
+    if (aZipHandle->mFile) aZipHandle->mFile.GetURIString(mURI);
   }
   return rv;
 }
@@ -472,9 +427,7 @@ nsZipItem *nsZipArchive::GetItem(const char *aEntryName) {
           (!memcmp(aEntryName, item->Name(), len))) {
         // Successful GetItem() is a good indicator that the file is about to be
         // read
-        if (mURI.Length()) {
-          zipLog.Write(mURI, aEntryName);
-        }
+        zipLog.Write(mURI, aEntryName);
         return item;  //-- found it
       }
       item = item->next;

python/mozbuild/mozpack/mozjar.py

@@ -14,6 +14,7 @@ from zipfile import (
     ZIP_DEFLATED,
 )
 from collections import OrderedDict
+from urlparse import urlparse, ParseResult
 import mozpack.path as mozpath
 from mozbuild.util import memoize
 
@@ -832,18 +833,56 @@ class BrotliCompress(object):
 class JarLog(dict):
     '''
     Helper to read the file Gecko generates when setting MOZ_JAR_LOG_FILE.
-    The jar log is then available as a dict with the jar path as key, and
-    the corresponding access log as a list value. Only the first access to
-    a given member of a jar is stored.
+    The jar log is then available as a dict with the jar path as key (see
+    canonicalize for more details on the key value), and the corresponding
+    access log as a list value. Only the first access to a given member of
+    a jar is stored.
     '''
 
     def __init__(self, file=None, fileobj=None):
         if not fileobj:
             fileobj = open(file, 'r')
+        urlmap = {}
         for line in fileobj:
-            jar, path = line.strip().split(None, 1)
-            if not jar or not path:
+            url, path = line.strip().split(None, 1)
+            if not url or not path:
                 continue
+            if url not in urlmap:
+                urlmap[url] = JarLog.canonicalize(url)
+            jar = urlmap[url]
             entry = self.setdefault(jar, [])
             if path not in entry:
                 entry.append(path)
+
+    @staticmethod
+    def canonicalize(url):
+        '''
+        The jar path is stored in a MOZ_JAR_LOG_FILE log as a url. This method
+        returns a unique value corresponding to such urls.
+        - file:///{path} becomes {path}
+        - jar:file:///{path}!/{subpath} becomes ({path}, {subpath})
+        - jar:jar:file:///{path}!/{subpath}!/{subpath2} becomes
+           ({path}, {subpath}, {subpath2})
+        '''
+        if not isinstance(url, ParseResult):
+            # Assume that if it doesn't start with jar: or file:, it's a path.
+            if not url.startswith(('jar:', 'file:')):
+                url = 'file:///' + os.path.abspath(url)
+            url = urlparse(url)
+        assert url.scheme
+        assert url.scheme in ('jar', 'file')
+        if url.scheme == 'jar':
+            path = JarLog.canonicalize(url.path)
+            if isinstance(path, tuple):
+                return path[:-1] + tuple(path[-1].split('!/', 1))
+            return tuple(path.split('!/', 1))
+        if url.scheme == 'file':
+            assert os.path.isabs(url.path)
+            path = url.path
+            # On Windows, url.path will be /drive:/path ; on Unix systems,
+            # /path. As we want drive:/path instead of /drive:/path on Windows,
+            # remove the leading /.
+            if os.path.isabs(path[1:]):
+                path = path[1:]
+            path = os.path.realpath(path)
+            return mozpath.normsep(os.path.normcase(path))

python/mozbuild/mozpack/test/test_mozjar.py

@@ -17,6 +17,7 @@ from mozpack.test.test_files import MockDest
 import unittest
 import mozunit
 from cStringIO import StringIO
+from urllib import pathname2url
 import mozpack.path as mozpath
 import os
 
@@ -289,32 +290,55 @@ class TestPreload(unittest.TestCase):
 
 class TestJarLog(unittest.TestCase):
     def test_jarlog(self):
+        base = 'file:' + pathname2url(os.path.abspath(os.curdir))
         s = StringIO('\n'.join([
-            'bar/baz.jar first',
-            'bar/baz.jar second',
-            'bar/baz.jar third',
-            'bar/baz.jar second',
-            'bar/baz.jar second',
-            'omni.ja stuff',
-            'bar/baz.jar first',
-            'omni.ja other/stuff',
-            'omni.ja stuff',
-            'bar/baz.jar third',
+            base + '/bar/baz.jar first',
+            base + '/bar/baz.jar second',
+            base + '/bar/baz.jar third',
+            base + '/bar/baz.jar second',
+            base + '/bar/baz.jar second',
+            'jar:' + base + '/qux.zip!/omni.ja stuff',
+            base + '/bar/baz.jar first',
+            'jar:' + base + '/qux.zip!/omni.ja other/stuff',
+            'jar:' + base + '/qux.zip!/omni.ja stuff',
+            base + '/bar/baz.jar third',
+            'jar:jar:' + base + '/qux.zip!/baz/baz.jar!/omni.ja nested/stuff',
+            'jar:jar:jar:' + base + '/qux.zip!/baz/baz.jar!/foo.zip!/omni.ja' +
+            ' deeply/nested/stuff',
         ]))
         log = JarLog(fileobj=s)
+
+        def canonicalize(p):
+            return mozpath.normsep(os.path.normcase(os.path.realpath(p)))
+
+        baz_jar = canonicalize('bar/baz.jar')
+        qux_zip = canonicalize('qux.zip')
         self.assertEqual(set(log.keys()), set([
-            'bar/baz.jar',
-            'omni.ja',
+            baz_jar,
+            (qux_zip, 'omni.ja'),
+            (qux_zip, 'baz/baz.jar', 'omni.ja'),
+            (qux_zip, 'baz/baz.jar', 'foo.zip', 'omni.ja'),
         ]))
-        self.assertEqual(log['bar/baz.jar'], [
+        self.assertEqual(log[baz_jar], [
             'first',
             'second',
             'third',
         ])
-        self.assertEqual(log['omni.ja'], [
+        self.assertEqual(log[(qux_zip, 'omni.ja')], [
             'stuff',
             'other/stuff',
         ])
+        self.assertEqual(log[(qux_zip, 'baz/baz.jar', 'omni.ja')],
+                         ['nested/stuff'])
+        self.assertEqual(log[(qux_zip, 'baz/baz.jar', 'foo.zip',
+                              'omni.ja')], ['deeply/nested/stuff'])
+
+        # The above tests also indirectly check the value returned by
+        # JarLog.canonicalize for various jar: and file: urls, but
+        # JarLog.canonicalize also supports plain paths.
+        self.assertEqual(JarLog.canonicalize(os.path.abspath('bar/baz.jar')),
+                         baz_jar)
+        self.assertEqual(JarLog.canonicalize('bar/baz.jar'), baz_jar)
 
 
 if __name__ == '__main__':

toolkit/mozapps/installer/packager.py

@@ -324,8 +324,9 @@ def main():
         for p, f in copier:
             if not isinstance(f, Jarrer):
                 continue
-            if p in log:
-                f.preload(log[p])
+            key = JarLog.canonicalize(os.path.join(args.destination, p))
+            if key in log:
+                f.preload(log[key])
 
     copier.copy(args.destination)
     generate_precomplete(os.path.normpath(os.path.join(args.destination,