Backed out changeset 2bc6db0a2574 (bug 1486524) android raptor failures ona CLOSED TREE

mobile/android/base/java/org/mozilla/gecko/db/SQLiteBridgeContentProvider.java

@@ -120,8 +120,9 @@ public abstract class SQLiteBridgeContentProvider extends ContentProvider {
 
         boolean dbNeedsSetup = true;
         try {
-            GeckoLoader.loadSQLiteLibs(context);
-            GeckoLoader.loadNSSLibs(context);
+            String resourcePath = context.getPackageResourcePath();
+            GeckoLoader.loadSQLiteLibs(context, resourcePath);
+            GeckoLoader.loadNSSLibs(context, resourcePath);
             bridge = SQLiteBridge.openDatabase(databasePath, null, 0);
             int version = bridge.getVersion();
             dbNeedsSetup = version != getDBVersion();

mobile/android/geckoview/src/main/java/org/mozilla/gecko/GeckoThread.java

@@ -260,10 +260,10 @@ public class GeckoThread extends Thread {
         return isState(State.RUNNING);
     }
 
-    private static void loadGeckoLibs(final Context context) {
-        GeckoLoader.loadSQLiteLibs(context);
-        GeckoLoader.loadNSSLibs(context);
-        GeckoLoader.loadGeckoLibs(context);
+    private static void loadGeckoLibs(final Context context, final String resourcePath) {
+        GeckoLoader.loadSQLiteLibs(context, resourcePath);
+        GeckoLoader.loadNSSLibs(context, resourcePath);
+        GeckoLoader.loadGeckoLibs(context, resourcePath);
         setState(State.LIBS_READY);
     }
 
@@ -282,7 +282,28 @@ public class GeckoThread extends Thread {
             res.updateConfiguration(config, null);
         }
 
-        loadGeckoLibs(context);
+        final String resourcePath = context.getPackageResourcePath();
+
+        try {
+            loadGeckoLibs(context, resourcePath);
+            return;
+        } catch (final Exception e) {
+            // Cannot load libs; try clearing the cached files.
+            Log.w(LOGTAG, "Clearing cache after load libs exception", e);
+        }
+
+        FileUtils.delTree(GeckoLoader.getCacheDir(context),
+                          new FileUtils.FilenameRegexFilter(".*\\.so(?:\\.crc)?$"),
+                          /* recurse */ true);
+
+        if (!GeckoLoader.verifyCRCs(resourcePath)) {
+            setState(State.CORRUPT_APK);
+            EventDispatcher.getInstance().dispatch("Gecko:CorruptAPK", null);
+            return;
+        }
+
+        // Then try loading again. If this throws again, we actually crash.
+        loadGeckoLibs(context, resourcePath);
     }
 
     private String[] getMainProcessArgs() {

mobile/android/geckoview/src/main/java/org/mozilla/gecko/NSSBridge.java

@@ -18,7 +18,8 @@ public class NSSBridge {
     @RobocopTarget
     static public String encrypt(Context context, String aValue)
       throws Exception {
-        GeckoLoader.loadNSSLibs(context);
+        String resourcePath = context.getPackageResourcePath();
+        GeckoLoader.loadNSSLibs(context, resourcePath);
 
         String path = GeckoProfile.get(context).getDir().toString();
         return nativeEncrypt(path, aValue);
@@ -27,7 +28,8 @@ public class NSSBridge {
     @RobocopTarget
     static public String encrypt(Context context, String profilePath, String aValue)
       throws Exception {
-        GeckoLoader.loadNSSLibs(context);
+        String resourcePath = context.getPackageResourcePath();
+        GeckoLoader.loadNSSLibs(context, resourcePath);
 
         return nativeEncrypt(profilePath, aValue);
     }
@@ -35,7 +37,8 @@ public class NSSBridge {
     @RobocopTarget
     static public String decrypt(Context context, String aValue)
       throws Exception {
-        GeckoLoader.loadNSSLibs(context);
+        String resourcePath = context.getPackageResourcePath();
+        GeckoLoader.loadNSSLibs(context, resourcePath);
 
         String path = GeckoProfile.get(context).getDir().toString();
         return nativeDecrypt(path, aValue);
@@ -44,7 +47,8 @@ public class NSSBridge {
     @RobocopTarget
     static public String decrypt(Context context, String profilePath, String aValue)
       throws Exception {
-        GeckoLoader.loadNSSLibs(context);
+        String resourcePath = context.getPackageResourcePath();
+        GeckoLoader.loadNSSLibs(context, resourcePath);
 
         return nativeDecrypt(profilePath, aValue);
     }

mobile/android/geckoview/src/main/java/org/mozilla/gecko/mozglue/GeckoLoader.java

@@ -180,28 +180,30 @@ public final class GeckoLoader {
     private static void loadLibsSetupLocked(Context context) {
         // setup the libs cache
         putenv("GRE_HOME=" + getGREDir(context).getPath());
+        putenv("MOZ_LINKER_CACHE=" + getCacheDir(context).getPath());
+        putenv("MOZ_LINKER_EXTRACT=1");
     }
 
     @RobocopTarget
-    public synchronized static void loadSQLiteLibs(final Context context) {
+    public synchronized static void loadSQLiteLibs(final Context context, final String apkName) {
         if (sSQLiteLibsLoaded) {
             return;
         }
 
         loadMozGlue(context);
         loadLibsSetupLocked(context);
-        loadSQLiteLibsNative();
+        loadSQLiteLibsNative(apkName);
         sSQLiteLibsLoaded = true;
     }
 
-    public synchronized static void loadNSSLibs(final Context context) {
+    public synchronized static void loadNSSLibs(final Context context, final String apkName) {
         if (sNSSLibsLoaded) {
             return;
         }
 
         loadMozGlue(context);
         loadLibsSetupLocked(context);
-        loadNSSLibsNative();
+        loadNSSLibsNative(apkName);
         sNSSLibsLoaded = true;
     }
 
@@ -462,9 +464,9 @@ public final class GeckoLoader {
         sMozGlueLoaded = true;
     }
 
-    public synchronized static void loadGeckoLibs(final Context context) {
+    public synchronized static void loadGeckoLibs(final Context context, final String apkName) {
         loadLibsSetupLocked(context);
-        loadGeckoLibsNative();
+        loadGeckoLibsNative(apkName);
     }
 
     @SuppressWarnings("serial")
@@ -490,9 +492,9 @@ public final class GeckoLoader {
 
     // These methods are implemented in mozglue/android/APKOpen.cpp
     public static native void nativeRun(String[] args, int prefsFd, int prefMapFd, int ipcFd, int crashFd, int crashAnnotationFd);
-    private static native void loadGeckoLibsNative();
-    private static native void loadSQLiteLibsNative();
-    private static native void loadNSSLibsNative();
+    private static native void loadGeckoLibsNative(String apkName);
+    private static native void loadSQLiteLibsNative(String apkName);
+    private static native void loadNSSLibsNative(String apkName);
     public static native boolean neonCompatible();
     public static native void suppressCrashDialog();
 }

mobile/android/gradle/with_gecko_binaries.gradle

@@ -76,6 +76,13 @@ ext.configureVariantWithGeckoBinaries = { variant ->
     }
 
     def syncAssetsFromDistDir = task("syncAssetsFromDistDirFor${variant.name.capitalize()}", type: Sync) {
+        onlyIf {
+            if (source.empty) {
+                throw new StopExecutionException("Required assets not found in ${distDir}/assets.  Have you built and packaged?")
+            }
+            return true
+        }
+
         into("${project.buildDir}/moz.build/src/${variant.name}/assets")
         from("${distDir}/assets") {
             exclude 'omni.ja'

mobile/android/installer/package-manifest.in

@@ -24,12 +24,12 @@
 @BINPATH@/hyphenation/*
 @BINPATH@/localization/*
 
-; We want fennec/lib for both Fennec and GeckoView, so we turn
-; geckoview/lib into fennec/lib.
+; We want fennec/assets for both Fennec and GeckoView, so we turn
+; geckoview/assets into fennec/assets.
 #ifndef MOZ_GECKOVIEW_JAR
-[lib destdir="lib/@ANDROID_CPU_ARCH@"]
+[assets xz_compress="1" destdir="assets/@ANDROID_CPU_ARCH@"]
 #else
-[lib destdir="../fennec/lib/@ANDROID_CPU_ARCH@"]
+[assets xz_compress="1" destdir="../fennec/assets/@ANDROID_CPU_ARCH@"]
 #endif
 
 #ifndef MOZ_STATIC_JS
@@ -78,6 +78,14 @@
 @BINPATH@/@DLL_PREFIX@mozsqlite3@DLL_SUFFIX@
 #endif
 
+; We want fennec/lib for both Fennec and GeckoView, so we turn
+; geckoview/lib into fennec/lib.
+#ifndef MOZ_GECKOVIEW_JAR
+[lib destdir="lib/@ANDROID_CPU_ARCH@"]
+#else
+[lib destdir="../fennec/lib/@ANDROID_CPU_ARCH@"]
+#endif
+
 @BINPATH@/@DLL_PREFIX@mozglue@DLL_SUFFIX@
 # This should be MOZ_CHILD_PROCESS_NAME, but that has a "lib/" prefix.
 @BINPATH@/@MOZ_CHILD_PROCESS_NAME@

mobile/android/tests/browser/robocop/src/org/mozilla/gecko/FennecNativeActions.java

@@ -38,7 +38,7 @@ public class FennecNativeActions implements Actions {
         mInstr = instrumentation;
         mAsserter = asserter;
 
-        GeckoLoader.loadSQLiteLibs(activity);
+        GeckoLoader.loadSQLiteLibs(activity, activity.getApplication().getPackageResourcePath());
     }
 
     class GeckoEventExpecter implements RepeatedEventExpecter {

mozglue/android/APKOpen.cpp

@@ -184,31 +184,34 @@ delete_mapping(const char *name)
 }
 
 static UniquePtr<char[]>
-getUnpackedLibraryName(const char* libraryName)
+getAPKLibraryName(const char* apkName, const char* libraryName)
 {
-  const char *datadir = getenv("GRE_HOME");
-
-  size_t len = strlen(datadir) + sizeof("/lib/") + strlen(libraryName);
-  auto file = MakeUnique<char[]>(len);
-  snprintf(file.get(), len, "%s/lib/%s", datadir, libraryName);
+#define APK_ASSETS_PATH "!/assets/" ANDROID_CPU_ARCH "/"
+  size_t filenameLength = strlen(apkName) +
+    sizeof(APK_ASSETS_PATH) + 	// includes \0 terminator
+    strlen(libraryName);
+  auto file = MakeUnique<char[]>(filenameLength);
+  snprintf(file.get(), filenameLength, "%s" APK_ASSETS_PATH "%s",
+	   apkName, libraryName);
   return file;
+#undef APK_ASSETS_PATH
 }
 
 static void*
-dlopenLibrary(const char* libraryName)
+dlopenAPKLibrary(const char* apkName, const char* libraryName)
 {
-  return __wrap_dlopen(getUnpackedLibraryName(libraryName).get(), RTLD_GLOBAL | RTLD_LAZY);
+  return __wrap_dlopen(getAPKLibraryName(apkName, libraryName).get(), RTLD_GLOBAL | RTLD_LAZY);
 }
 
 static mozglueresult
-loadGeckoLibs()
+loadGeckoLibs(const char *apkName)
 {
   TimeStamp t0 = TimeStamp::Now();
   struct rusage usage1_thread, usage1;
   getrusage(RUSAGE_THREAD, &usage1_thread);
   getrusage(RUSAGE_SELF, &usage1);
 
-  gBootstrap = GetBootstrap(getUnpackedLibraryName("libxul.so").get());
+  gBootstrap = GetBootstrap(getAPKLibraryName(apkName, "libxul.so").get());
   if (!gBootstrap) {
     __android_log_print(ANDROID_LOG_ERROR, "GeckoLibLoad", "Couldn't get a handle to libxul!");
     return FAILURE;
@@ -237,20 +240,20 @@ loadGeckoLibs()
   return SUCCESS;
 }
 
-static mozglueresult loadNSSLibs();
+static mozglueresult loadNSSLibs(const char *apkName);
 
 static mozglueresult
-loadSQLiteLibs()
+loadSQLiteLibs(const char *apkName)
 {
   if (sqlite_handle)
     return SUCCESS;
 
 #ifdef MOZ_FOLD_LIBS
-  if (loadNSSLibs() != SUCCESS)
+  if (loadNSSLibs(apkName) != SUCCESS)
     return FAILURE;
 #else
 
-  sqlite_handle = dlopenLibrary("libmozsqlite3.so");
+  sqlite_handle = dlopenAPKLibrary(apkName, "libmozsqlite3.so");
   if (!sqlite_handle) {
     __android_log_print(ANDROID_LOG_ERROR, "GeckoLibLoad", "Couldn't get a handle to libmozsqlite3!");
     return FAILURE;
@@ -262,17 +265,17 @@ loadSQLiteLibs()
 }
 
 static mozglueresult
-loadNSSLibs()
+loadNSSLibs(const char *apkName)
 {
   if (nss_handle && nspr_handle && plc_handle)
     return SUCCESS;
 
-  nss_handle = dlopenLibrary("libnss3.so");
+  nss_handle = dlopenAPKLibrary(apkName, "libnss3.so");
 
 #ifndef MOZ_FOLD_LIBS
-  nspr_handle = dlopenLibrary("libnspr4.so");
+  nspr_handle = dlopenAPKLibrary(apkName, "libnspr4.so");
 
-  plc_handle = dlopenLibrary("libplc4.so");
+  plc_handle = dlopenAPKLibrary(apkName, "libplc4.so");
 #endif
 
   if (!nss_handle) {
@@ -296,34 +299,58 @@ loadNSSLibs()
 }
 
 extern "C" APKOPEN_EXPORT void MOZ_JNICALL
-Java_org_mozilla_gecko_mozglue_GeckoLoader_loadGeckoLibsNative(JNIEnv *jenv, jclass jGeckoAppShellClass)
+Java_org_mozilla_gecko_mozglue_GeckoLoader_loadGeckoLibsNative(JNIEnv *jenv, jclass jGeckoAppShellClass, jstring jApkName)
 {
   jenv->GetJavaVM(&sJavaVM);
 
-  int res = loadGeckoLibs();
+  const char* str;
+  // XXX: java doesn't give us true UTF8, we should figure out something
+  // better to do here
+  str = jenv->GetStringUTFChars(jApkName, nullptr);
+  if (str == nullptr)
+    return;
+
+  int res = loadGeckoLibs(str);
   if (res != SUCCESS) {
     JNI_Throw(jenv, "java/lang/Exception", "Error loading gecko libraries");
   }
+  jenv->ReleaseStringUTFChars(jApkName, str);
 }
 
 extern "C" APKOPEN_EXPORT void MOZ_JNICALL
-Java_org_mozilla_gecko_mozglue_GeckoLoader_loadSQLiteLibsNative(JNIEnv *jenv, jclass jGeckoAppShellClass) {
+Java_org_mozilla_gecko_mozglue_GeckoLoader_loadSQLiteLibsNative(JNIEnv *jenv, jclass jGeckoAppShellClass, jstring jApkName) {
+  const char* str;
+  // XXX: java doesn't give us true UTF8, we should figure out something
+  // better to do here
+  str = jenv->GetStringUTFChars(jApkName, nullptr);
+  if (str == nullptr)
+    return;
+
   __android_log_print(ANDROID_LOG_ERROR, "GeckoLibLoad", "Load sqlite start\n");
-  mozglueresult rv = loadSQLiteLibs();
+  mozglueresult rv = loadSQLiteLibs(str);
   if (rv != SUCCESS) {
       JNI_Throw(jenv, "java/lang/Exception", "Error loading sqlite libraries");
   }
   __android_log_print(ANDROID_LOG_ERROR, "GeckoLibLoad", "Load sqlite done\n");
+  jenv->ReleaseStringUTFChars(jApkName, str);
 }
 
 extern "C" APKOPEN_EXPORT void MOZ_JNICALL
-Java_org_mozilla_gecko_mozglue_GeckoLoader_loadNSSLibsNative(JNIEnv *jenv, jclass jGeckoAppShellClass) {
+Java_org_mozilla_gecko_mozglue_GeckoLoader_loadNSSLibsNative(JNIEnv *jenv, jclass jGeckoAppShellClass, jstring jApkName) {
+  const char* str;
+  // XXX: java doesn't give us true UTF8, we should figure out something
+  // better to do here
+  str = jenv->GetStringUTFChars(jApkName, nullptr);
+  if (str == nullptr)
+    return;
+
   __android_log_print(ANDROID_LOG_ERROR, "GeckoLibLoad", "Load nss start\n");
-  mozglueresult rv = loadNSSLibs();
+  mozglueresult rv = loadNSSLibs(str);
   if (rv != SUCCESS) {
     JNI_Throw(jenv, "java/lang/Exception", "Error loading nss libraries");
   }
   __android_log_print(ANDROID_LOG_ERROR, "GeckoLibLoad", "Load nss done\n");
+  jenv->ReleaseStringUTFChars(jApkName, str);
 }
 
 static char**
@@ -395,13 +422,22 @@ Java_org_mozilla_gecko_mozglue_GeckoLoader_nativeRun(JNIEnv *jenv, jclass jc, jo
 extern "C" APKOPEN_EXPORT mozglueresult
 ChildProcessInit(int argc, char* argv[])
 {
-  if (loadNSSLibs() != SUCCESS) {
+  int i;
+  for (i = 0; i < (argc - 1); i++) {
+    if (strcmp(argv[i], "-greomni"))
+      continue;
+
+    i = i + 1;
+    break;
+  }
+
+  if (loadNSSLibs(argv[i]) != SUCCESS) {
     return FAILURE;
   }
-  if (loadSQLiteLibs() != SUCCESS) {
+  if (loadSQLiteLibs(argv[i]) != SUCCESS) {
     return FAILURE;
   }
-  if (loadGeckoLibs() != SUCCESS) {
+  if (loadGeckoLibs(argv[i]) != SUCCESS) {
     return FAILURE;
   }
 

python/mozbuild/mozbuild/action/package_fennec_apk.py

@@ -84,7 +84,16 @@ def package_fennec_apk(inputs=[], omni_ja=None,
     for assets_dir in assets_dirs:
         finder = FileFinder(assets_dir)
         for p, f in finder.find('**'):
-            add(mozpath.join('assets', p), f)
+            compress = None  # Take default from Jarrer.
+            if p.endswith('.so'):
+                # Asset libraries are special.
+                if f.open().read(5)[1:] == '7zXZ':
+                    print('%s is already compressed' % p)
+                    # We need to store (rather than deflate) compressed libraries
+                    # (even if we don't compress them ourselves).
+                    compress = False
+
+            add(mozpath.join('assets', p), f, compress=compress)
 
     for lib_dir in lib_dirs:
         finder = FileFinder(lib_dir)