Bug 1012462 - Part 5: Add distribution support in SuggestedSites (r=rnewman)

mobile/android/base/db/SuggestedSites.java

@@ -6,6 +6,7 @@
 package org.mozilla.gecko.db;
 
 import android.content.Context;
+import android.content.ContentResolver;
 import android.content.SharedPreferences;
 import android.database.Cursor;
 import android.database.MatrixCursor;
@@ -14,7 +15,11 @@ import android.net.Uri;
 import android.text.TextUtils;
 import android.util.Log;
 
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.OutputStreamWriter;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
@@ -22,14 +27,18 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Scanner;
 import java.util.Set;
 
 import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
 
+import org.mozilla.gecko.BrowserLocaleManager;
 import org.mozilla.gecko.GeckoSharedPrefs;
+import org.mozilla.gecko.GeckoProfile;
 import org.mozilla.gecko.R;
+import org.mozilla.gecko.distribution.Distribution;
 import org.mozilla.gecko.db.BrowserContract;
 import org.mozilla.gecko.mozglue.RobocopTarget;
 import org.mozilla.gecko.preferences.GeckoPreferences;
@@ -62,6 +71,9 @@ public class SuggestedSites {
     // SharedPreference key for suggested sites that should be hidden.
     public static final String PREF_SUGGESTED_SITES_HIDDEN = "suggestedSites.hidden";
 
+    // File in profile dir with the list of suggested sites.
+    private static final String FILENAME = "suggestedsites.json";
+
     private static final String[] COLUMNS = new String[] {
         BrowserContract.SuggestedSites._ID,
         BrowserContract.SuggestedSites.URL,
@@ -129,15 +141,57 @@ public class SuggestedSites {
     }
 
     private final Context context;
+    private final Distribution distribution;
+    private final File file;
     private Map<String, Site> cachedSites;
     private Locale cachedLocale;
     private Set<String> cachedBlacklist;
 
     public SuggestedSites(Context appContext) {
-        context = appContext;
+        this(appContext, null);
     }
 
-    private Map<String, Site> loadSites(String jsonString) {
+    public SuggestedSites(Context appContext, Distribution distribution) {
+        this(appContext, distribution,
+             GeckoProfile.get(appContext).getFile(FILENAME));
+    }
+
+    public SuggestedSites(Context appContext, Distribution distribution, File file) {
+        this.context = appContext;
+        this.distribution = distribution;
+        this.file = file;
+    }
+
+    /**
+     * Return the current locale and its fallback (en_US) in order.
+     */
+    private static List<Locale> getAcceptableLocales() {
+        final List<Locale> locales = new ArrayList<Locale>();
+
+        final Locale defaultLocale = Locale.getDefault();
+        locales.add(defaultLocale);
+
+        if (!defaultLocale.equals(Locale.US)) {
+            locales.add(Locale.US);
+        }
+
+        return locales;
+    }
+
+    private static Map<String, Site> loadSites(File f) throws IOException {
+        Scanner scanner = null;
+
+        try {
+            scanner = new Scanner(f, "UTF-8");
+            return loadSites(scanner.useDelimiter("\\A").next());
+        } finally {
+            if (scanner != null) {
+                scanner.close();
+            }
+        }
+    }
+
+    private static Map<String, Site> loadSites(String jsonString) {
         if (TextUtils.isEmpty(jsonString)) {
             return null;
         }
@@ -150,7 +204,7 @@ public class SuggestedSites {
 
             final int count = jsonSites.length();
             for (int i = 0; i < count; i++) {
-                final Site site = new Site((JSONObject) jsonSites.get(i));
+                final Site site = new Site(jsonSites.getJSONObject(i));
                 sites.put(site.url, site);
             }
         } catch (Exception e) {
@@ -161,8 +215,122 @@ public class SuggestedSites {
         return sites;
     }
 
-    private Map<String, Site> loadFromFile() {
+    /**
-        // Do nothing for now
+     * Saves suggested sites file to disk. Access to this method should
+     * be synchronized on 'file'.
+     */
+    private static void saveSites(File f, Map<String, Site> sites) {
+        ThreadUtils.assertNotOnUiThread();
+
+        if (sites == null || sites.isEmpty()) {
+            return;
+        }
+
+        OutputStreamWriter osw = null;
+
+        try {
+            final JSONArray jsonSites = new JSONArray();
+            for (Site site : sites.values()) {
+                jsonSites.put(site.toJSON());
+            }
+
+            osw = new OutputStreamWriter(new FileOutputStream(f), "UTF-8");
+
+            final String jsonString = jsonSites.toString();
+            osw.write(jsonString, 0, jsonString.length());
+        } catch (Exception e) {
+            Log.e(LOGTAG, "Failed to save suggested sites", e);
+        } finally {
+            if (osw != null) {
+                try {
+                    osw.close();
+                } catch (IOException e) {
+                    // Ignore.
+                }
+            }
+        }
+    }
+
+    private void maybeWaitForDistribution() {
+        if (distribution == null) {
+            return;
+        }
+
+        distribution.addOnDistributionReadyCallback(new Runnable() {
+            @Override
+            public void run() {
+                Log.d(LOGTAG, "Running post-distribution task: suggested sites.");
+
+                // If distribution doesn't exist, simply continue to load
+                // suggested sites directly from resources. See refresh().
+                if (!distribution.exists()) {
+                    return;
+                }
+
+                // Merge suggested sites from distribution with the
+                // default ones. Distribution takes precedence.
+                Map<String, Site> sites = loadFromDistribution(distribution);
+                if (sites == null) {
+                    sites = new LinkedHashMap<String, Site>();
+                }
+                sites.putAll(loadFromResource());
+
+                // Update cached list of sites.
+                setCachedSites(sites);
+
+                // Save the result to disk.
+                synchronized (file) {
+                    saveSites(file, sites);
+                }
+
+                // Then notify any active loaders about the changes.
+                final ContentResolver cr = context.getContentResolver();
+                cr.notifyChange(BrowserContract.SuggestedSites.CONTENT_URI, null);
+            }
+        });
+    }
+
+    /**
+     * Loads suggested sites from a distribution file either matching the
+     * current locale or with the fallback locale (en-US).
+     *
+     * It's assumed that the given distribution instance is ready to be
+     * used and exists.
+     */
+    private static Map<String, Site> loadFromDistribution(Distribution dist) {
+        for (Locale locale : getAcceptableLocales()) {
+            try {
+                final String languageTag = BrowserLocaleManager.getLanguageTag(locale);
+                final String path = String.format("suggestedsites/locales/%s/%s",
+                                                  languageTag, FILENAME);
+
+                final File f = dist.getDistributionFile(path);
+                if (f == null) {
+                    Log.d(LOGTAG, "No suggested sites for locale: " + languageTag);
+                    continue;
+                }
+
+                return loadSites(f);
+            } catch (Exception e) {
+                Log.e(LOGTAG, "Failed to open suggested sites for locale " +
+                              locale + " in distribution.", e);
+            }
+        }
+
+        return null;
+    }
+
+    private Map<String, Site> loadFromProfile() {
+        try {
+            synchronized (file) {
+                return loadSites(file);
+            }
+        } catch (FileNotFoundException e) {
+            maybeWaitForDistribution();
+        } catch (IOException e) {
+            // Fall through, return null.
+        }
+
         return null;
     }
 
@@ -174,27 +342,28 @@ public class SuggestedSites {
         }
     }
 
+    private synchronized void setCachedSites(Map<String, Site> sites) {
+        cachedSites = Collections.unmodifiableMap(sites);
+        cachedLocale = Locale.getDefault();
+    }
+
     /**
      * Refreshes the cached list of sites either from the default raw
      * source or standard file location. This will be called on every
      * cache miss during a {@code get()} call.
      */
     private void refresh() {
-        Log.d(LOGTAG, "Refreshing tiles from file");
+        Log.d(LOGTAG, "Refreshing suggested sites from file");
 
-        Map<String, Site> sites = loadFromFile();
+        Map<String, Site> sites = loadFromProfile();
         if (sites == null) {
             sites = loadFromResource();
         }
 
-        // Nothing to cache, bail.
-        if (sites == null) {
-            return;
-        }
-
         // Update cached list of sites.
-        cachedSites = Collections.unmodifiableMap(sites);
+        if (sites != null) {
-        cachedLocale = Locale.getDefault();
+            setCachedSites(sites);
+        }
     }
 
     private boolean isEnabled() {

mobile/android/tests/browser/junit3/src/tests/TestSuggestedSites.java

@@ -4,18 +4,30 @@
 package org.mozilla.gecko.browser.tests;
 
 import android.content.Context;
+import android.content.ContentResolver;
 import android.content.res.Resources;
 import android.content.SharedPreferences;
 import android.database.Cursor;
+import android.database.ContentObserver;
 import android.net.Uri;
+import android.os.Handler;
+import android.os.SystemClock;
 import android.test.mock.MockResources;
 import android.test.RenamingDelegatingContext;
 
 import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileOutputStream;
 import java.io.InputStream;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URI;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
+import java.util.jar.JarInputStream;
+import java.util.Map;
 import java.util.List;
 import java.util.Locale;
 import java.util.Set;
@@ -23,9 +35,11 @@ import java.util.Set;
 import org.json.JSONArray;
 import org.json.JSONObject;
 
+import org.mozilla.gecko.BrowserLocaleManager;
 import org.mozilla.gecko.R;
 import org.mozilla.gecko.db.BrowserContract;
 import org.mozilla.gecko.db.SuggestedSites;
+import org.mozilla.gecko.distribution.Distribution;
 import org.mozilla.gecko.GeckoSharedPrefs;
 import org.mozilla.gecko.preferences.GeckoPreferences;
 
@@ -79,10 +93,64 @@ public class TestSuggestedSites extends BrowserTestCase {
         }
     }
 
+    private static class TestDistribution extends Distribution {
+        private final Context context;
+        private final Map<Locale, File> filesPerLocale;
+
+        public TestDistribution(Context context) {
+            super(context);
+            this.context = context;
+            this.filesPerLocale = new HashMap<Locale, File>();
+        }
+
+        @Override
+        public File getDistributionFile(String name) {
+            for (Locale locale : filesPerLocale.keySet()) {
+                if (name.startsWith("suggestedsites/locales/" + BrowserLocaleManager.getLanguageTag(locale))) {
+                    return filesPerLocale.get(locale);
+                }
+            }
+
+            return null;
+        }
+
+        @Override
+        public boolean exists() {
+            return true;
+        }
+
+        public void setFileForLocale(Locale locale, File file) {
+            filesPerLocale.put(locale, file);
+        }
+
+        public void start() {
+            doInit();
+        }
+    }
+
+    class TestObserver extends ContentObserver {
+        private final Object changeLock;
+
+        public TestObserver(Object changeLock) {
+            super(null);
+            this.changeLock = changeLock;
+        }
+
+        @Override
+        public void onChange(boolean selfChange) {
+            synchronized(changeLock) {
+                changeLock.notifyAll();
+            }
+        }
+    }
+
     private static final int DEFAULT_LIMIT = 6;
 
+    private static final String DIST_PREFIX = "dist";
+
     private TestContext context;
     private TestResources resources;
+    private List<File> tempFiles;
 
     private String generateSites(int n) {
         return generateSites(n, "");
@@ -108,6 +176,32 @@ public class TestSuggestedSites extends BrowserTestCase {
         return sites.toString();
     }
 
+    private File createDistSuggestedSitesFile(int n) {
+        FileOutputStream fos = null;
+
+        try {
+            File distFile = File.createTempFile("distrosites", ".json",
+                                                context.getCacheDir());
+
+            fos = new FileOutputStream(distFile);
+            fos.write(generateSites(n, DIST_PREFIX).getBytes());
+
+            return distFile;
+        } catch (IOException e) {
+            fail("Failed to create temp suggested sites file");
+        } finally {
+            if (fos != null) {
+                try {
+                    fos.close();
+                } catch (IOException e) {
+                    // Ignore.
+                }
+            }
+        }
+
+        return null;
+    }
+
     private void checkCursorCount(String content, int expectedCount) {
         checkCursorCount(content, expectedCount, DEFAULT_LIMIT);
     }
@@ -122,10 +216,14 @@ public class TestSuggestedSites extends BrowserTestCase {
     protected void setUp() {
         context = new TestContext(getApplicationContext());
         resources = (TestResources) context.getResources();
+        tempFiles = new ArrayList<File>();
     }
 
     protected void tearDown() {
         context.clearUsedPrefs();
+        for (File f : tempFiles) {
+            f.delete();
+        }
     }
 
     public void testCount() {
@@ -308,4 +406,95 @@ public class TestSuggestedSites extends BrowserTestCase {
         assertEquals(5, c.getCount());
         c.close();
     }
+
+    public void testDistribution() {
+        final int DIST_COUNT = 2;
+        final int DEFAULT_COUNT = 3;
+
+        File sitesFile = new File(context.getCacheDir(),
+                                  "suggestedsites-" + SystemClock.uptimeMillis() + ".json");
+        tempFiles.add(sitesFile);
+        assertFalse(sitesFile.exists());
+
+        File distFile = createDistSuggestedSitesFile(DIST_COUNT);
+        tempFiles.add(distFile);
+        assertTrue(distFile.exists());
+
+        // Init distribution with the mock file.
+        TestDistribution distribution = new TestDistribution(context);
+        distribution.setFileForLocale(Locale.getDefault(), distFile);
+        distribution.start();
+
+        // Init suggested sites with default values.
+        resources.setSuggestedSitesResource(generateSites(DEFAULT_COUNT));
+        SuggestedSites suggestedSites =
+                new SuggestedSites(context, distribution, sitesFile);
+
+        Object changeLock = new Object();
+
+        // Watch for change notifications on suggested sites.
+        ContentResolver cr = context.getContentResolver();
+        ContentObserver observer = new TestObserver(changeLock);
+        cr.registerContentObserver(BrowserContract.SuggestedSites.CONTENT_URI,
+                                   false, observer);
+
+        // The initial query will not contain the distribution sites
+        // yet. This will happen asynchronously once the distribution
+        // is installed.
+        Cursor c1 = null;
+        try {
+            c1 = suggestedSites.get(DEFAULT_LIMIT);
+            assertEquals(DEFAULT_COUNT, c1.getCount());
+        } finally {
+            if (c1 != null) {
+                c1.close();
+            }
+        }
+
+        synchronized(changeLock) {
+            try {
+                changeLock.wait(5000);
+            } catch (InterruptedException ie) {
+                fail("No change notification after fetching distribution file");
+            }
+        }
+
+        // Target file should exist after distribution is deployed.
+        assertTrue(sitesFile.exists());
+        cr.unregisterContentObserver(observer);
+
+        Cursor c2 = null;
+        try {
+            c2 = suggestedSites.get(DEFAULT_LIMIT);
+
+            // The next query should contain the distribution contents.
+            assertEquals(DIST_COUNT + DEFAULT_COUNT, c2.getCount());
+
+            // The first items should be from the distribution
+            for (int i = 0; i < DIST_COUNT; i++) {
+                c2.moveToPosition(i);
+
+                String url = c2.getString(c2.getColumnIndexOrThrow(BrowserContract.SuggestedSites.URL));
+                assertEquals(DIST_PREFIX +  "url" + i, url);
+
+                String title = c2.getString(c2.getColumnIndexOrThrow(BrowserContract.SuggestedSites.TITLE));
+                assertEquals(DIST_PREFIX +  "title" + i, title);
+            }
+
+            // The remaining items should be the default ones
+            for (int i = 0; i < c2.getCount() - DIST_COUNT; i++) {
+                c2.moveToPosition(i + DIST_COUNT);
+
+                String url = c2.getString(c2.getColumnIndexOrThrow(BrowserContract.SuggestedSites.URL));
+                assertEquals("url" + i, url);
+
+                String title = c2.getString(c2.getColumnIndexOrThrow(BrowserContract.SuggestedSites.TITLE));
+                assertEquals("title" + i, title);
+            }
+        } finally {
+            if (c2 != null) {
+                c2.close();
+            }
+        }
+    }
 }