Bug 713283 - Limit imported history entries. Sort by approximate frecency. Update visits count. r=lucasr

mobile/android/base/ProfileMigrator.java

@@ -69,12 +69,6 @@ public class ProfileMigrator {
     private File mProfileDir;
     private ContentResolver mCr;
 
-    /*
-      Amount of Android history entries we will remember
-      to prevent moving their last access date backwards.
-    */
-    private static final int MAX_HISTORY_TO_CHECK = 1000;
-
     /*
        These queries are derived from the low-level Places schema
        https://developer.mozilla.org/en/The_Places_database
@@ -88,15 +82,24 @@ public class ProfileMigrator {
     private final String bookmarkUrl   = "a_url";
     private final String bookmarkTitle = "a_title";
 
+    /*
+      The sort criterion here corresponds to the one used for the
+      Awesomebar results. It's an simplification of Frecency.
+      We must divide date by 1000 due to the micro (Places)
+      vs milli (Android) distiction.
+    */
     private final String historyQuery =
-        "SELECT places.url AS a_url, places.title AS a_title, "
+        "SELECT places.url AS a_url, places.title AS a_title,"
-        + "history.visit_date AS a_date FROM "
+        + "MAX(history.visit_date) AS a_date, COUNT(*) AS a_visits, "
-        + "(moz_historyvisits AS history JOIN moz_places AS places ON "
+        // see BrowserDB.filterAllSites for this formula
-        + "places.id = history.place_id) WHERE places.hidden <> 1 "
+        + "MAX(1, (((MAX(history.visit_date)/1000) - ?) / 86400000 + 120)) AS a_recent "
-        + "ORDER BY history.visit_date DESC";
+        + "FROM (moz_historyvisits AS history JOIN moz_places AS places "
-    private final String historyUrl   = "a_url";
+        + "ON places.id = history.place_id) WHERE places.hidden <> 1 "
-    private final String historyTitle = "a_title";
+        + "GROUP BY a_url ORDER BY a_visits * a_recent DESC LIMIT ?";
-    private final String historyDate  = "a_date";
+    private final String historyUrl    = "a_url";
+    private final String historyTitle  = "a_title";
+    private final String historyDate   = "a_date";
+    private final String historyVisits = "a_visits";
 
     private final String faviconQuery =
         "SELECT places.url AS a_url, favicon.data AS a_data, "
@@ -117,10 +120,11 @@ public class ProfileMigrator {
 
     private class PlacesTask implements Runnable {
         // Get a list of the last times an URL was accessed
-        protected Map<String, Long> gatherAndroidHistory() {
+        protected Map<String, Long> gatherBrowserDBHistory() {
             Map<String, Long> history = new HashMap<String, Long>();
 
-            Cursor cursor = BrowserDB.getRecentHistory(mCr, MAX_HISTORY_TO_CHECK);
+            Cursor cursor =
+                BrowserDB.getRecentHistory(mCr, BrowserDB.getMaxHistoryCount());
             final int urlCol =
                 cursor.getColumnIndexOrThrow(BrowserDB.URLColumns.URL);
             final int dateCol =
@@ -142,47 +146,57 @@ public class ProfileMigrator {
             return history;
         }
 
-        protected void addHistory(Map<String, Long> androidHistory,
+        protected void addHistory(Map<String, Long> browserDBHistory,
-                                  String url, String title, long date) {
+                                  String url, String title, long date, int visits) {
             boolean allowUpdate = false;
 
-            if (!androidHistory.containsKey(url)) {
+            if (!browserDBHistory.containsKey(url)) {
-                // Android doesn't know the URL, allow it to be
+                // BrowserDB doesn't know the URL, allow it to be
-                // inserted with places date
+                // inserted with places date.
                 allowUpdate = true;
             } else {
-                long androidDate = androidHistory.get(url);
+                long androidDate = browserDBHistory.get(url);
                 if (androidDate < date) {
-                    // Places URL hit is newer than Android,
+                    // Places URL hit is newer than BrowserDB,
-                    // allow it to be updated with places date
+                    // allow it to be updated with places date.
                     allowUpdate = true;
                 }
             }
 
             if (allowUpdate) {
                 BrowserDB.updateVisitedHistory(mCr, url);
-                BrowserDB.updateHistoryDate(mCr, url, date);
+                // The above records one visit. Subtract that one visit here.
-                if (title != null) {
+                BrowserDB.updateHistoryEntry(mCr, url, title, date, visits - 1);
-                    BrowserDB.updateHistoryTitle(mCr, url, title);
-                }
             }
         }
 
         protected void migrateHistory(SQLiteBridge db) {
-            Map<String, Long> androidHistory = gatherAndroidHistory();
+            Map<String, Long> browserDBHistory = gatherBrowserDBHistory();
             final ArrayList<String> placesHistory = new ArrayList<String>();
 
             try {
-                ArrayList<Object[]> queryResult = db.query(historyQuery);
+                final String[] queryParams = new String[] {
+                    /* current time */
+                    Long.toString(System.currentTimeMillis()),
+                    /*
+                       History entries to return. No point
+                       in retrieving more than we can store.
+                     */
+                    Integer.toString(BrowserDB.getMaxHistoryCount())
+                };
+                ArrayList<Object[]> queryResult =
+                    db.query(historyQuery, queryParams);
                 final int urlCol = db.getColumnIndex(historyUrl);
                 final int titleCol = db.getColumnIndex(historyTitle);
                 final int dateCol = db.getColumnIndex(historyDate);
+                final int visitsCol = db.getColumnIndex(historyVisits);
 
                 for (Object[] resultRow: queryResult) {
                     String url = (String)resultRow[urlCol];
                     String title = (String)resultRow[titleCol];
                     long date = Long.parseLong((String)(resultRow[dateCol])) / (long)1000;
-                    addHistory(androidHistory, url, title, date);
+                    int visits = Integer.parseInt((String)(resultRow[visitsCol]));
+                    addHistory(browserDBHistory, url, title, date, visits);
                     placesHistory.add(url);
                 }
             } catch (SQLiteBridgeException e) {
@@ -240,6 +254,9 @@ public class ProfileMigrator {
         }
 
         protected void migrateFavicons(SQLiteBridge db) {
+            // Determine which URLs are really stored.
+            Map<String, Long> browserDBHistory = gatherBrowserDBHistory();
+
             try {
                 ArrayList<Object[]> queryResult = db.query(faviconQuery);
                 final int urlCol = db.getColumnIndex(faviconUrl);
@@ -248,9 +265,16 @@ public class ProfileMigrator {
 
                 for (Object[] resultRow: queryResult) {
                     String url = (String)resultRow[urlCol];
-                    String mime = (String)resultRow[mimeCol];
+                    // Don't try to store a favicon if we don't know the URL.
-                    ByteBuffer dataBuff = (ByteBuffer)resultRow[dataCol];
+                    if (browserDBHistory.containsKey(url)) {
-                    addFavicon(url, mime, dataBuff);
+                        String mime = (String)resultRow[mimeCol];
+                        // Some GIFs can cause us to lock up completely
+                        // without exceptions or anything. Not cool.
+                        if (mime.compareTo("image/gif") != 0) {
+                            ByteBuffer dataBuff = (ByteBuffer)resultRow[dataCol];
+                            addFavicon(url, mime, dataBuff);
+                        }
+                    }
                 }
             } catch (SQLiteBridgeException e) {
                 Log.i(LOGTAG, "Failed to get favicons: " + e.getMessage());

mobile/android/base/db/AndroidBrowserDB.java

@@ -118,9 +118,31 @@ public class AndroidBrowserDB implements BrowserDB.BrowserDBIface {
                   new String[] { uri });
     }
 
-    public void updateHistoryDate(ContentResolver cr, String uri, long date) {
+    public void updateHistoryEntry(ContentResolver cr, String uri, String title,
+                                   long date, int visits) {
+        int oldVisits = 0;
+        Cursor cursor = null;
+        try {
+            cursor = cr.query(Browser.BOOKMARKS_URI,
+                              new String[] { Browser.BookmarkColumns.VISITS },
+                              Browser.BookmarkColumns.URL + " = ?",
+                              new String[] { uri },
+                              null);
+
+            if (cursor.moveToFirst()) {
+                oldVisits = cursor.getInt(0);
+            }
+        } finally {
+            if (cursor != null)
+                cursor.close();
+        }
+
         ContentValues values = new ContentValues();
         values.put(Browser.BookmarkColumns.DATE, date);
+        values.put(Browser.BookmarkColumns.VISITS, oldVisits + visits);
+        if (title != null) {
+            values.put(Browser.BookmarkColumns.TITLE, title);
+        }
 
         cr.update(Browser.BOOKMARKS_URI,
                   values,
@@ -145,7 +167,8 @@ public class AndroidBrowserDB implements BrowserDB.BrowserDBIface {
                                            BookmarkColumns.URL,
                                            BookmarkColumns.TITLE,
                                            BookmarkColumns.FAVICON,
-                                           BookmarkColumns.DATE },
+                                           BookmarkColumns.DATE,
+                                           BookmarkColumns.VISITS },
                             // Bookmarks that have not been visited have a date value
                             // of 0, so don't pick them up in the history view.
                             Browser.BookmarkColumns.DATE + " > 0",
@@ -155,6 +178,11 @@ public class AndroidBrowserDB implements BrowserDB.BrowserDBIface {
         return new AndroidDBCursor(c);
     }
 
+    public int getMaxHistoryCount() {
+        // Valid for Android versions up to 4.0.
+        return 250;
+    }
+
     public void clearHistory(ContentResolver cr) {
         Browser.clearHistory(cr);
     }
@@ -357,6 +385,8 @@ public class AndroidBrowserDB implements BrowserDB.BrowserDBIface {
                 columnName = URL_COLUMN_THUMBNAIL;
             } else if (columnName.equals(BrowserDB.URLColumns.DATE_LAST_VISITED)) {
                 columnName = Browser.BookmarkColumns.DATE;
+            } else if (columnName.equals(BrowserDB.URLColumns.VISITS)) {
+                columnName = Browser.BookmarkColumns.VISITS;
             }
 
             return columnName;

mobile/android/base/db/BrowserDB.java

@@ -50,6 +50,7 @@ public class BrowserDB {
         public static String FAVICON = "favicon";
         public static String THUMBNAIL = "thumbnail";
         public static String DATE_LAST_VISITED = "date-last-visited";
+        public static String VISITS = "visits";
     }
 
     private static BrowserDBIface sDb;
@@ -63,12 +64,15 @@ public class BrowserDB {
 
         public void updateHistoryTitle(ContentResolver cr, String uri, String title);
 
-        public void updateHistoryDate(ContentResolver cr, String uri, long date);
+        public void updateHistoryEntry(ContentResolver cr, String uri, String title,
+                                       long date, int visits);
 
         public Cursor getAllVisitedHistory(ContentResolver cr);
 
         public Cursor getRecentHistory(ContentResolver cr, int limit);
 
+        public int getMaxHistoryCount();
+
         public void clearHistory(ContentResolver cr);
 
         public Cursor getAllBookmarks(ContentResolver cr);
@@ -107,8 +111,9 @@ public class BrowserDB {
         sDb.updateHistoryTitle(cr, uri, title);
     }
 
-    public static void updateHistoryDate(ContentResolver cr, String uri, long date) {
+    public static void updateHistoryEntry(ContentResolver cr, String uri, String title,
-        sDb.updateHistoryDate(cr, uri, date);
+                                          long date, int visits) {
+        sDb.updateHistoryEntry(cr, uri, title, date, visits);
     }
 
     public static Cursor getAllVisitedHistory(ContentResolver cr) {
@@ -119,6 +124,10 @@ public class BrowserDB {
         return sDb.getRecentHistory(cr, limit);
     }
 
+    public static int getMaxHistoryCount() {
+        return sDb.getMaxHistoryCount();
+    }
+
     public static void clearHistory(ContentResolver cr) {
         sDb.clearHistory(cr);
     }

mobile/android/base/db/LocalBrowserDB.java

@@ -200,9 +200,31 @@ public class LocalBrowserDB implements BrowserDB.BrowserDBIface {
                   new String[] { uri });
     }
 
-    public void updateHistoryDate(ContentResolver cr, String uri, long date) {
+    public void updateHistoryEntry(ContentResolver cr, String uri, String title,
+                                   long date, int visits) {
+        int oldVisits = 0;
+        Cursor cursor = null;
+        try {
+            cursor = cr.query(appendProfile(History.CONTENT_URI),
+                              new String[] { History.VISITS },
+                              History.URL + " = ?",
+                              new String[] { uri },
+                              null);
+
+            if (cursor.moveToFirst()) {
+                oldVisits = cursor.getInt(0);
+            }
+        } finally {
+            if (cursor != null)
+                cursor.close();
+        }
+
         ContentValues values = new ContentValues();
         values.put(History.DATE_LAST_VISITED, date);
+        values.put(History.VISITS, oldVisits + visits);
+        if (title != null) {
+            values.put(History.TITLE, title);
+        }
 
         cr.update(appendProfile(History.CONTENT_URI),
                   values,
@@ -226,7 +248,8 @@ public class LocalBrowserDB implements BrowserDB.BrowserDBIface {
                                            History.URL,
                                            History.TITLE,
                                            History.FAVICON,
-                                           History.DATE_LAST_VISITED },
+                                           History.DATE_LAST_VISITED,
+                                           History.VISITS },
                             History.DATE_LAST_VISITED + " > 0",
                             null,
                             History.DATE_LAST_VISITED + " DESC");
@@ -234,6 +257,10 @@ public class LocalBrowserDB implements BrowserDB.BrowserDBIface {
         return new LocalDBCursor(c);
     }
 
+    public int getMaxHistoryCount() {
+        return MAX_HISTORY_COUNT;
+    }
+
     public void clearHistory(ContentResolver cr) {
         cr.delete(appendProfile(History.CONTENT_URI), null, null);
     }
@@ -401,6 +428,8 @@ public class LocalBrowserDB implements BrowserDB.BrowserDBIface {
                 columnName = ImageColumns.THUMBNAIL;
             } else if (columnName.equals(BrowserDB.URLColumns.DATE_LAST_VISITED)) {
                 columnName = History.DATE_LAST_VISITED;
+            } else if (columnName.equals(BrowserDB.URLColumns.VISITS)) {
+                columnName = History.VISITS;
             }
 
             return columnName;