Bug 394038 ? make url bar autocomplete frecency algorithm global (r=sspitzer/mconnor)

browser/app/profile/firefox.js

@@ -599,6 +599,42 @@ pref("browser.places.createdSmartBookmarks", false);
 // XXX to be removed after beta 2 (bug 391419)
 pref("browser.places.migratePostDataAnnotations", true);
 
+// the (maximum) number of the recent visits to sample
+// when calculating frecency
+pref("places.frecency.numVisits", 10);
+
+// Perform frecency recalculation after this amount of idle, repeating.
+// A value of zero disables updating of frecency on idle.
+// Default is 1 minute (60000ms).
+pref("places.frecency.updateIdleTime", 60000);
+
+// buckets (in days) for frecency calculation
+pref("places.frecency.firstBucketCutoff", 4);
+pref("places.frecency.secondBucketCutoff", 14);
+pref("places.frecency.thirdBucketCutoff", 31);
+pref("places.frecency.fourthBucketCutoff", 90);
+
+// weights for buckets for frecency calculations
+pref("places.frecency.firstBucketWeight", 100);
+pref("places.frecency.secondBucketWeight", 70);
+pref("places.frecency.thirdBucketWeight", 50);
+pref("places.frecency.fourthBucketWeight", 30);
+pref("places.frecency.defaultBucketWeight", 10);
+
+// bonus (in percent) for visit transition types for frecency calculations
+pref("places.frecency.embedVisitBonus", 0);
+pref("places.frecency.linkVisitBonus", 120);
+pref("places.frecency.typedVisitBonus", 200);
+pref("places.frecency.bookmarkVisitBonus", 140);
+pref("places.frecency.downloadVisitBonus", 0);
+pref("places.frecency.permRedirectVisitBonus", 0);
+pref("places.frecency.tempRedirectVisitBonus", 0);
+pref("places.frecency.defaultVisitBonus", 0);
+
+// bonus (in percent) for place types for frecency calculations
+pref("places.frecency.unvisitedBookmarkBonus", 140);
+pref("places.frecency.unvisitedTypedBonus", 200);
+
 // Controls behavior of the "Add Exception" dialog launched from SSL error pages
 // 0 - don't pre-populate anything
 // 1 - pre-populate site URL, but don't fetch certificate

toolkit/components/places/public/nsILivemarkService.idl

@@ -161,3 +161,7 @@ interface nsILivemarkService : nsISupports
 	 */
   void reloadLivemarkFolder(in long long folderID);
 };
+
+%{C++
+#define LMANNO_FEEDURI "livemark/feedURI"
+%}

toolkit/components/places/src/nsAnnotationService.cpp

@@ -1691,7 +1691,7 @@ nsAnnotationService::CopyItemAnnotations(PRInt64 aSourceItemId,
                                          PRInt64 aDestItemId,
                                          PRBool aOverwriteDest)
 {
- // XXX: Implment Me!
+ // XXX: Implement Me!
  return NS_ERROR_NOT_IMPLEMENTED;
 }
 

toolkit/components/places/src/nsNavBookmarks.cpp

@@ -50,6 +50,7 @@
 #include "nsAutoLock.h"
 #include "nsIUUIDGenerator.h"
 #include "prprf.h"
+#include "nsILivemarkService.h"
 
 const PRInt32 nsNavBookmarks::kFindBookmarksIndex_ID = 0;
 const PRInt32 nsNavBookmarks::kFindBookmarksIndex_Type = 1;
@@ -796,8 +797,6 @@ nsNavBookmarks::AdjustIndices(PRInt64 aFolder,
 {
   NS_ASSERTION(aStartIndex <= aEndIndex, "start index must be <= end index");
 
-  mozIStorageConnection *dbConn = DBConn();
-
   nsCAutoString buffer;
   buffer.AssignLiteral("UPDATE moz_bookmarks SET position = position + ");
   buffer.AppendInt(aDelta);
@@ -910,6 +909,43 @@ nsNavBookmarks::InsertBookmark(PRInt64 aFolder, nsIURI *aItem, PRInt32 aIndex,
   NS_ENSURE_SUCCESS(rv, rv);
   *aNewBookmarkId = rowId;
 
+  // XXX
+  // 0n import / fx 2 migration, is the frecency work going to slow us down?
+  // We might want to skip this stuff, as well as the frecency work
+  // caused by GetUrlIdFor() which calls InternalAddNewPage().
+  // If we do skip this, after import, we will
+  // need to call FixInvalidFrecenciesForExcludedPlaces().
+  // We might need to call it anyways, if items aren't properly annotated
+  // as livemarks feeds yet.
+
+  nsCAutoString url;
+  rv = aItem->GetSpec(url);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  // prevent place: queries from showing up in the URL bar autocomplete results
+  PRBool isBookmark = !IsQueryURI(url);
+
+  if (isBookmark) {
+    // if it is a livemark item (the parent is a livemark), 
+    // we pass in false for isBookmark.  otherwise, unvisited livemark 
+    // items will appear in URL autocomplete before we visit them.
+    PRBool parentIsLivemark;
+    nsCOMPtr<nsILivemarkService> lms = 
+      do_GetService(NS_LIVEMARKSERVICE_CONTRACTID, &rv);
+    NS_ENSURE_SUCCESS(rv, rv);
+
+    rv = lms->IsLivemark(aFolder, &parentIsLivemark);
+    NS_ENSURE_SUCCESS(rv, rv);
+ 
+    isBookmark = !parentIsLivemark;
+  }
+  
+  // when we created the moz_place entry for the new bookmark 
+  // (a side effect of calling GetUrlIdFor()) frecency -1;
+  // now we re-calculate the frecency for this moz_place entry. 
+  rv = History()->UpdateFrecency(childID, isBookmark);
+  NS_ENSURE_SUCCESS(rv, rv);
+
   rv = SetItemLastModified(aFolder, PR_Now());
   NS_ENSURE_SUCCESS(rv, rv);
 
@@ -1004,6 +1040,17 @@ nsNavBookmarks::RemoveItem(PRInt64 aItemId)
   rv = UpdateBookmarkHashOnRemove(placeId);
   NS_ENSURE_SUCCESS(rv, rv);
 
+  // XXX is this too expensive when updating livemarks?
+  // UpdateBookmarkHashOnRemove() does a sanity check using
+  // IsBookmarkedInDatabase(),  so it might not have actually 
+  // removed the bookmark.  should we have a boolean out param
+  // for if we actually removed it, and use that to decide if we call
+  // UpdateFrecency() and the rest of this code?
+  if (itemType == TYPE_BOOKMARK) {
+    rv = History()->UpdateFrecency(placeId, PR_FALSE /* isBookmark */);
+    NS_ENSURE_SUCCESS(rv, rv);
+  }
+
   ENUMERATE_WEAKARRAY(mObservers, nsINavBookmarkObserver,
                       OnItemRemoved(aItemId, folderId, childIndex))
 
@@ -2144,6 +2191,19 @@ nsNavBookmarks::ChangeBookmarkURI(PRInt64 aBookmarkId, nsIURI *aNewURI)
   rv = transaction.Commit();
   NS_ENSURE_SUCCESS(rv, rv);
 
+  // upon changing the uri for a bookmark, update the frecency for the new place
+  // no need to check if this is a livemark, because...
+  rv = History()->UpdateFrecency(placeId, PR_TRUE /* isBookmark */);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+#if 0
+  // upon changing the uri for a bookmark, update the frecency for the old place
+  // XXX todo, we need to get the oldPlaceId (fk) before changing it above
+  // and then here, we need to determine if that oldPlaceId is still a bookmark (and not a livemark)
+  rv = History()->UpdateFrecency(oldPlaceId,  PR_FALSE /* isBookmark */);
+  NS_ENSURE_SUCCESS(rv, rv);
+#endif
+
   nsCAutoString spec;
   rv = aNewURI->GetSpec(spec);
   NS_ENSURE_SUCCESS(rv, rv);

toolkit/components/places/src/nsNavHistory.h

@@ -185,6 +185,12 @@ public:
   nsresult GetUrlIdFor(nsIURI* aURI, PRInt64* aEntryID,
                        PRBool aAutoCreate);
 
+  nsresult CalculateVisitCount(PRInt64 aPlaceId, PRBool aForFrecency, PRInt32 *aVisitCount);
+
+  nsresult UpdateFrecency(PRInt64 aPageID, PRBool isBookmark);
+
+  nsresult FixInvalidFrecenciesForExcludedPlaces();
+
   /**
    * Returns a pointer to the storage connection used by history. This
    * connection object is also used by the annotation service and bookmarks, so
@@ -390,14 +396,28 @@ protected:
   nsCOMPtr<mozIStorageStatement> mDBUrlToUrlResult; // kGetInfoIndex_* results
   nsCOMPtr<mozIStorageStatement> mDBBookmarkToUrlResult; // kGetInfoIndex_* results
 
+  nsresult RecalculateFrecencies();
+  nsresult RecalculateFrecenciesInternal(mozIStorageStatement *aStatement, PRInt64 aBindParameter);
+
+  nsresult CalculateFrecency(PRInt64 aPageID, PRInt32 aTyped, PRInt32 aVisitCount, nsCAutoString &aURL, PRInt32 *aFrecency);
+  nsresult CalculateFrecencyInternal(PRInt64 aPageID, PRInt32 aTyped, PRInt32 aVisitCount, PRBool aIsBookmarked, PRInt32 *aFrecency);
+  nsCOMPtr<mozIStorageStatement> mDBVisitsForFrecency;
+  nsCOMPtr<mozIStorageStatement> mDBInvalidFrecencies;
+  nsCOMPtr<mozIStorageStatement> mDBOldFrecencies;
+  nsCOMPtr<mozIStorageStatement> mDBUpdateFrecencyAndHidden;
+  nsCOMPtr<mozIStorageStatement> mDBGetPlaceVisitStats;
+  nsCOMPtr<mozIStorageStatement> mDBGetBookmarkParentsForPlace;
+  nsCOMPtr<mozIStorageStatement> mDBVisitCountForFrecency;
+  nsCOMPtr<mozIStorageStatement> mDBTrueVisitCount;
+
   nsresult InitDBFile(PRBool aForceInit);
   nsresult BackupDBFile();
-  nsresult InitDB(PRBool *aDoImport);
+  nsresult InitDB(PRInt16 *aMadeChanges);
   nsresult InitStatements();
   nsresult ForceMigrateBookmarksDB(mozIStorageConnection *aDBConn);
   nsresult MigrateV3Up(mozIStorageConnection *aDBConn);
   nsresult MigrateV6Up(mozIStorageConnection *aDBConn);
-  nsresult EnsureCurrentSchema(mozIStorageConnection* aDBConn);
+  nsresult EnsureCurrentSchema(mozIStorageConnection* aDBConn, PRBool *aMadeChanges);
   nsresult CleanUpOnQuit();
 
 #ifdef IN_MEMORY_LINKS
@@ -418,14 +438,15 @@ protected:
                          PRInt64* aSessionID, PRInt64* aRedirectBookmark);
   nsresult InternalAddNewPage(nsIURI* aURI, const nsAString& aTitle,
                               PRBool aHidden, PRBool aTyped,
-                              PRInt32 aVisitCount, PRInt64* aPageID);
+                              PRInt32 aVisitCount, PRBool aCalculateFrecency,
+                              PRInt64* aPageID);
   nsresult InternalAddVisit(PRInt64 aPageID, PRInt64 aReferringVisit,
                             PRInt64 aSessionID, PRTime aTime,
                             PRInt32 aTransitionType, PRInt64* aVisitID);
   PRBool FindLastVisit(nsIURI* aURI, PRInt64* aVisitID,
                        PRInt64* aSessionID);
   PRBool IsURIStringVisited(const nsACString& url);
-  nsresult LoadPrefs();
+  nsresult LoadPrefs(PRBool aInitializing);
 
   // Current time optimization
   PRTime mLastNow;
@@ -587,6 +608,7 @@ protected:
   static const PRInt32 kAutoCompleteIndex_FaviconURL;
   static const PRInt32 kAutoCompleteIndex_ItemId;
   static const PRInt32 kAutoCompleteIndex_ParentId;
+  static const PRInt32 kAutoCompleteIndex_BookmarkTitle;
   nsCOMPtr<mozIStorageStatement> mDBAutoCompleteQuery; //  kAutoCompleteIndex_* results
   nsCOMPtr<mozIStorageStatement> mDBTagAutoCompleteQuery; //  kAutoCompleteIndex_* results
 
@@ -596,20 +618,21 @@ protected:
   nsCOMPtr<nsITimer> mAutoCompleteTimer;
 
   nsString mCurrentSearchString;
+  nsString mCurrentSearchStringEscaped;
+
 #ifdef MOZ_XUL
   nsCOMPtr<nsIAutoCompleteObserver> mCurrentListener;
   nsCOMPtr<nsIAutoCompleteSimpleResult> mCurrentResult;
 #endif
+
   nsDataHashtable<nsStringHashKey, PRBool> mCurrentResultURLs;
-  PRTime mCurrentChunkEndTime;
+  PRInt32 mCurrentChunkOffset;
-  PRTime mCurrentOldestVisit;
-  PRBool mFirstChunk;
 
   nsDataHashtable<nsTrimInt64HashKey, PRBool> mLivemarkFeedItemIds;
   nsDataHashtable<nsStringHashKey, PRBool> mLivemarkFeedURIs;
 
   nsresult AutoCompleteTypedSearch();
-  nsresult AutoCompleteFullHistorySearch();
+  nsresult AutoCompleteFullHistorySearch(PRBool* aHasMoreResults);
   nsresult AutoCompleteTagsSearch();
 
   nsresult PerformAutoComplete();
@@ -621,12 +644,36 @@ protected:
   PRInt32 mExpireDaysMax;
   PRInt32 mExpireSites;
 
+  // frecency prefs
+  PRInt32 mNumVisitsForFrecency;
+  PRInt32 mFrecencyUpdateIdleTime;
+  PRInt32 mFirstBucketCutoffInDays;
+  PRInt32 mSecondBucketCutoffInDays;
+  PRInt32 mThirdBucketCutoffInDays;
+  PRInt32 mFourthBucketCutoffInDays;
+  PRInt32 mFirstBucketWeight;
+  PRInt32 mSecondBucketWeight;
+  PRInt32 mThirdBucketWeight;
+  PRInt32 mFourthBucketWeight;
+  PRInt32 mDefaultWeight;
+  PRInt32 mEmbedVisitBonus;
+  PRInt32 mLinkVisitBonus;
+  PRInt32 mTypedVisitBonus;
+  PRInt32 mBookmarkVisitBonus;
+  PRInt32 mDownloadVisitBonus;
+  PRInt32 mPermRedirectVisitBonus;
+  PRInt32 mTempRedirectVisitBonus;
+  PRInt32 mDefaultVisitBonus;
+  PRInt32 mUnvisitedBookmarkBonus;
+  PRInt32 mUnvisitedTypedBonus;
+
   // in nsNavHistoryQuery.cpp
   nsresult TokensToQueries(const nsTArray<QueryKeyValuePair>& aTokens,
                            nsCOMArray<nsNavHistoryQuery>* aQueries,
                            nsNavHistoryQueryOptions* aOptions);
 
   nsCOMPtr<nsITimer> mIdleTimer;
+  nsresult InitializeIdleTimer();
   static void IdleTimerCallback(nsITimer* aTimer, void* aClosure);
   nsresult OnIdle();
 

toolkit/components/places/src/nsNavHistoryAutoComplete.cpp

@@ -23,6 +23,8 @@
  *   Brett Wilson <brettw@gmail.com>
  *   Joe Hewitt <hewitt@netscape.com>
  *   Blake Ross <blaker@netscape.com>
+ *   Seth Spitzer <sspitzer@mozilla.org>
+ *   Dietrich Ayala <dietrich@mozilla.com>
  *
  * Alternatively, the contents of this file may be used under the terms of
  * either the GNU General Public License Version 2 or later (the "GPL"), or
@@ -42,11 +44,18 @@
 /**
  * Autocomplete algorithm:
  *
- * The current algorithm searches history from now backwards to the oldest
+ * Searches moz_places by frecency (in descending order)
- * visit in chunks of time (AUTOCOMPLETE_SEARCH_CHUNK).  We currently
+ * in chunks (AUTOCOMPLETE_SEARCH_CHUNK_SIZE).  We currently
- * do SQL LIKE searches of the search term in the url and title
+ * do SQL LIKE searches of the search term in the place title, place url
- * within in each chunk of time.  the results are ordered by visit_count 
+ * and bookmark titles (since a "place" can have multiple bookmarks)
- * (and then visit_date), giving us poor man's "frecency".
+ * within in each chunk. The results are ordered by frecency.
+ * Note, we exclude places with no frecency (0) because 
+ * frecency = 0 means "don't show this in autocomplete". place: queries should
+ * have that, as should unvisited children of livemark feeds (that aren't
+ * bookmarked elsewhere).
+ *
+ * But places with frecency (-1) are included, as that means that these items
+ * have not had their frecency calculated yet (will happen on idle).
  */
 
 #include "nsNavHistory.h"
@@ -63,12 +72,14 @@
 #include "nsUnicharUtils.h"
 #include "nsNavBookmarks.h"
 #include "nsPrintfCString.h"
+#include "nsILivemarkService.h"
 
 #define NS_AUTOCOMPLETESIMPLERESULT_CONTRACTID \
   "@mozilla.org/autocomplete/simple-result;1"
 
 // This is the maximum results we'll return for a "typed" search
 // This happens in response to clicking the down arrow next to the URL.
+// XXX todo, check if doing rich autocomplete, and if so, limit to the max results?
 #define AUTOCOMPLETE_MAX_PER_TYPED 100
 
 // nsNavHistory::InitAutoComplete
@@ -100,33 +111,48 @@ nsresult
 nsNavHistory::CreateAutoCompleteQueries()
 {
   nsCString sql = NS_LITERAL_CSTRING(
-    "SELECT h.url, h.title, f.url, b.id, b.parent "
+    "SELECT h.url, h.title, f.url, b.id, b.parent, b.title "
     "FROM moz_places h "
-    "JOIN moz_historyvisits v ON h.id = v.place_id "
     "LEFT OUTER JOIN moz_bookmarks b ON b.fk = h.id "
     "LEFT OUTER JOIN moz_favicons f ON h.favicon_id = f.id "
-    "WHERE v.visit_date >= ?1 AND v.visit_date <= ?2 AND h.hidden <> 1 AND "
+    "WHERE h.frecency <> 0 AND ");
-    " v.visit_type NOT IN(0,4) AND ");
 
   if (mAutoCompleteOnlyTyped)
     sql += NS_LITERAL_CSTRING("h.typed = 1 AND ");
 
+  // NOTE:
+  // after migration or clear all private data, we might end up with
+  // a lot of places with frecency = -1 (until idle)
+  //
+  // XXX bug 412736
+  // in the case of a frecency tie, break it with h.typed and h.visit_count
+  // which is better than nothing.  but this is slow, so not doing it yet.
   sql += NS_LITERAL_CSTRING(
-    "(h.title LIKE ?3 ESCAPE '/' OR h.url LIKE ?3 ESCAPE '/') "
+    "(b.title LIKE ?1 ESCAPE '/' OR " 
-    "GROUP BY h.id ORDER BY h.typed DESC, h.visit_count DESC, MAX(v.visit_date) DESC;");
+     "h.title LIKE ?1 ESCAPE '/' OR "
+     "h.url LIKE ?1 ESCAPE '/') "
+    "ORDER BY h.frecency DESC LIMIT ?2 OFFSET ?3");
 
-  nsresult rv = mDBConn->CreateStatement(sql, getter_AddRefs(mDBAutoCompleteQuery));
+  nsresult rv = mDBConn->CreateStatement(sql, 
+    getter_AddRefs(mDBAutoCompleteQuery));
   NS_ENSURE_SUCCESS(rv, rv);
 
+  // NOTE:
+  // after migration or clear all private data, we might end up with
+  // a lot of places with frecency = -1 (until idle)
+  // 
+  // XXX bug 412736
+  // in the case of a frecency tie, break it with h.typed and h.visit_count
+  // which is better than nothing.  but this is slow, so not doing it yet.
   sql = NS_LITERAL_CSTRING(
     "SELECT h.url, h.title, f.url, b.id, b.parent "
     "FROM moz_places h "
     "JOIN moz_bookmarks b ON b.fk = h.id "
-    "LEFT OUTER JOIN moz_historyvisits v ON h.id = v.place_id "
     "LEFT OUTER JOIN moz_favicons f ON h.favicon_id = f.id "
-    "WHERE "
+    "WHERE h.frecency <> 0 AND "
     "(b.parent in (SELECT t.id FROM moz_bookmarks t WHERE t.parent = ?1 AND LOWER(t.title) = LOWER(?2))) "
-    "GROUP BY h.id ORDER BY h.visit_count DESC, MAX(v.visit_date) DESC;");
+    "ORDER BY h.frecency DESC");
+
   rv = mDBConn->CreateStatement(sql, getter_AddRefs(mDBTagAutoCompleteQuery));
   NS_ENSURE_SUCCESS(rv, rv);
 
@@ -152,25 +178,16 @@ nsNavHistory::StartAutoCompleteTimer(PRUint32 aMilliseconds)
   return NS_OK;
 }
 
-static const PRInt64 USECS_PER_DAY = LL_INIT(20, 500654080);
+// number of places to search per chunk
-
+// too big, and the UI will be unresponsive
-// search in day chunks.  too big, and the UI will be unresponsive
 // as we will be off searching the database.
-// too short, and because of AUTOCOMPLETE_SEARCH_TIMEOUT
+// too small, and because of AUTOCOMPLETE_SEARCH_TIMEOUT
 // results won't come back in fast enough to feel snappy.
-// because we sort within chunks by visit_count (then visit_date)
+#define AUTOCOMPLETE_SEARCH_CHUNK_SIZE 100
-// choose 4 days so that Friday's searches are in the first chunk
-// and those will affect Monday's results
-#define AUTOCOMPLETE_SEARCH_CHUNK (USECS_PER_DAY * 4)
 
 // wait this many milliseconds between searches
-// too short, and the UI will be unresponsive
-// as we will be off searching the database.
-// too big, and results won't come back in fast enough to feel snappy.
 #define AUTOCOMPLETE_SEARCH_TIMEOUT 100
 
-#define LMANNO_FEEDURI "livemark/feedURI"
-
 // nsNavHistory::AutoCompleteTimerCallback
 
 void // static
@@ -196,15 +213,14 @@ nsNavHistory::PerformAutoComplete()
     rv = AutoCompleteTypedSearch();
   else {
     // only search tags on the first chunk,
-    // but before we search history, as we want tagged
+    // but before we search places, as we want tagged
     // items to show up first.
-    if (mFirstChunk) {
+    if (!mCurrentChunkOffset) {
       rv = AutoCompleteTagsSearch();
       NS_ENSURE_SUCCESS(rv, rv);
     }
 
-    rv = AutoCompleteFullHistorySearch();
+    rv = AutoCompleteFullHistorySearch(&moreChunksToSearch);
-    moreChunksToSearch = (mCurrentChunkEndTime >= mCurrentOldestVisit);
   }
   NS_ENSURE_SUCCESS(rv, rv);
  
@@ -229,11 +245,10 @@ nsNavHistory::PerformAutoComplete()
 
   mCurrentListener->OnSearchResult(this, mCurrentResult);
  
-  // if we're not done searching, adjust our end time and 
+  // if we're not done searching, adjust our current offset
-  // search the next earlier chunk of time
+  // and search the next chunk
   if (moreChunksToSearch) {
-    mFirstChunk = PR_FALSE;
+    mCurrentChunkOffset += AUTOCOMPLETE_SEARCH_CHUNK_SIZE;
-    mCurrentChunkEndTime -= AUTOCOMPLETE_SEARCH_CHUNK;
     rv = StartAutoCompleteTimer(AUTOCOMPLETE_SEARCH_TIMEOUT);
     NS_ENSURE_SUCCESS(rv, rv);
   } else {
@@ -262,15 +277,20 @@ nsNavHistory::StartSearch(const nsAString & aSearchString,
   mCurrentSearchString = aSearchString;
   // remove whitespace, see bug #392141 for details
   mCurrentSearchString.Trim(" \r\n\t\b");
+
+  nsresult rv = mDBAutoCompleteQuery->EscapeStringForLIKE(mCurrentSearchString, PRUnichar('/'), mCurrentSearchStringEscaped);
+  NS_ENSURE_SUCCESS(rv, rv);
+
   mCurrentListener = aListener;
-  nsresult rv;
 
   // determine if we can start by searching through the previous search results.
-  // if we can't, we need to reset mCurrentChunkEndTime and mCurrentOldestVisit.
+  // if we can't, we need to reset mCurrentChunkOffset
   // if we can, we will search through our previous search results and then resume 
-  // searching using the previous mCurrentChunkEndTime and mCurrentOldestVisit values.
+  // searching using the previous mCurrentOldestVisit and mLast values.
   PRBool searchPrevious = PR_FALSE;
-  if (aPreviousResult) {
+  // XXX Re-use of previous search results is disabled due to bug 412730.
+  //if (aPreviousResult) {
+  if (0) {
     nsAutoString prevSearchString;
     aPreviousResult->GetSearchString(prevSearchString);
 
@@ -386,30 +406,8 @@ nsNavHistory::StartSearch(const nsAString & aSearchString,
     }
   }
   else if (!mCurrentSearchString.IsEmpty()) {
-    // reset to mCurrentChunkEndTime 
+    // reset mCurrentChunkOffset 
-    mCurrentChunkEndTime = PR_Now();
+    mCurrentChunkOffset = 0;
-    mCurrentOldestVisit = 0;
-    mFirstChunk = PR_TRUE;
-
-    // determine our earliest visit
-    nsCOMPtr<mozIStorageStatement> dbSelectStatement;
-    rv = mDBConn->CreateStatement(
-      NS_LITERAL_CSTRING("SELECT MIN(visit_date) id FROM moz_historyvisits WHERE visit_type NOT IN(0,4)"),
-      getter_AddRefs(dbSelectStatement));
-    NS_ENSURE_SUCCESS(rv, rv);
-    PRBool hasMinVisit;
-    rv = dbSelectStatement->ExecuteStep(&hasMinVisit);
-    NS_ENSURE_SUCCESS(rv, rv);
-  
-    if (hasMinVisit) {
-      rv = dbSelectStatement->GetInt64(0, &mCurrentOldestVisit);
-      NS_ENSURE_SUCCESS(rv, rv);
-    }
-
-    if (!mCurrentOldestVisit) {
-      // if we have no visits, use a reasonable value
-      mCurrentOldestVisit = PR_Now() - USECS_PER_DAY;
-    }
   }
 
   // fire right away, we already waited to start searching
@@ -435,32 +433,43 @@ nsNavHistory::StopSearch()
 // nsNavHistory::AutoCompleteTypedSearch
 //
 //    Called when there is no search string. This happens when you press
-//    down arrow from the URL bar: the most recent things you typed are listed.
+//    down arrow from the URL bar: the most "frecent" things you typed are listed.
 //
-//    Ordering here is simpler because there are no boosts for typing, and there
-//    is no URL information to use. The ordering just comes out of the DB by
-//    visit count (primary) and time since last visited (secondary).
 
 nsresult nsNavHistory::AutoCompleteTypedSearch()
 {
   nsCOMPtr<mozIStorageStatement> dbSelectStatement;
 
+  // NOTE:
+  // after migration or clear all private data, we might end up with
+  // a lot of places with frecency = -1 (until idle)
+  //
+  // XXX bug 412736
+  // in the case of a frecency tie, break it with h.typed and h.visit_count
+  // which is better than nothing.  but this is slow, so not doing it yet.
+  //
+  // GROUP BY h.id to prevent duplicates.  For mDBAutoCompleteQuery,
+  // we use the mCurrentResultURLs hash to accomplish this.
+  //
+  // NOTE:  because we are grouping by h.id, b.id and b.parent 
+  // get collapsed, so if something is both a livemark and a bookmark
+  // we might not show it as a "star" if the parentId we return is 
+  // the one for the livemark item, and not the bookmark item.
+  // XXX bug 412734
   nsCString sql = NS_LITERAL_CSTRING(
     "SELECT h.url, h.title, f.url, b.id, b.parent "
     "FROM moz_places h "
     "LEFT OUTER JOIN moz_bookmarks b ON b.fk = h.id "
     "LEFT OUTER JOIN moz_favicons f ON h.favicon_id = f.id "
-    "JOIN moz_historyvisits v ON h.id = v.place_id WHERE (h.id IN "
+    "WHERE h.frecency <> 0 AND h.typed = 1 "
-    "(SELECT DISTINCT h.id from moz_historyvisits v, moz_places h WHERE "
+    "GROUP BY h.id ORDER BY h.frecency DESC LIMIT ");
-    "v.place_id = h.id AND h.typed = 1 AND v.visit_type NOT IN(0,4) "
-    "ORDER BY v.visit_date DESC LIMIT ");
   sql.AppendInt(AUTOCOMPLETE_MAX_PER_TYPED);
-  sql += NS_LITERAL_CSTRING(")) GROUP BY h.id ORDER BY MAX(v.visit_date) DESC");  
   
   nsFaviconService* faviconService = nsFaviconService::GetFaviconService();
   NS_ENSURE_TRUE(faviconService, NS_ERROR_OUT_OF_MEMORY);
 
-  nsresult rv = mDBConn->CreateStatement(sql, getter_AddRefs(dbSelectStatement));
+  nsresult rv = mDBConn->CreateStatement(sql, 
+    getter_AddRefs(dbSelectStatement));
   NS_ENSURE_SUCCESS(rv, rv);
  
   PRBool hasMore = PR_FALSE;
@@ -472,12 +481,13 @@ nsresult nsNavHistory::AutoCompleteTypedSearch()
     PRInt64 itemId = 0;
     dbSelectStatement->GetInt64(kAutoCompleteIndex_ItemId, &itemId);
     PRInt64 parentId = 0;
-    dbSelectStatement->GetInt64(kAutoCompleteIndex_ParentId, &parentId);
+    if (itemId)
+      dbSelectStatement->GetInt64(kAutoCompleteIndex_ParentId, &parentId);
 
     PRBool dummy;
     // don't show rss feed items as bookmarked,
     // but do show rss feed URIs as bookmarked.
-    PRBool isBookmark = (itemId != 0 && 
+    PRBool isBookmark = (itemId && 
                          !mLivemarkFeedItemIds.Get(parentId, &dummy)) ||
                         mLivemarkFeedURIs.Get(entryURL, &dummy);   
 
@@ -543,9 +553,8 @@ nsNavHistory::AutoCompleteTagsSearch()
       "SELECT h.url, h.title, f.url, b.id, b.parent "
       "FROM moz_places h "
       "JOIN moz_bookmarks b ON b.fk = h.id "
-      "LEFT OUTER JOIN moz_historyvisits v ON h.id = v.place_id "
       "LEFT OUTER JOIN moz_favicons f ON h.favicon_id = f.id "
-      "WHERE "
+      "WHERE h.frecency <> 0 AND "
       "(b.parent in "
       " (SELECT t.id FROM moz_bookmarks t WHERE t.parent = ?1 AND (");
 
@@ -562,7 +571,7 @@ nsNavHistory::AutoCompleteTagsSearch()
     }
 
     tagQuery += NS_LITERAL_CSTRING("))) "
-      "GROUP BY h.id ORDER BY h.visit_count DESC, MAX(v.visit_date) DESC;");
+      "GROUP BY h.id ORDER BY h.frecency DESC");
 
     rv = mDBConn->CreateStatement(tagQuery, getter_AddRefs(tagAutoCompleteQuery));
     NS_ENSURE_SUCCESS(rv, rv);
@@ -586,24 +595,34 @@ nsNavHistory::AutoCompleteTagsSearch()
 
   // Determine the result of the search
   while (NS_SUCCEEDED(tagAutoCompleteQuery->ExecuteStep(&hasMore)) && hasMore) {
-    nsAutoString entryURL, entryTitle, entryFavicon;
+    nsAutoString entryURL;
     rv = tagAutoCompleteQuery->GetString(kAutoCompleteIndex_URL, entryURL);
     NS_ENSURE_SUCCESS(rv, rv);
-    rv = tagAutoCompleteQuery->GetString(kAutoCompleteIndex_Title, entryTitle);
-    NS_ENSURE_SUCCESS(rv, rv);
-    rv = tagAutoCompleteQuery->GetString(kAutoCompleteIndex_FaviconURL, entryFavicon);
-    NS_ENSURE_SUCCESS(rv, rv);
-    PRInt64 itemId = 0;
-    rv = tagAutoCompleteQuery->GetInt64(kAutoCompleteIndex_ItemId, &itemId);
-    NS_ENSURE_SUCCESS(rv, rv);
-    PRInt64 parentId = 0;
-    rv = tagAutoCompleteQuery->GetInt64(kAutoCompleteIndex_ParentId, &parentId);
-    NS_ENSURE_SUCCESS(rv, rv);
 
     PRBool dummy;
     // prevent duplicates.  this can happen when chunking as we
-    // may have already seen this URL from an earlier chunk of time
+    // may have already seen this URL.
+    // NOTE:  because we use mCurrentResultURLs to remove duplicates,
+    // the first url wins.
+    // so we might not show it as a "star" if the parentId we get first is
+    // the one for the livemark item, and not the bookmark item,
+    // we may not show the "star" even though we should.
+    // XXX bug 412734
     if (!mCurrentResultURLs.Get(entryURL, &dummy)) {
+      nsAutoString entryTitle, entryFavicon;
+      rv = tagAutoCompleteQuery->GetString(kAutoCompleteIndex_Title, entryTitle);
+      NS_ENSURE_SUCCESS(rv, rv);
+      rv = tagAutoCompleteQuery->GetString(kAutoCompleteIndex_FaviconURL, entryFavicon);
+      NS_ENSURE_SUCCESS(rv, rv);
+      PRInt64 itemId = 0;
+      rv = tagAutoCompleteQuery->GetInt64(kAutoCompleteIndex_ItemId, &itemId);
+      NS_ENSURE_SUCCESS(rv, rv);
+      PRInt64 parentId = 0;
+      if (itemId) {
+        rv = tagAutoCompleteQuery->GetInt64(kAutoCompleteIndex_ParentId, &parentId);
+        NS_ENSURE_SUCCESS(rv, rv);
+      }
+
       // new item, append to our results and put it in our hash table.
       nsCAutoString faviconSpec;
       faviconService->GetFaviconSpecForIconString(
@@ -621,28 +640,26 @@ nsNavHistory::AutoCompleteTagsSearch()
 
 // nsNavHistory::AutoCompleteFullHistorySearch
 //
-// Search history for visits that have a url or title that contains mCurrentSearchString
+// Search for places that have a title, url, 
-// and are within our current chunk of time:
+// or bookmark title(s) that contains mCurrentSearchString
-// between (mCurrentChunkEndTime - AUTOCOMPLETE_SEARCH_CHUNK) and (mCurrentChunkEndTime)
+// and are within our current chunk of "frecency".
+//
+// @param aHasMoreResults is false if the query found no matching items
 //
 
 nsresult
-nsNavHistory::AutoCompleteFullHistorySearch()
+nsNavHistory::AutoCompleteFullHistorySearch(PRBool* aHasMoreResults)
 {
   mozStorageStatementScoper scope(mDBAutoCompleteQuery);
 
-  nsresult rv = mDBAutoCompleteQuery->BindInt64Parameter(0, mCurrentChunkEndTime - AUTOCOMPLETE_SEARCH_CHUNK);
-  NS_ENSURE_SUCCESS(rv, rv);
-
-  rv = mDBAutoCompleteQuery->BindInt64Parameter(1, mCurrentChunkEndTime);
-  NS_ENSURE_SUCCESS(rv, rv);
-
-  nsString escapedSearchString;
-  rv = mDBAutoCompleteQuery->EscapeStringForLIKE(mCurrentSearchString, PRUnichar('/'), escapedSearchString);
-  NS_ENSURE_SUCCESS(rv, rv);
-
   // prepend and append with % for "contains"
-  rv = mDBAutoCompleteQuery->BindStringParameter(2, NS_LITERAL_STRING("%") + escapedSearchString + NS_LITERAL_STRING("%"));
+  nsresult rv = mDBAutoCompleteQuery->BindStringParameter(0, NS_LITERAL_STRING("%") + mCurrentSearchStringEscaped + NS_LITERAL_STRING("%"));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  rv = mDBAutoCompleteQuery->BindInt32Parameter(1, AUTOCOMPLETE_SEARCH_CHUNK_SIZE);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  rv = mDBAutoCompleteQuery->BindInt32Parameter(2, mCurrentChunkOffset);
   NS_ENSURE_SUCCESS(rv, rv);
 
   nsFaviconService* faviconService = nsFaviconService::GetFaviconService();
@@ -652,36 +669,62 @@ nsNavHistory::AutoCompleteFullHistorySearch()
 
   // Determine the result of the search
   while (NS_SUCCEEDED(mDBAutoCompleteQuery->ExecuteStep(&hasMore)) && hasMore) {
-    nsAutoString entryURL, entryTitle, entryFavicon;
+    *aHasMoreResults = PR_TRUE;
+    nsAutoString entryURL;
     rv = mDBAutoCompleteQuery->GetString(kAutoCompleteIndex_URL, entryURL);
     NS_ENSURE_SUCCESS(rv, rv);
-    rv = mDBAutoCompleteQuery->GetString(kAutoCompleteIndex_Title, entryTitle);
-    NS_ENSURE_SUCCESS(rv, rv);
-    rv = mDBAutoCompleteQuery->GetString(kAutoCompleteIndex_FaviconURL, entryFavicon);
-    NS_ENSURE_SUCCESS(rv, rv);
-    PRInt64 itemId = 0;
-    rv = mDBAutoCompleteQuery->GetInt64(kAutoCompleteIndex_ItemId, &itemId);
-    NS_ENSURE_SUCCESS(rv, rv);
-    PRInt64 parentId = 0;
-    rv = mDBAutoCompleteQuery->GetInt64(kAutoCompleteIndex_ParentId, &parentId);
-    NS_ENSURE_SUCCESS(rv, rv);
-
-    PRBool dummy;
-    // don't show rss feed items as bookmarked,
-    // but do show rss feed URIs as bookmarked.
-    PRBool isBookmark = (itemId != 0 && 
-                         !mLivemarkFeedItemIds.Get(parentId, &dummy)) ||
-                         mLivemarkFeedURIs.Get(entryURL, &dummy);  
 
     // prevent duplicates.  this can happen when chunking as we
-    // may have already seen this URL from an earlier chunk of time
+    // may have already seen this URL from our tag search or an earlier
+    // chunk.
+    PRBool dummy;
     if (!mCurrentResultURLs.Get(entryURL, &dummy)) {
+      nsAutoString entryTitle, entryFavicon, entryBookmarkTitle;
+      rv = mDBAutoCompleteQuery->GetString(kAutoCompleteIndex_Title, entryTitle);
+      NS_ENSURE_SUCCESS(rv, rv);
+      rv = mDBAutoCompleteQuery->GetString(kAutoCompleteIndex_FaviconURL, entryFavicon);
+      NS_ENSURE_SUCCESS(rv, rv);
+      PRInt64 itemId = 0;
+      rv = mDBAutoCompleteQuery->GetInt64(kAutoCompleteIndex_ItemId, &itemId);
+      NS_ENSURE_SUCCESS(rv, rv);
+
+      PRInt64 parentId = 0;
+      // only bother to fetch parent id and bookmark title 
+      // if we have a bookmark (itemId != 0)
+      if (itemId) {
+        rv = mDBAutoCompleteQuery->GetInt64(kAutoCompleteIndex_ParentId, &parentId);
+        NS_ENSURE_SUCCESS(rv, rv);
+        rv = mDBAutoCompleteQuery->GetString(kAutoCompleteIndex_BookmarkTitle, entryBookmarkTitle);
+        NS_ENSURE_SUCCESS(rv, rv);
+      }
+
+      // don't show rss feed items as bookmarked,
+      // but do show rss feed URIs as bookmarked.
+      //
+      // NOTE:  because we use mCurrentResultURLs to remove duplicates,
+      // the first url wins.
+      // so we might not show it as a "star" if the parentId we get first is
+      // the one for the livemark item, and not the bookmark item,
+      // we may not show the "star" even though we should.
+      // XXX bug 412734
+      PRBool isBookmark = (itemId && 
+                           !mLivemarkFeedItemIds.Get(parentId, &dummy)) ||
+                           mLivemarkFeedURIs.Get(entryURL, &dummy);  
+
       // new item, append to our results and put it in our hash table.
       nsCAutoString faviconSpec;
       faviconService->GetFaviconSpecForIconString(
         NS_ConvertUTF16toUTF8(entryFavicon), faviconSpec);
-      rv = mCurrentResult->AppendMatch(entryURL, entryTitle, 
+
-        NS_ConvertUTF8toUTF16(faviconSpec), isBookmark ? NS_LITERAL_STRING("bookmark") : NS_LITERAL_STRING("favicon"));
+      // if the search string is in the bookmark title, show that in the
+      // result (instead of the page title)
+      PRBool matchInBookmarkTitle = itemId && 
+        CaseInsensitiveFindInReadable(mCurrentSearchString, entryBookmarkTitle);
+
+      rv = mCurrentResult->AppendMatch(entryURL, 
+        matchInBookmarkTitle ? entryBookmarkTitle : entryTitle, 
+        NS_ConvertUTF8toUTF16(faviconSpec), 
+        isBookmark ? NS_LITERAL_STRING("bookmark") : NS_LITERAL_STRING("favicon"));
       NS_ENSURE_SUCCESS(rv, rv);
 
       mCurrentResultURLs.Put(entryURL, PR_TRUE);

toolkit/components/places/src/nsNavHistoryExpire.cpp

@@ -259,6 +259,26 @@ nsNavHistoryExpire::ClearHistory()
   if (NS_FAILED(rv))
     NS_WARNING("ExpireAnnotationsParanoid failed.");
 
+  // for all remaining places, reset the frecency
+  // Note, we don't reset the visit_count, as we use that in our "on idle"
+  // query to figure out which places to recalcuate frecency first.
+  rv = connection->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+    "UPDATE moz_places SET frecency = -1"));
+  if (NS_FAILED(rv))
+    NS_WARNING("failed to recent frecency");
+
+  // some of the remaining places could be place: urls or
+  // unvisited livemark items, so setting the frecency to -1
+  // will cause them to show up in the url bar autocomplete
+  // call FixInvalidFrecenciesForExcludedPlaces() to handle that scenario
+  rv = mHistory->FixInvalidFrecenciesForExcludedPlaces();
+  if (NS_FAILED(rv))
+    NS_WARNING("failed to fix invalid frecencies");
+
+  // XXX todo
+  // forcibly call the "on idle" timer here to do a little work
+  // but the rest will happen on idle.
+
   ENUMERATE_WEAKARRAY(mHistory->mObservers, nsINavHistoryObserver,
                       OnClearHistory())
 
@@ -506,21 +526,54 @@ nsNavHistoryExpire::EraseVisits(mozIStorageConnection* aConnection,
     const nsTArray<nsNavHistoryExpireRecord>& aRecords)
 {
   // build a comma separated string of visit ids to delete
+  // also build a comma separated string of place ids to reset frecency and
+  // visit_count.
   nsCString deletedVisitIds;
+  nsCString placeIds;
+  nsTArray<PRInt64> deletedPlaceIdsArray, deletedVisitIdsArray;
   for (PRUint32 i = 0; i < aRecords.Length(); i ++) {
-    // Do not add comma separator for the first entry
+    // Do not add comma separator for the first visit id
-    if (! deletedVisitIds.IsEmpty())
+    if (deletedVisitIdsArray.IndexOf(aRecords[i].visitID) == -1) {
-      deletedVisitIds.AppendLiteral(",");  
+      if (!deletedVisitIds.IsEmpty())
-    deletedVisitIds.AppendInt(aRecords[i].visitID);
+        deletedVisitIds.AppendLiteral(",");  
+      deletedVisitIds.AppendInt(aRecords[i].visitID);
+    }
+
+    // Do not add comma separator for the first place id
+    if (deletedPlaceIdsArray.IndexOf(aRecords[i].placeID) == -1) {
+      if (!placeIds.IsEmpty())
+        placeIds.AppendLiteral(",");
+      placeIds.AppendInt(aRecords[i].placeID);
+    }
   }
 
   if (deletedVisitIds.IsEmpty())
     return NS_OK;
 
-  return aConnection->ExecuteSimpleSQL(
+  nsresult rv = aConnection->ExecuteSimpleSQL(
     NS_LITERAL_CSTRING("DELETE FROM moz_historyvisits WHERE id IN (") +
     deletedVisitIds +
     NS_LITERAL_CSTRING(")"));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  if (placeIds.IsEmpty())
+    return NS_OK;
+
+  // reset the frecencies for these places.
+  // Note, we don't reset the visit_count, as we use that in our "on idle"
+  // query to figure out which places to recalcuate frecency first.
+  rv = aConnection->ExecuteSimpleSQL(
+    NS_LITERAL_CSTRING("UPDATE moz_places SET "
+      "frecency = -1 WHERE id IN (") +
+    placeIds +
+    NS_LITERAL_CSTRING(")"));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  // XXX todo
+  // forcibly call the "on idle" timer here to do a little work
+  // but the rest will happen on idle.
+
+  return NS_OK;
 }
 
 
@@ -548,7 +601,7 @@ nsNavHistoryExpire::EraseHistory(mozIStorageConnection* aConnection,
     // avoid trying to delete the same place id twice
     if (deletedPlaceIdsArray.IndexOf(aRecords[i].placeID) == -1) {
       // Do not add comma separator for the first entry
-      if (! deletedPlaceIds.IsEmpty())
+      if (!deletedPlaceIds.IsEmpty())
         deletedPlaceIds.AppendLiteral(",");
       deletedPlaceIdsArray.AppendElement(aRecords[i].placeID);
       deletedPlaceIds.AppendInt(aRecords[i].placeID);
@@ -583,12 +636,12 @@ nsNavHistoryExpire::EraseFavicons(mozIStorageConnection* aConnection,
   nsTArray<PRInt64> deletedFaviconIdsArray;  
   for (PRUint32 i = 0; i < aRecords.Length(); i ++) {
     // IF main entry not expired OR no favicon DO NOT DELETE
-    if (! aRecords[i].erased || aRecords[i].faviconID == 0)
+    if (!aRecords[i].erased || aRecords[i].faviconID == 0)
       continue;
     // avoid trying to delete the same favicon id twice
     if (deletedFaviconIdsArray.IndexOf(aRecords[i].faviconID) == -1) {
       // Do not add comma separator for the first entry
-      if (! deletedFaviconIds.IsEmpty())
+      if (!deletedFaviconIds.IsEmpty())
         deletedFaviconIds.AppendLiteral(",");
       deletedFaviconIdsArray.AppendElement(aRecords[i].faviconID);
       deletedFaviconIds.AppendInt(aRecords[i].faviconID);
@@ -893,7 +946,7 @@ nsNavHistoryExpire::ComputeNextExpirationTime(
 
   PRBool hasMore;
   rv = statement->ExecuteStep(&hasMore);
-  if (NS_FAILED(rv) || ! hasMore)
+  if (NS_FAILED(rv) || !hasMore)
     return; // no items, we'll leave mNextExpirationTime = 0 and try to expire
             // again next time
 
@@ -907,7 +960,7 @@ nsNavHistoryExpire::ComputeNextExpirationTime(
 nsresult
 nsNavHistoryExpire::StartTimer(PRUint32 aMilleseconds)
 {
-  if (! mTimer)
+  if (!mTimer)
     mTimer = do_CreateInstance("@mozilla.org/timer;1");
   NS_ENSURE_STATE(mTimer); // returns on error
   nsresult rv = mTimer->InitWithFuncCallback(TimerCallback, this,