mozilla
/
gecko-dev
зеркало из https://github.com/mozilla/gecko-dev.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 1461753 - Add places.sqlite migration for calculating initial frecency stats. r=mak 

MozReview-Commit-ID: F7fZiVkLXxW

--HG--
extra : rebase_source : 34ba8dabd7305af07d6af7b9bf57f2820385cdd5
This commit is contained in:
Drew Willcoxon 2018-05-18 22:50:54 -07:00
Родитель 8e7e2dbdb4
Коммит 69302b1d31
16 изменённых файлов: 352 добавлений и 340 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

28
toolkit/components/places/Database.cpp
Просмотреть файл

@ -1296,7 +1296,12 @@ Database::InitSchema(bool* aDatabaseMigrated)
NS_ENSURE_SUCCESS(rv, rv);
}
// Firefox 62 uses schema version 48.
if (currentSchemaVersion < 49) {
rv = MigrateV49Up();
NS_ENSURE_SUCCESS(rv, rv);
}
// Firefox 62 uses schema version 49.
// Schema Upgrades must add migration code here.
// >>> IMPORTANT! <<<
@ -1582,7 +1587,7 @@ Database::InitFunctions()
NS_ENSURE_SUCCESS(rv, rv);
rv = IsFrecencyDecayingFunction::create(mMainConn);
NS_ENSURE_SUCCESS(rv, rv);
rv = UpdateFrecencyStatsFunction::create(mMainConn);
rv = SqrtFunction::create(mMainConn);
NS_ENSURE_SUCCESS(rv, rv);
return NS_OK;
@ -2472,6 +2477,25 @@ Database::MigrateV48Frecencies()
Unused << target->Dispatch(runnable, NS_DISPATCH_NORMAL);
}
nsresult
Database::MigrateV49Up() {
// Calculate initial frecency stats, which should have been done as part of
// the v48 migration but wasn't.
nsNavHistory *navHistory = nsNavHistory::GetHistoryService();
NS_ENSURE_STATE(navHistory);
nsresult rv = navHistory->RecalculateFrecencyStats(nullptr);
NS_ENSURE_SUCCESS(rv, rv);
// These hidden preferences were added along with the v48 migration as part of
// the frecency stats implementation but are now replaced with entries in the
// moz_meta table.
Unused << Preferences::ClearUser("places.frecency.stats.count");
Unused << Preferences::ClearUser("places.frecency.stats.sum");
Unused << Preferences::ClearUser("places.frecency.stats.sumOfSquares");
return NS_OK;
}
nsresult
Database::GetItemsWithAnno(const nsACString& aAnnoName, int32_t aItemType,
nsTArray<int64_t>& aItemIds)

3
toolkit/components/places/Database.h
Просмотреть файл

@ -19,7 +19,7 @@
// This is the schema version. Update it at any schema change and add a
// corresponding migrateVxx method below.
#define DATABASE_SCHEMA_VERSION 48
#define DATABASE_SCHEMA_VERSION 49
// Fired after Places inited.
#define TOPIC_PLACES_INIT_COMPLETE "places-init-complete"
@ -335,6 +335,7 @@ protected:
nsresult MigrateV46Up();
nsresult MigrateV47Up();
nsresult MigrateV48Up();
nsresult MigrateV49Up();
void MigrateV48Frecencies();

13
toolkit/components/places/PlacesDBUtils.jsm
Просмотреть файл

@ -40,6 +40,7 @@ var PlacesDBUtils = {
this.invalidateCaches,
this.checkCoherence,
this._refreshUI,
this.frecencyStats,
this.incrementalVacuum
];
let telemetryStartTime = Date.now();
@ -71,6 +72,7 @@ var PlacesDBUtils = {
this.invalidateCaches,
this.checkCoherence,
this.expire,
this.frecencyStats,
this.vacuum,
this.stats,
this._refreshUI,
@ -861,6 +863,17 @@ var PlacesDBUtils = {
return logs;
},
/**
* Recalculates statistical data on the frecencies in the database.
*
* @return {Promise} resolves when statistics are collected.
*/
frecencyStats() {
return new Promise(resolve => {
PlacesUtils.history.recalculateFrecencyStats(() => resolve());
});
},
/**
* Collects telemetry data and reports it to Telemetry.
*

29
toolkit/components/places/SQLFunctions.cpp
Просмотреть файл

@ -1391,50 +1391,43 @@ namespace places {
}
////////////////////////////////////////////////////////////////////////////////
//// Update frecency stats function
//// sqrt function
/* static */
nsresult
UpdateFrecencyStatsFunction::create(mozIStorageConnection *aDBConn)
SqrtFunction::create(mozIStorageConnection *aDBConn)
{
RefPtr<UpdateFrecencyStatsFunction> function =
new UpdateFrecencyStatsFunction();
RefPtr<SqrtFunction> function = new SqrtFunction();
nsresult rv = aDBConn->CreateFunction(
NS_LITERAL_CSTRING("update_frecency_stats"), 3, function
NS_LITERAL_CSTRING("sqrt"), 1, function
);
NS_ENSURE_SUCCESS(rv, rv);
return NS_OK;
}
NS_IMPL_ISUPPORTS(
UpdateFrecencyStatsFunction,
SqrtFunction,
mozIStorageFunction
)
NS_IMETHODIMP
UpdateFrecencyStatsFunction::OnFunctionCall(mozIStorageValueArray *aArgs,
nsIVariant **_result)
SqrtFunction::OnFunctionCall(mozIStorageValueArray *aArgs,
nsIVariant **_result)
{
MOZ_ASSERT(aArgs);
uint32_t numArgs;
nsresult rv = aArgs->GetNumEntries(&numArgs);
NS_ENSURE_SUCCESS(rv, rv);
MOZ_ASSERT(numArgs == 3);
MOZ_ASSERT(numArgs == 1);
int64_t placeId = aArgs->AsInt64(0);
int32_t oldFrecency = aArgs->AsInt32(1);
int32_t newFrecency = aArgs->AsInt32(2);
const nsNavHistory* navHistory = nsNavHistory::GetConstHistoryService();
NS_ENSURE_STATE(navHistory);
navHistory->DispatchFrecencyStatsUpdate(placeId, oldFrecency, newFrecency);
double value = aArgs->AsDouble(0);
RefPtr<nsVariant> result = new nsVariant();
rv = result->SetAsVoid();
rv = result->SetAsDouble(sqrt(value));
NS_ENSURE_SUCCESS(rv, rv);
result.forget(_result);
return NS_OK;
}

17
toolkit/components/places/SQLFunctions.h
Просмотреть файл

@ -567,20 +567,12 @@ private:
////////////////////////////////////////////////////////////////////////////////
//// Update frecency stats function
//// sqrt function
/**
* Calls nsNavHistory::UpdateFrecencyStats with the old and new frecencies of a
* particular moz_places row.
*
* @param placeID
* The moz_places row ID.
* @param oldFrecency
* The old frecency of a moz_places row that changed frecencies.
* @param newFrecency
* The new frecency of the same moz_places row.
* Gets the square root of a given value.
*/
class UpdateFrecencyStatsFunction final : public mozIStorageFunction
class SqrtFunction final : public mozIStorageFunction
{
public:
NS_DECL_THREADSAFE_ISUPPORTS
@ -594,10 +586,9 @@ public:
*/
static nsresult create(mozIStorageConnection *aDBConn);
private:
~UpdateFrecencyStatsFunction() {}
~SqrtFunction() {}
};
} // namespace places
} // namespace mozilla

52
toolkit/components/places/UnifiedComplete.js
Просмотреть файл

@ -248,8 +248,37 @@ const QUERYINDEX_ORIGIN_AUTOFILLED_VALUE = 1;
const QUERYINDEX_ORIGIN_URL = 2;
const QUERYINDEX_ORIGIN_FRECENCY = 3;
// `WITH` clause for the autofill queries. autofill_frecency_threshold.value is
// the frecency mean plus one standard deviation. This is inlined directly in
// the SQL (as opposed to being a custom Sqlite function for example) in order
// to be as efficient as possible. The MAX() is to make sure that places with
// <= 0 frecency are never autofilled.
const SQL_AUTOFILL_WITH = `
WITH
frecency_stats(count, sum, squares) AS (
SELECT
CAST((SELECT IFNULL(value, 0.0) FROM moz_meta WHERE key = "frecency_count") AS REAL),
CAST((SELECT IFNULL(value, 0.0) FROM moz_meta WHERE key = "frecency_sum") AS REAL),
CAST((SELECT IFNULL(value, 0.0) FROM moz_meta WHERE key = "frecency_sum_of_squares") AS REAL)
),
autofill_frecency_threshold(value) AS (
SELECT MAX(1,
CASE count
WHEN 0 THEN 0.0
WHEN 1 THEN sum
ELSE (sum / count) + sqrt((squares - ((sum * sum) / count)) / count)
END
) FROM frecency_stats
)
`;
const SQL_AUTOFILL_FRECENCY_THRESHOLD = `(
SELECT value FROM autofill_frecency_threshold
)`;
function originQuery(conditions = "", bookmarkedFragment = "NULL") {
return `SELECT :query_type,
return `${SQL_AUTOFILL_WITH}
SELECT :query_type,
host || '/',
prefix || host || '/',
frecency,
@ -257,7 +286,7 @@ function originQuery(conditions = "", bookmarkedFragment = "NULL") {
id
FROM moz_origins
WHERE host BETWEEN :searchString AND :searchString || X'FFFF'
AND frecency >= :frecencyThreshold
AND frecency >= ${SQL_AUTOFILL_FRECENCY_THRESHOLD}
${conditions}
UNION ALL
SELECT :query_type,
@ -268,7 +297,7 @@ function originQuery(conditions = "", bookmarkedFragment = "NULL") {
id
FROM moz_origins
WHERE host BETWEEN 'www.' || :searchString AND 'www.' || :searchString || X'FFFF'
AND frecency >= :frecencyThreshold
AND frecency >= ${SQL_AUTOFILL_FRECENCY_THRESHOLD}
${conditions}
ORDER BY frecency DESC, id DESC
LIMIT 1 `;
@ -300,6 +329,7 @@ const QUERYINDEX_URL_FRECENCY = 3;
function urlQuery(conditions1, conditions2) {
return `/* do not warn (bug no): cannot use an index to sort */
${SQL_AUTOFILL_WITH}
SELECT :query_type,
url,
:strippedURL,
@ -308,7 +338,7 @@ function urlQuery(conditions1, conditions2) {
id
FROM moz_places
WHERE rev_host = :revHost
AND frecency >= :frecencyThreshold
AND frecency >= ${SQL_AUTOFILL_FRECENCY_THRESHOLD}
${conditions1}
UNION ALL
SELECT :query_type,
@ -319,7 +349,7 @@ function urlQuery(conditions1, conditions2) {
id
FROM moz_places
WHERE rev_host = :revHost || 'www.'
AND frecency >= :frecencyThreshold
AND frecency >= ${SQL_AUTOFILL_FRECENCY_THRESHOLD}
${conditions2}
ORDER BY frecency DESC, id DESC
LIMIT 1 `;
@ -2295,7 +2325,6 @@ Search.prototype = {
let opts = {
query_type: QUERYTYPE_AUTOFILL_ORIGIN,
searchString: searchStr.toLowerCase(),
frecencyThreshold: this._autofillFrecencyThreshold,
};
let bookmarked = this.hasBehavior("bookmark") &&
@ -2350,7 +2379,6 @@ Search.prototype = {
query_type: QUERYTYPE_AUTOFILL_URL,
revHost,
strippedURL,
frecencyThreshold: this._autofillFrecencyThreshold,
};
let bookmarked = this.hasBehavior("bookmark") &&
@ -2369,16 +2397,6 @@ Search.prototype = {
return [SQL_URL_QUERY, opts];
},
get _autofillFrecencyThreshold() {
// Places with 0 frecency (and below) shouldn't be autofilled, so use 1 as a
// lower bound.
return Math.max(
1,
PlacesUtils.history.frecencyMean +
PlacesUtils.history.frecencyStandardDeviation
);
},
// The result is notified to the search listener on a timer, to chunk multiple
// match updates together and avoid rebuilding the popup at every new match.
_notifyTimer: null,

18
toolkit/components/places/nsINavHistoryService.idl
Просмотреть файл

@ -12,9 +12,10 @@
#include "nsISupports.idl"
interface nsIArray;
interface nsIURI;
interface nsIVariant;
interface nsIFile;
interface nsIObserver;
interface nsIVariant;
interface nsIURI;
interface mozIStorageConnection;
interface mozIStorageStatementCallback;
@ -1390,13 +1391,14 @@ interface nsINavHistoryService : nsISupports
unsigned long long hashURL(in ACString aSpec, [optional] in ACString aMode);
/**
* The mean and standard deviation of all frecencies > 0 in the database.
* These are used to determine a frecency threshold for URLs that should be
* autofilled in the urlbar. They are updated whenever a frecency changes,
* but not when frecencies decay.
* Resets and recalculates the frecency statistics that are kept in the
* moz_meta table.
*
* @param aCallback
* Called when the recalculation is complete. The arguments passed to
* the observer are not defined.
*/
readonly attribute double frecencyMean;
readonly attribute double frecencyStandardDeviation;
void recalculateFrecencyStats([optional] in nsIObserver aCallback);
/**
* The database connection used by Places.

188
toolkit/components/places/nsNavHistory.cpp
Просмотреть файл

@ -109,10 +109,7 @@ using namespace mozilla::places;
// This is a 'hidden' pref for the purposes of unit tests.
#define PREF_FREC_DECAY_RATE "places.frecency.decayRate"
#define PREF_FREC_STATS_COUNT "places.frecency.stats.count"
#define PREF_FREC_STATS_SUM "places.frecency.stats.sum"
#define PREF_FREC_STATS_SQUARES "places.frecency.stats.sumOfSquares"
#define PREF_FREC_DECAY_RATE_DEF 0.975f
// In order to avoid calling PR_now() too often we use a cached "now" value
// for repeating stuff. These are milliseconds between "now" cache refreshes.
@ -269,30 +266,6 @@ const int32_t nsNavHistory::kGetInfoIndex_VisitType = 17;
// nsNavBookmarks::kGetChildrenIndex_Type = 20;
// nsNavBookmarks::kGetChildrenIndex_PlaceID = 21;
static uint64_t
GetUInt64Pref(const char *prefName)
{
// `Preferences` doesn't support uint64_t, so we store it as a string instead.
nsAutoCString strVal;
nsresult rv = Preferences::GetCString(prefName, strVal);
if (NS_SUCCEEDED(rv)) {
int64_t val = strVal.ToInteger64(&rv);
if (NS_SUCCEEDED(rv)) {
return static_cast<uint64_t>(val);
}
}
return 0U;
}
static void
SetUInt64Pref(const char *prefName,
uint64_t val)
{
nsAutoCString strVal;
strVal.AppendInt(val);
Unused << Preferences::SetCString(prefName, strVal);
}
PLACES_FACTORY_SINGLETON_IMPLEMENTATION(nsNavHistory, gHistoryService)
@ -357,10 +330,6 @@ nsNavHistory::Init()
mDB = Database::GetDatabase();
NS_ENSURE_STATE(mDB);
mFrecencyStatsCount = GetUInt64Pref(PREF_FREC_STATS_COUNT);
mFrecencyStatsSum = GetUInt64Pref(PREF_FREC_STATS_SUM);
mFrecencyStatsSumOfSquares = GetUInt64Pref(PREF_FREC_STATS_SQUARES);
/*****************************************************************************
*** IMPORTANT NOTICE!
***
@ -628,118 +597,65 @@ nsNavHistory::DispatchFrecencyChangedNotification(const nsACString& aSpec,
);
}
void
nsNavHistory::DispatchFrecencyStatsUpdate(int64_t aPlaceId,
int32_t aOldFrecency,
int32_t aNewFrecency) const
NS_IMETHODIMP
nsNavHistory::RecalculateFrecencyStats(nsIObserver *aCallback)
{
MOZ_ASSERT(aPlaceId >= 0);
Unused << NS_DispatchToMainThread(
NewRunnableMethod<int64_t, int32_t, int32_t>(
"nsNavHistory::UpdateFrecencyStats",
const_cast<nsNavHistory*>(this),
&nsNavHistory::UpdateFrecencyStats,
aPlaceId, aOldFrecency, aNewFrecency
RefPtr<nsNavHistory> self(this);
nsMainThreadPtrHandle<nsIObserver> callback(
!aCallback ? nullptr :
new nsMainThreadPtrHolder<nsIObserver>(
"nsNavHistory::RecalculateFrecencyStats callback",
aCallback
)
);
}
void
nsNavHistory::UpdateFrecencyStats(int64_t aPlaceId,
int32_t aOldFrecency,
int32_t aNewFrecency)
{
MOZ_ASSERT(NS_IsMainThread(), "Must be called on the main thread");
MOZ_ASSERT(aPlaceId >= 0);
nsCOMPtr<mozIStorageConnection> conn = mDB->MainConn();
nsCOMPtr<nsIEventTarget> target = do_GetInterface(conn);
MOZ_ASSERT(target);
nsresult rv = target->Dispatch(NS_NewRunnableFunction(
"nsNavHistory::RecalculateFrecencyStats",
[self, callback] {
Unused << self->RecalculateFrecencyStatsInternal();
Unused << NS_DispatchToMainThread(NS_NewRunnableFunction(
"nsNavHistory::RecalculateFrecencyStats callback",
[callback] {
if (callback) {
Unused << callback->Observe(nullptr, "", nullptr);
}
}
));
}
));
NS_ENSURE_SUCCESS(rv, rv);
if (aOldFrecency > 0) {
MOZ_ASSERT(mFrecencyStatsCount > 0);
mFrecencyStatsCount--;
uint64_t uOld = static_cast<uint64_t>(aOldFrecency);
MOZ_ASSERT(mFrecencyStatsSum >= uOld);
mFrecencyStatsSum -= uOld;
uint64_t square = uOld * uOld;
MOZ_ASSERT(mFrecencyStatsSumOfSquares >= square);
mFrecencyStatsSumOfSquares -= square;
}
if (aNewFrecency > 0) {
mFrecencyStatsCount++;
uint64_t uNew = static_cast<uint64_t>(aNewFrecency);
mFrecencyStatsSum += uNew;
mFrecencyStatsSumOfSquares += uNew * uNew;
}
// This method can be called many times very quickly when many frecencies
// change at once. (Note though that it is *not* called when frecencies
// decay.) To avoid hammering preferences, update them only every so often.
// There's actually a browser mochitest that makes sure preferences aren't
// accessed too much, and it fails without throttling like this.
if (!mUpdateFrecencyStatsPrefsTimer) {
Unused << NS_NewTimerWithFuncCallback(
getter_AddRefs(mUpdateFrecencyStatsPrefsTimer),
&UpdateFrecencyStatsPrefs,
this,
5000, // ms
nsITimer::TYPE_ONE_SHOT,
"nsNavHistory::UpdateFrecencyStatsPrefs",
nullptr
);
}
}
void // static
nsNavHistory::UpdateFrecencyStatsPrefs(nsITimer *aTimer,
void *aClosure)
{
nsNavHistory *history = static_cast<nsNavHistory *>(aClosure);
if (!history) {
return;
}
SetUInt64Pref(PREF_FREC_STATS_COUNT, history->mFrecencyStatsCount);
SetUInt64Pref(PREF_FREC_STATS_SUM, history->mFrecencyStatsSum);
SetUInt64Pref(PREF_FREC_STATS_SQUARES, history->mFrecencyStatsSumOfSquares);
history->mUpdateFrecencyStatsPrefsTimer = nullptr;
// This is only so that tests can know when the prefs are written. Observing
// nsPref:changed isn't sufficient because that's not fired when a pref value
// is the same as the previous value.
nsCOMPtr<nsIObserverService> obs = mozilla::services::GetObserverService();
if (obs) {
MOZ_ALWAYS_SUCCEEDS(obs->NotifyObservers(
nullptr,
"places-frecency-stats-prefs-updated",
nullptr
));
}
}
NS_IMETHODIMP
nsNavHistory::GetFrecencyMean(double *_retval)
{
NS_ENSURE_ARG_POINTER(_retval);
if (mFrecencyStatsCount == 0) {
*_retval = 0.0;
return NS_OK;
}
*_retval =
static_cast<double>(mFrecencyStatsSum) /
static_cast<double>(mFrecencyStatsCount);
return NS_OK;
}
NS_IMETHODIMP
nsNavHistory::GetFrecencyStandardDeviation(double *_retval)
nsresult
nsNavHistory::RecalculateFrecencyStatsInternal()
{
NS_ENSURE_ARG_POINTER(_retval);
if (mFrecencyStatsCount <= 1) {
*_retval = 0.0;
return NS_OK;
}
double squares = static_cast<double>(mFrecencyStatsSumOfSquares);
double sum = static_cast<double>(mFrecencyStatsSum);
double count = static_cast<double>(mFrecencyStatsCount);
*_retval = sqrt((squares - ((sum * sum) / count)) / count);
nsCOMPtr<mozIStorageConnection> conn(mDB->MainConn());
NS_ENSURE_STATE(conn);
nsresult rv = conn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
"INSERT OR REPLACE INTO moz_meta (key, value) " \
"SELECT '" MOZ_META_KEY_FRECENCY_COUNT "' AS key, COUNT(*) AS value " \
"FROM moz_places " \
"WHERE id >= 0 AND frecency > 0 " \
"UNION "\
"SELECT '" MOZ_META_KEY_FRECENCY_SUM "' AS key, IFNULL(SUM(frecency), 0) AS value " \
"FROM moz_places " \
"WHERE id >= 0 AND frecency > 0 " \
"UNION " \
"SELECT '" MOZ_META_KEY_FRECENCY_SUM_OF_SQUARES "' AS key, IFNULL(SUM(frecency_squared), 0) AS value " \
"FROM ( " \
"SELECT frecency * frecency AS frecency_squared " \
"FROM moz_places " \
"WHERE id >= 0 AND frecency > 0 " \
"); "
));
NS_ENSURE_SUCCESS(rv, rv);
return NS_OK;
}
@ -2605,7 +2521,7 @@ nsNavHistory::DecayFrecency()
nsresult rv = FixInvalidFrecencies();
NS_ENSURE_SUCCESS(rv, rv);
float decayRate = Preferences::GetFloat(PREF_FREC_DECAY_RATE, FRECENCY_DECAY_RATE);
float decayRate = Preferences::GetFloat(PREF_FREC_DECAY_RATE, PREF_FREC_DECAY_RATE_DEF);
// Globally decay places frecency rankings to estimate reduced frecency
// values of pages that haven't been visited for a while, i.e., they do

36
toolkit/components/places/nsNavHistory.h
Просмотреть файл

@ -470,35 +470,6 @@ public:
*/
bool IsFrecencyDecaying() const;
/**
* Updates frecencyMean and frecencyStandardDeviation given a change in
* frecency of a particular moz_places row.
*
* @param aPlaceId
* The moz_places row ID.
* @param aOldFrecency
* The old value of the frecency.
* @param aNewFrecency
* The new value of the frecency.
*/
void UpdateFrecencyStats(int64_t aPlaceId,
int32_t aOldFrecency,
int32_t aNewFrecency);
/**
* Dispatches a runnable to the main thread that calls UpdateFrecencyStats.
*
* @param aPlaceId
* The moz_places row ID.
* @param aOldFrecency
* The old value of the frecency.
* @param aNewFrecency
* The new value of the frecency.
*/
void DispatchFrecencyStatsUpdate(int64_t aPlaceId,
int32_t aOldFrecency,
int32_t aNewFrecency) const;
/**
* Store last insterted id for a table.
*/
@ -660,12 +631,7 @@ protected:
void DecayFrecencyCompleted(uint16_t reason);
uint32_t mDecayFrecencyPendingCount;
uint64_t mFrecencyStatsCount;
uint64_t mFrecencyStatsSum;
uint64_t mFrecencyStatsSumOfSquares;
nsCOMPtr<nsITimer> mUpdateFrecencyStatsPrefsTimer;
static void UpdateFrecencyStatsPrefs(nsITimer *aTimer,
void *aClosure);
nsresult RecalculateFrecencyStatsInternal();
// in nsNavHistoryQuery.cpp
nsresult TokensToQuery(const nsTArray<QueryKeyValuePair>& aTokens,

4
toolkit/components/places/nsPlacesTables.h
Просмотреть файл

@ -234,5 +234,9 @@
") WITHOUT ROWID " \
)
// Keys in the moz_meta table.
#define MOZ_META_KEY_FRECENCY_COUNT "frecency_count"
#define MOZ_META_KEY_FRECENCY_SUM "frecency_sum"
#define MOZ_META_KEY_FRECENCY_SUM_OF_SQUARES "frecency_sum_of_squares"
#endif // __nsPlacesTables_h__

59
toolkit/components/places/nsPlacesTriggers.h
Просмотреть файл

@ -69,6 +69,49 @@
"END" \
)
// This fragment updates frecency stats after a moz_places row is deleted.
#define UPDATE_FRECENCY_STATS_AFTER_DELETE \
"INSERT OR REPLACE INTO moz_meta(key, value) VALUES " \
"( " \
"'" MOZ_META_KEY_FRECENCY_COUNT "', " \
"CAST((SELECT IFNULL(value, 0) FROM moz_meta WHERE key = '" MOZ_META_KEY_FRECENCY_COUNT "') AS INTEGER) " \
"- (CASE WHEN OLD.frecency <= 0 OR OLD.id < 0 THEN 0 ELSE 1 END) " \
"), " \
"( " \
"'" MOZ_META_KEY_FRECENCY_SUM "', " \
"CAST((SELECT IFNULL(value, 0) FROM moz_meta WHERE key = '" MOZ_META_KEY_FRECENCY_SUM "') AS INTEGER) " \
"- (CASE WHEN OLD.frecency <= 0 OR OLD.id < 0 THEN 0 ELSE OLD.frecency END) " \
"), " \
"( " \
"'" MOZ_META_KEY_FRECENCY_SUM_OF_SQUARES "', " \
"CAST((SELECT IFNULL(value, 0) FROM moz_meta WHERE key = '" MOZ_META_KEY_FRECENCY_SUM_OF_SQUARES "') AS INTEGER) " \
"- (CASE WHEN OLD.frecency <= 0 OR OLD.id < 0 THEN 0 ELSE OLD.frecency * OLD.frecency END) " \
"); "
// This fragment updates frecency stats after frecency changes in a moz_places
// row. It's the same as UPDATE_FRECENCY_STATS_AFTER_DELETE except it accounts
// for NEW values.
#define UPDATE_FRECENCY_STATS_AFTER_UPDATE \
"INSERT OR REPLACE INTO moz_meta(key, value) VALUES " \
"( " \
"'" MOZ_META_KEY_FRECENCY_COUNT "', " \
"CAST(IFNULL((SELECT value FROM moz_meta WHERE key = '" MOZ_META_KEY_FRECENCY_COUNT "'), 0) AS INTEGER) " \
"- (CASE WHEN OLD.frecency <= 0 OR OLD.id < 0 THEN 0 ELSE 1 END) " \
"+ (CASE WHEN NEW.frecency <= 0 OR NEW.id < 0 THEN 0 ELSE 1 END) " \
"), " \
"( " \
"'" MOZ_META_KEY_FRECENCY_SUM "', " \
"CAST(IFNULL((SELECT value FROM moz_meta WHERE key = '" MOZ_META_KEY_FRECENCY_SUM "'), 0) AS INTEGER) " \
"- (CASE WHEN OLD.frecency <= 0 OR OLD.id < 0 THEN 0 ELSE OLD.frecency END) " \
"+ (CASE WHEN NEW.frecency <= 0 OR NEW.id < 0 THEN 0 ELSE NEW.frecency END) " \
"), " \
"( " \
"'" MOZ_META_KEY_FRECENCY_SUM_OF_SQUARES "', " \
"CAST(IFNULL((SELECT value FROM moz_meta WHERE key = '" MOZ_META_KEY_FRECENCY_SUM_OF_SQUARES "'), 0) AS INTEGER) " \
"- (CASE WHEN OLD.frecency <= 0 OR OLD.id < 0 THEN 0 ELSE OLD.frecency * OLD.frecency END) " \
"+ (CASE WHEN NEW.frecency <= 0 OR NEW.id < 0 THEN 0 ELSE NEW.frecency * NEW.frecency END) " \
"); "
// See CREATE_PLACES_AFTERINSERT_TRIGGER. For each delete in moz_places we
// add the origin to moz_updateoriginsdelete_temp - we then delete everything
// from moz_updateoriginsdelete_temp, allowing us to run a trigger only once
@ -79,8 +122,7 @@
"BEGIN " \
"INSERT OR IGNORE INTO moz_updateoriginsdelete_temp (origin_id, host) " \
"VALUES (OLD.origin_id, get_host_and_port(OLD.url)); " \
"SELECT update_frecency_stats(OLD.id, OLD.frecency, -1) " \
"WHERE OLD.id >= 0; " \
UPDATE_FRECENCY_STATS_AFTER_DELETE \
"END" \
)
@ -123,9 +165,6 @@
"END" \
)
#define FRECENCY_DECAY_RATE 0.975f
#define FRECENCY_DECAY_RATE_STR "0.975"
// This trigger keeps frecencies in the moz_origins table in sync with
// frecencies in moz_places. However, we skip this when frecency changes are
// due to frecency decay since (1) decay updates all frecencies at once, so this
@ -135,12 +174,7 @@
#define CREATE_PLACES_AFTERUPDATE_FRECENCY_TRIGGER NS_LITERAL_CSTRING( \
"CREATE TEMP TRIGGER moz_places_afterupdate_frecency_trigger " \
"AFTER UPDATE OF frecency ON moz_places FOR EACH ROW " \
"WHEN NEW.frecency >= 0 AND NOT ( " \
"OLD.frecency > 0 " \
"AND is_frecency_decaying() " \
"AND NEW.frecency < OLD.frecency " \
"AND (OLD.frecency - NEW.frecency) / OLD.frecency <= " FRECENCY_DECAY_RATE_STR \
") " \
"WHEN NEW.frecency >= 0 AND NOT is_frecency_decaying() " \
"BEGIN " \
"UPDATE moz_origins " \
"SET frecency = ( " \
@ -149,8 +183,7 @@
"WHERE moz_places.origin_id = moz_origins.id " \
") " \
"WHERE id = NEW.origin_id; " \
"SELECT update_frecency_stats(NEW.id, OLD.frecency, NEW.frecency) " \
"WHERE NEW.id >= 0; " \
UPDATE_FRECENCY_STATS_AFTER_UPDATE \
"END" \
)

2
toolkit/components/places/tests/head_common.js
Просмотреть файл

@ -3,7 +3,7 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
const CURRENT_SCHEMA_VERSION = 48;
const CURRENT_SCHEMA_VERSION = 49;
const FIRST_UPGRADABLE_SCHEMA_VERSION = 30;
const NS_APP_USER_PROFILE_50_DIR = "ProfD";

68
toolkit/components/places/tests/maintenance/test_preventive_maintenance.js
Просмотреть файл

@ -1688,6 +1688,74 @@ tests.push({
// ------------------------------------------------------------------------------
tests.push({
name: "T.1",
desc: "history.recalculateFrecencyStats() is called",
async setup() {
let urls = [
"http://example.com/1",
"http://example.com/2",
"http://example.com/3",
];
await PlacesTestUtils.addVisits(urls.map(u => ({ uri: u })));
this._frecencies = urls.map(u => frecencyForUrl(u));
let stats = await this._promiseStats();
Assert.equal(stats.count, this._frecencies.length, "Sanity check");
Assert.equal(stats.sum, this._sum(this._frecencies), "Sanity check");
Assert.equal(stats.squares, this._squares(this._frecencies), "Sanity check");
await PlacesUtils.withConnectionWrapper(
"T.1",
db => db.execute(`
INSERT OR REPLACE INTO moz_meta VALUES
('frecency_count', 99),
('frecency_sum', 99999),
('frecency_sum_of_squares', 99999 * 99999);
`)
);
stats = await this._promiseStats();
Assert.equal(stats.count, 99);
Assert.equal(stats.sum, 99999);
Assert.equal(stats.squares, 99999 * 99999);
},
async check() {
let stats = await this._promiseStats();
Assert.equal(stats.count, this._frecencies.length);
Assert.equal(stats.sum, this._sum(this._frecencies));
Assert.equal(stats.squares, this._squares(this._frecencies));
},
_sum(frecs) {
return frecs.reduce((memo, f) => memo + f, 0);
},
_squares(frecs) {
return frecs.reduce((memo, f) => memo + (f * f), 0);
},
async _promiseStats() {
let db = await PlacesUtils.promiseDBConnection();
let rows = await db.execute(`
SELECT
IFNULL((SELECT value FROM moz_meta WHERE key = "frecency_count"), 0),
IFNULL((SELECT value FROM moz_meta WHERE key = "frecency_sum"), 0),
IFNULL((SELECT value FROM moz_meta WHERE key = "frecency_sum_of_squares"), 0)
`);
return {
count: rows[0].getResultByIndex(0),
sum: rows[0].getResultByIndex(1),
squares: rows[0].getResultByIndex(2),
};
},
});
// ------------------------------------------------------------------------------
tests.push({
name: "Z",
desc: "Sanity: Preventive maintenance does not touch valid items",

4
toolkit/components/places/tests/maintenance/test_preventive_maintenance_checkAndFixDatabase.js
Просмотреть файл

@ -24,7 +24,7 @@ add_task(async function() {
failedTasks.push(val);
}
});
Assert.equal(numberOfTasksRun, 7, "Check that we have run all tasks.");
Assert.equal(successfulTasks.length, 7, "Check that we have run all tasks successfully");
Assert.equal(numberOfTasksRun, 8, "Check that we have run all tasks.");
Assert.equal(successfulTasks.length, 8, "Check that we have run all tasks successfully");
Assert.equal(failedTasks.length, 0, "Check that no task is failing");
});

42
toolkit/components/places/tests/migration/test_current_from_v47.js
Просмотреть файл

@ -7,20 +7,26 @@ add_task(async function setup() {
await setupPlacesDatabase("places_v43.sqlite");
});
// Accessing the database for the first time should trigger migration, and the
// schema version should be updated.
add_task(async function database_is_valid() {
// Accessing the database for the first time triggers migration.
Assert.equal(PlacesUtils.history.databaseStatus,
PlacesUtils.history.DATABASE_STATUS_UPGRADED);
let db = await PlacesUtils.promiseDBConnection();
Assert.equal((await db.getSchemaVersion()), CURRENT_SCHEMA_VERSION);
});
add_task(async function test_origins() {
// Now wait for moz_origins.frecency to be populated before continuing with
// other test tasks.
await TestUtils.waitForCondition(() => {
return !Services.prefs.getBoolPref("places.database.migrateV48Frecencies", false);
}, "Waiting for v48 origin frecencies to be migrated", 100, 3000);
});
// moz_origins should be populated.
add_task(async function test_origins() {
let db = await PlacesUtils.promiseDBConnection();
// Collect origins.
@ -83,3 +89,33 @@ add_task(async function test_origins() {
`);
Assert.equal(rows.length, 0);
});
// Frecency stats should have been collected.
add_task(async function test_frecency_stats() {
let db = await PlacesUtils.promiseDBConnection();
// Collect positive frecencies from moz_places.
let rows = await db.execute(`
SELECT frecency
FROM moz_places
WHERE id >= 0 AND frecency > 0;
`);
Assert.notEqual(rows.length, 0);
let frecencies = rows.map(r => r.getResultByName("frecency"));
// Collect stats.
rows = await db.execute(`
SELECT
(SELECT value FROM moz_meta WHERE key = "frecency_count"),
(SELECT value FROM moz_meta WHERE key = "frecency_sum"),
(SELECT value FROM moz_meta WHERE key = "frecency_sum_of_squares")
`);
let count = rows[0].getResultByIndex(0);
let sum = rows[0].getResultByIndex(1);
let squares = rows[0].getResultByIndex(2);
Assert.equal(count, frecencies.length);
Assert.equal(sum, frecencies.reduce((memo, f) => memo + f, 0));
Assert.equal(squares, frecencies.reduce((memo, f) => memo + (f * f), 0));
});

129
toolkit/components/places/tests/unit/test_frecency_stats.js
Просмотреть файл

@ -11,8 +11,7 @@ add_task(async function init() {
// Adds/removes some visits and bookmarks and makes sure the stats are updated.
add_task(async function basic() {
Assert.equal(PlacesUtils.history.frecencyMean, 0);
Assert.equal(PlacesUtils.history.frecencyStandardDeviation, 0);
await checkStats([]);
let frecenciesByURL = {};
let urls = [0, 1, 2].map(i => "http://example.com/" + i);
@ -21,45 +20,35 @@ add_task(async function basic() {
await PlacesTestUtils.addVisits([{ uri: urls[0] }]);
frecenciesByURL[urls[0]] = frecencyForUrl(urls[0]);
Assert.ok(frecenciesByURL[urls[0]] > 0, "Sanity check");
Assert.equal(PlacesUtils.history.frecencyMean,
mean(Object.values(frecenciesByURL)));
Assert.equal(PlacesUtils.history.frecencyStandardDeviation,
stddev(Object.values(frecenciesByURL)));
await checkStats(frecenciesByURL);
// Add a URL 1 visit.
await PlacesTestUtils.addVisits([{ uri: urls[1] }]);
frecenciesByURL[urls[1]] = frecencyForUrl(urls[1]);
Assert.ok(frecenciesByURL[urls[1]] > 0, "Sanity check");
Assert.equal(PlacesUtils.history.frecencyMean,
mean(Object.values(frecenciesByURL)));
Assert.equal(PlacesUtils.history.frecencyStandardDeviation,
stddev(Object.values(frecenciesByURL)));
await checkStats(frecenciesByURL);
// Add a URL 2 visit.
await PlacesTestUtils.addVisits([{ uri: urls[2] }]);
frecenciesByURL[urls[2]] = frecencyForUrl(urls[2]);
Assert.ok(frecenciesByURL[urls[2]] > 0, "Sanity check");
Assert.equal(PlacesUtils.history.frecencyMean,
mean(Object.values(frecenciesByURL)));
Assert.equal(PlacesUtils.history.frecencyStandardDeviation,
stddev(Object.values(frecenciesByURL)));
await checkStats(frecenciesByURL);
// Add another URL 2 visit.
await PlacesTestUtils.addVisits([{ uri: urls[2] }]);
frecenciesByURL[urls[2]] = frecencyForUrl(urls[2]);
Assert.ok(frecenciesByURL[urls[2]] > 0, "Sanity check");
Assert.equal(PlacesUtils.history.frecencyMean,
mean(Object.values(frecenciesByURL)));
Assert.equal(PlacesUtils.history.frecencyStandardDeviation,
stddev(Object.values(frecenciesByURL)));
await checkStats(frecenciesByURL);
// Remove URL 2's visits.
await PlacesUtils.history.remove([urls[2]]);
delete frecenciesByURL[urls[2]];
Assert.equal(PlacesUtils.history.frecencyMean,
mean(Object.values(frecenciesByURL)));
Assert.equal(PlacesUtils.history.frecencyStandardDeviation,
stddev(Object.values(frecenciesByURL)));
await checkStats(frecenciesByURL);
// Bookmark URL 1.
let parentGuid =
@ -73,19 +62,15 @@ add_task(async function basic() {
frecenciesByURL[urls[1]] = frecencyForUrl(urls[1]);
Assert.ok(frecenciesByURL[urls[1]] > 0, "Sanity check");
Assert.equal(PlacesUtils.history.frecencyMean,
mean(Object.values(frecenciesByURL)));
Assert.equal(PlacesUtils.history.frecencyStandardDeviation,
stddev(Object.values(frecenciesByURL)));
await checkStats(frecenciesByURL);
// Remove URL 1's visit.
await PlacesUtils.history.remove([urls[1]]);
frecenciesByURL[urls[1]] = frecencyForUrl(urls[1]);
Assert.ok(frecenciesByURL[urls[1]] > 0, "Sanity check");
Assert.equal(PlacesUtils.history.frecencyMean,
mean(Object.values(frecenciesByURL)));
Assert.equal(PlacesUtils.history.frecencyStandardDeviation,
stddev(Object.values(frecenciesByURL)));
await checkStats(frecenciesByURL);
// Remove URL 1's bookmark. Also need to call history.remove() again to
// remove the URL from moz_places. Otherwise it sticks around and keeps
@ -93,78 +78,40 @@ add_task(async function basic() {
await PlacesUtils.bookmarks.remove(bookmark);
await PlacesUtils.history.remove(urls[1]);
delete frecenciesByURL[urls[1]];
Assert.equal(PlacesUtils.history.frecencyMean,
mean(Object.values(frecenciesByURL)));
Assert.equal(PlacesUtils.history.frecencyStandardDeviation,
stddev(Object.values(frecenciesByURL)));
await checkStats(frecenciesByURL);
// Remove URL 0.
await PlacesUtils.history.remove([urls[0]]);
delete frecenciesByURL[urls[0]];
Assert.equal(PlacesUtils.history.frecencyMean, 0);
Assert.equal(PlacesUtils.history.frecencyStandardDeviation, 0);
await checkStats(frecenciesByURL);
await cleanUp();
});
// Makes sure the prefs that store the stats are updated.
add_task(async function preferences() {
Assert.equal(PlacesUtils.history.frecencyMean, 0);
Assert.equal(PlacesUtils.history.frecencyStandardDeviation, 0);
let url = "http://example.com/";
await PlacesTestUtils.addVisits([{ uri: url }]);
let frecency = frecencyForUrl(url);
Assert.ok(frecency > 0, "Sanity check");
Assert.equal(PlacesUtils.history.frecencyMean, frecency);
Assert.equal(PlacesUtils.history.frecencyStandardDeviation, 0);
let expectedValuesByName = {
"places.frecency.stats.count": "1",
"places.frecency.stats.sum": String(frecency),
"places.frecency.stats.sumOfSquares": String(frecency * frecency),
};
info("Waiting for preferences to be updated...");
await TestUtils.topicObserved("places-frecency-stats-prefs-updated", () => {
return Object.entries(expectedValuesByName).every(([name, expected]) => {
let actual = Services.prefs.getCharPref(name, "");
info(`${name} => ${actual} (expected=${expected})`);
return actual == expected;
});
});
Assert.ok(true, "Preferences updated as expected");
await cleanUp();
});
function mean(values) {
if (values.length == 0) {
return 0;
}
return values.reduce((sum, value) => {
sum += value;
return sum;
}, 0) / values.length;
async function checkStats(frecenciesByURL) {
let stats = await promiseStats();
let fs = Object.values(frecenciesByURL);
Assert.equal(stats.count, fs.length);
Assert.equal(stats.sum, fs.reduce((memo, f) => memo + f, 0));
Assert.equal(stats.squares, fs.reduce((memo, f) => memo + (f * f), 0));
}
function stddev(values) {
if (values.length <= 1) {
return 0;
}
let sum = values.reduce((memo, value) => {
memo += value;
return memo;
}, 0);
let sumOfSquares = values.reduce((memo, value) => {
memo += value * value;
return memo;
}, 0);
return Math.sqrt(
(sumOfSquares - ((sum * sum) / values.length)) / values.length
);
async function promiseStats() {
let db = await PlacesUtils.promiseDBConnection();
let rows = await db.execute(`
SELECT
IFNULL((SELECT value FROM moz_meta WHERE key = "frecency_count"), 0),
IFNULL((SELECT value FROM moz_meta WHERE key = "frecency_sum"), 0),
IFNULL((SELECT value FROM moz_meta WHERE key = "frecency_sum_of_squares"), 0)
`);
return {
count: rows[0].getResultByIndex(0),
sum: rows[0].getResultByIndex(1),
squares: rows[0].getResultByIndex(2),
};
}
async function cleanUp() {