mozilla
/
gecko-dev
зеркало из https://github.com/mozilla/gecko-dev.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 1813986 - Add an asyncVacuum() method to storage async connections, and let VacuumManager use it. r=asuth 

Add asyncVacuum to mozIStorageAsyncConnection, that dispatches a runnable to
the helper thread, where it will execute a full or incremental vacuum, depending
on the connection auto_vacuum value.
It also supports vacuuming attached schemas.
asyncVacuum() supports changing both the page_size and auto_vacuum.
Change mozIStorageVacuumParticipant to return a mozIStorageAsyncConnection and
allow specifying whether incremental vacuum should be enabled.
Change vacuumManager notification from heavy-io-task to vacuum-begin and vacuum-end
since the original proposal of notifying heavy IO didn't take off.
Cleanup test_vacuum to be able to use instances of the test VacuumParticipant,
that means we can remove the no more necessary registerESM hack.
Fix Places History as the only cpp consumer.

Differential Revision: https://phabricator.services.mozilla.com/D168298
This commit is contained in:
Marco Bonardo 2023-03-22 15:36:37 +00:00
Родитель 7791ea4730
Коммит f3e842d937
12 изменённых файлов: 852 добавлений и 558 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

6
dom/cache/Connection.cpp поставляемый
Просмотреть файл

@ -49,6 +49,12 @@ Connection::Close() {
// mozIStorageAsyncConnection methods
NS_IMETHODIMP
Connection::AsyncVacuum(mozIStorageCompletionCallback*, bool, int32_t) {
// async methods are not supported
return NS_ERROR_NOT_IMPLEMENTED;
}
NS_IMETHODIMP
Connection::AsyncClose(mozIStorageCompletionCallback*) {
// async methods are not supported

231
storage/VacuumManager.cpp
Просмотреть файл

@ -8,6 +8,7 @@
#include "VacuumManager.h"
#include "mozilla/ErrorNames.h"
#include "mozilla/Services.h"
#include "mozilla/Preferences.h"
#include "nsIObserverService.h"
@ -18,104 +19,63 @@
#include "mozilla/StaticPrefs_storage.h"
#include "mozStorageConnection.h"
#include "mozStoragePrivateHelpers.h"
#include "mozIStorageStatement.h"
#include "mozIStorageStatementCallback.h"
#include "mozIStorageCompletionCallback.h"
#include "mozIStorageAsyncStatement.h"
#include "mozIStoragePendingStatement.h"
#include "mozIStorageError.h"
#include "mozStorageHelper.h"
#include "nsXULAppAPI.h"
#include "xpcpublic.h"
#define OBSERVER_TOPIC_IDLE_DAILY "idle-daily"
#define OBSERVER_TOPIC_XPCOM_SHUTDOWN "xpcom-shutdown"
// Used to notify begin and end of a heavy IO task.
#define OBSERVER_TOPIC_HEAVY_IO "heavy-io-task"
#define OBSERVER_DATA_VACUUM_BEGIN u"vacuum-begin"
#define OBSERVER_DATA_VACUUM_END u"vacuum-end"
// Used to notify the begin and end of a vacuum operation.
#define OBSERVER_TOPIC_VACUUM_BEGIN "vacuum-begin"
#define OBSERVER_TOPIC_VACUUM_END "vacuum-end"
// This notification is sent only in automation when vacuum for a database is
// skipped, and can thus be used to verify that.
#define OBSERVER_TOPIC_VACUUM_SKIP "vacuum-skip"
// This preferences root will contain last vacuum timestamps (in seconds) for
// each database. The database filename is used as a key.
#define PREF_VACUUM_BRANCH "storage.vacuum.last."
// Time between subsequent vacuum calls for a certain database.
#define VACUUM_INTERVAL_SECONDS 30 * 86400 // 30 days.
#define VACUUM_INTERVAL_SECONDS (30 * 86400) // 30 days.
extern mozilla::LazyLogModule gStorageLog;
namespace mozilla {
namespace storage {
namespace mozilla::storage {
namespace {
////////////////////////////////////////////////////////////////////////////////
//// BaseCallback
class BaseCallback : public mozIStorageStatementCallback {
public:
NS_DECL_ISUPPORTS
NS_DECL_MOZISTORAGESTATEMENTCALLBACK
BaseCallback() {}
protected:
virtual ~BaseCallback() {}
};
NS_IMETHODIMP
BaseCallback::HandleError(mozIStorageError* aError) {
#ifdef DEBUG
int32_t result;
nsresult rv = aError->GetResult(&result);
NS_ENSURE_SUCCESS(rv, rv);
nsAutoCString message;
rv = aError->GetMessage(message);
NS_ENSURE_SUCCESS(rv, rv);
nsAutoCString warnMsg;
warnMsg.AppendLiteral("An error occured during async execution: ");
warnMsg.AppendInt(result);
warnMsg.Append(' ');
warnMsg.Append(message);
NS_WARNING(warnMsg.get());
#endif
return NS_OK;
}
NS_IMETHODIMP
BaseCallback::HandleResult(mozIStorageResultSet* aResultSet) {
// We could get results from PRAGMA statements, but we don't mind them.
return NS_OK;
}
NS_IMETHODIMP
BaseCallback::HandleCompletion(uint16_t aReason) {
// By default BaseCallback will just be silent on completion.
return NS_OK;
}
NS_IMPL_ISUPPORTS(BaseCallback, mozIStorageStatementCallback)
////////////////////////////////////////////////////////////////////////////////
//// Vacuumer declaration.
class Vacuumer : public BaseCallback {
class Vacuumer final : public mozIStorageCompletionCallback {
public:
NS_DECL_MOZISTORAGESTATEMENTCALLBACK
NS_DECL_ISUPPORTS
NS_DECL_MOZISTORAGECOMPLETIONCALLBACK
explicit Vacuumer(mozIStorageVacuumParticipant* aParticipant);
bool execute();
nsresult notifyCompletion(bool aSucceeded);
private:
nsresult notifyCompletion(bool aSucceeded);
~Vacuumer() = default;
nsCOMPtr<mozIStorageVacuumParticipant> mParticipant;
nsCString mDBFilename;
nsCOMPtr<mozIStorageConnection> mDBConn;
nsCOMPtr<mozIStorageAsyncConnection> mDBConn;
};
////////////////////////////////////////////////////////////////////////////////
//// Vacuumer implementation.
NS_IMPL_ISUPPORTS(Vacuumer, mozIStorageCompletionCallback)
Vacuumer::Vacuumer(mozIStorageVacuumParticipant* aParticipant)
: mParticipant(aParticipant) {}
@ -124,30 +84,21 @@ bool Vacuumer::execute() {
// Get the connection and check its validity.
nsresult rv = mParticipant->GetDatabaseConnection(getter_AddRefs(mDBConn));
NS_ENSURE_SUCCESS(rv, false);
bool ready = false;
if (!mDBConn || NS_FAILED(mDBConn->GetConnectionReady(&ready)) || !ready) {
NS_WARNING("Unable to get a connection to vacuum database");
return false;
}
if (NS_FAILED(rv) || !mDBConn) return false;
// Ask for the expected page size. Vacuum can change the page size, unless
// the database is using WAL journaling.
// TODO Bug 634374: figure out a strategy to fix page size with WAL.
int32_t expectedPageSize = 0;
rv = mParticipant->GetExpectedDatabasePageSize(&expectedPageSize);
if (NS_FAILED(rv) || !Service::pageSizeIsValid(expectedPageSize)) {
NS_WARNING("Invalid page size requested for database, will use default ");
NS_WARNING(mDBFilename.get());
expectedPageSize = Service::kDefaultPageSize;
}
nsCOMPtr<nsIObserverService> os = mozilla::services::GetObserverService();
bool inAutomation = xpc::IsInAutomation();
// Get the database filename. Last vacuum time is stored under this name
// in PREF_VACUUM_BRANCH.
nsCOMPtr<nsIFile> databaseFile;
mDBConn->GetDatabaseFile(getter_AddRefs(databaseFile));
if (!databaseFile) {
NS_WARNING("Trying to vacuum a in-memory database!");
if (inAutomation && os) {
mozilla::Unused << os->NotifyObservers(
nullptr, OBSERVER_TOPIC_VACUUM_SKIP, u":memory:");
}
return false;
}
nsAutoString databaseFilename;
@ -164,6 +115,11 @@ bool Vacuumer::execute() {
rv = Preferences::GetInt(prefName.get(), &lastVacuum);
if (NS_SUCCEEDED(rv) && (now - lastVacuum) < VACUUM_INTERVAL_SECONDS) {
// This database was vacuumed recently, skip it.
if (inAutomation && os) {
mozilla::Unused << os->NotifyObservers(
nullptr, OBSERVER_TOPIC_VACUUM_SKIP,
NS_ConvertUTF8toUTF16(mDBFilename).get());
}
return false;
}
@ -174,86 +130,48 @@ bool Vacuumer::execute() {
rv = mParticipant->OnBeginVacuum(&vacuumGranted);
NS_ENSURE_SUCCESS(rv, false);
if (!vacuumGranted) {
if (inAutomation && os) {
mozilla::Unused << os->NotifyObservers(
nullptr, OBSERVER_TOPIC_VACUUM_SKIP,
NS_ConvertUTF8toUTF16(mDBFilename).get());
}
return false;
}
// Notify a heavy IO task is about to start.
nsCOMPtr<nsIObserverService> os = mozilla::services::GetObserverService();
if (os) {
rv = os->NotifyObservers(nullptr, OBSERVER_TOPIC_HEAVY_IO,
OBSERVER_DATA_VACUUM_BEGIN);
MOZ_ASSERT(NS_SUCCEEDED(rv), "Should be able to notify");
// Ask for the expected page size. Vacuum can change the page size, unless
// the database is using WAL journaling.
// TODO Bug 634374: figure out a strategy to fix page size with WAL.
int32_t expectedPageSize = 0;
rv = mParticipant->GetExpectedDatabasePageSize(&expectedPageSize);
if (NS_FAILED(rv) || !Service::pageSizeIsValid(expectedPageSize)) {
NS_WARNING("Invalid page size requested for database, won't set it. ");
NS_WARNING(mDBFilename.get());
expectedPageSize = 0;
}
// Execute the statements separately, since the pragma may conflict with the
// vacuum, if they are executed in the same transaction.
nsCOMPtr<mozIStorageAsyncStatement> pageSizeStmt;
nsAutoCString pageSizeQuery(MOZ_STORAGE_UNIQUIFY_QUERY_STR
"PRAGMA page_size = ");
pageSizeQuery.AppendInt(expectedPageSize);
rv = mDBConn->CreateAsyncStatement(pageSizeQuery,
getter_AddRefs(pageSizeStmt));
NS_ENSURE_SUCCESS(rv, false);
RefPtr<BaseCallback> callback = new BaseCallback();
nsCOMPtr<mozIStoragePendingStatement> ps;
rv = pageSizeStmt->ExecuteAsync(callback, getter_AddRefs(ps));
NS_ENSURE_SUCCESS(rv, false);
bool incremental = false;
mozilla::Unused << mParticipant->GetUseIncrementalVacuum(&incremental);
nsCOMPtr<mozIStorageAsyncStatement> stmt;
rv = mDBConn->CreateAsyncStatement("VACUUM"_ns, getter_AddRefs(stmt));
NS_ENSURE_SUCCESS(rv, false);
rv = stmt->ExecuteAsync(this, getter_AddRefs(ps));
NS_ENSURE_SUCCESS(rv, false);
// Notify vacuum is about to start.
if (os) {
mozilla::Unused << os->NotifyObservers(
nullptr, OBSERVER_TOPIC_VACUUM_BEGIN,
NS_ConvertUTF8toUTF16(mDBFilename).get());
}
rv = mDBConn->AsyncVacuum(this, incremental, expectedPageSize);
if (NS_FAILED(rv)) {
// The connection is not ready.
mozilla::Unused << Complete(rv, nullptr);
return false;
}
return true;
}
////////////////////////////////////////////////////////////////////////////////
//// mozIStorageStatementCallback
NS_IMETHODIMP
Vacuumer::HandleError(mozIStorageError* aError) {
int32_t result;
nsresult rv;
nsAutoCString message;
#ifdef DEBUG
rv = aError->GetResult(&result);
NS_ENSURE_SUCCESS(rv, rv);
rv = aError->GetMessage(message);
NS_ENSURE_SUCCESS(rv, rv);
nsAutoCString warnMsg;
warnMsg.AppendLiteral("Unable to vacuum database: ");
warnMsg.Append(mDBFilename);
warnMsg.AppendLiteral(" - ");
warnMsg.AppendInt(result);
warnMsg.Append(' ');
warnMsg.Append(message);
NS_WARNING(warnMsg.get());
#endif
if (MOZ_LOG_TEST(gStorageLog, LogLevel::Error)) {
rv = aError->GetResult(&result);
NS_ENSURE_SUCCESS(rv, rv);
rv = aError->GetMessage(message);
NS_ENSURE_SUCCESS(rv, rv);
MOZ_LOG(gStorageLog, LogLevel::Error,
("Vacuum failed with error: %d '%s'. Database was: '%s'", result,
message.get(), mDBFilename.get()));
}
return NS_OK;
}
NS_IMETHODIMP
Vacuumer::HandleResult(mozIStorageResultSet* aResultSet) {
MOZ_ASSERT_UNREACHABLE("Got a resultset from a vacuum?");
return NS_OK;
}
NS_IMETHODIMP
Vacuumer::HandleCompletion(uint16_t aReason) {
if (aReason == REASON_FINISHED) {
Vacuumer::Complete(nsresult aStatus, nsISupports* aValue) {
if (NS_SUCCEEDED(aStatus)) {
// Update last vacuum time.
int32_t now = static_cast<int32_t>(PR_Now() / PR_USEC_PER_SEC);
MOZ_ASSERT(!mDBFilename.IsEmpty(), "Database filename cannot be empty");
@ -261,18 +179,30 @@ Vacuumer::HandleCompletion(uint16_t aReason) {
prefName += mDBFilename;
DebugOnly<nsresult> rv = Preferences::SetInt(prefName.get(), now);
MOZ_ASSERT(NS_SUCCEEDED(rv), "Should be able to set a preference");
notifyCompletion(true);
return NS_OK;
}
notifyCompletion(aReason == REASON_FINISHED);
nsAutoCString errName;
GetErrorName(aStatus, errName);
nsCString errMsg = nsPrintfCString(
"Vacuum failed on '%s' with error %s - code %" PRIX32, mDBFilename.get(),
errName.get(), static_cast<uint32_t>(aStatus));
NS_WARNING(errMsg.get());
if (MOZ_LOG_TEST(gStorageLog, LogLevel::Error)) {
MOZ_LOG(gStorageLog, LogLevel::Error, ("%s", errMsg.get()));
}
notifyCompletion(false);
return NS_OK;
}
nsresult Vacuumer::notifyCompletion(bool aSucceeded) {
nsCOMPtr<nsIObserverService> os = mozilla::services::GetObserverService();
if (os) {
os->NotifyObservers(nullptr, OBSERVER_TOPIC_HEAVY_IO,
OBSERVER_DATA_VACUUM_END);
mozilla::Unused << os->NotifyObservers(
nullptr, OBSERVER_TOPIC_VACUUM_END,
NS_ConvertUTF8toUTF16(mDBFilename).get());
}
nsresult rv = mParticipant->OnEndVacuum(aSucceeded);
@ -353,5 +283,4 @@ VacuumManager::Observe(nsISupports* aSubject, const char* aTopic,
return NS_OK;
}
} // namespace storage
} // namespace mozilla
} // namespace mozilla::storage

32
storage/mozIStorageAsyncConnection.idl
Просмотреть файл

@ -193,6 +193,38 @@ interface mozIStorageAsyncConnection : nsISupports {
*/
void interrupt();
/**
* Vacuum the main database plus all the attached one.
* If the database is in auto_vacuum = INCREMENTAL mode, this executes an
* incremental_vacuum, otherwise it will always execute a full vacuum.
*
* While it's possible to invoke this method directly, it's suggested, when
* possible, to use the VacuumManager instead.
* That means registering your component for the "vacuum-participant" XPCOM
* category, and implement the mozIStorageVacuumParticipant interface.
*
* @param [aCallback] Completion callback invoked once the operation is
* complete.
* @param [aUseIncremental] When set to true, this will try to convert the
* main schema to auto_vacuum = INCREMENTAL mode, if it's not set yet.
* When set to false, it will try to set auto_vacuum = NONE.
* Note a full vacuum will be executed if the auto_vacuum mode must be
* changed, otherwise an incremental vacuum will happen if the database
* is already in INCREMENTAL mode.
* @param [aSetPageSize] This can be used to change the database page_size, a
* full vacuum will be executed to persist the change. If the page
* size is already correct, or you pass 0, this will be a no-op.
* @throws If it's not possible to start the async vacuum operation, note in
* this case the callback won't be invoked.
* @note Vacuum will fail inside a transaction, or if there is an ongoing
* read statement.
*/
void asyncVacuum(
[optional] in mozIStorageCompletionCallback aCallback,
[optional] in boolean aUseIncremental,
[optional] in long aSetPageSize
);
//////////////////////////////////////////////////////////////////////////////
//// Statement creation

22
storage/mozIStorageVacuumParticipant.idl
Просмотреть файл

@ -6,7 +6,7 @@
#include "nsISupports.idl"
interface mozIStorageConnection;
interface mozIStorageAsyncConnection;
/**
* This interface contains the information that the Storage service needs to
@ -19,19 +19,27 @@ interface mozIStorageConnection;
interface mozIStorageVacuumParticipant : nsISupports {
/**
* The expected page size in bytes for the database. The vacuum manager will
* try to correct the page size during idle based on this value.
* try to correct the page size by executing a full vacuum.
*
* @note If the database is using the WAL journal mode, the page size won't
* be changed to the requested value. See bug 634374.
* be changed to the requested value. See bug 634374.
* @note Valid page size values are powers of 2 between 512 and 65536.
* The suggested value is mozIStorageConnection::defaultPageSize.
*/
readonly attribute long expectedDatabasePageSize;
/**
* Whether the main schema should be using auto_vacuum = INCREMENTAL.
* This will cause auto_vacuum to change to INCREMENTAL if it's not set yet.
* It is not possible to change mode of any attached databases through this,
* to do that you must open a separate connection and use asyncVacuum() on it.
*/
readonly attribute boolean useIncrementalVacuum;
/**
* Connection to the database file to be vacuumed.
*/
readonly attribute mozIStorageConnection databaseConnection;
readonly attribute mozIStorageAsyncConnection databaseConnection;
/**
* Notifies when a vacuum operation begins. Listeners should avoid using the
@ -41,9 +49,9 @@ interface mozIStorageVacuumParticipant : nsISupports {
* opt-out for now, it will be retried later. Useful when participant
* is running some other heavy operation that can't be interrupted.
*
* @note When a vacuum operation starts or ends it will also dispatch a global
* "heavy-io-task" notification through the observer service with the
* data argument being either "vacuum-begin" or "vacuum-end".
* @note When a vacuum operation starts or ends it will also dispatch global
* "vacuum-begin" and "vacuum-end" notifications through the observer
* service with the data argument being the database filename.
*/
boolean onBeginVacuum();

245
storage/mozStorageConnection.cpp
Просмотреть файл

@ -8,6 +8,7 @@
#include "nsIFile.h"
#include "nsIFileURL.h"
#include "nsIXPConnect.h"
#include "mozilla/AppShutdown.h"
#include "mozilla/Telemetry.h"
#include "mozilla/Mutex.h"
#include "mozilla/CondVar.h"
@ -419,6 +420,151 @@ class CloseListener final : public mozIStorageCompletionCallback {
NS_IMPL_ISUPPORTS(CloseListener, mozIStorageCompletionCallback)
class AsyncVacuumEvent final : public Runnable {
public:
AsyncVacuumEvent(Connection* aConnection,
mozIStorageCompletionCallback* aCallback,
bool aUseIncremental, int32_t aSetPageSize)
: Runnable("storage::AsyncVacuum"),
mConnection(aConnection),
mCallback(aCallback),
mUseIncremental(aUseIncremental),
mSetPageSize(aSetPageSize),
mStatus(NS_ERROR_UNEXPECTED) {}
NS_IMETHOD Run() override {
// This is initially dispatched to the helper thread, then re-dispatched
// to the opener thread, where it will callback.
if (IsOnCurrentSerialEventTarget(mConnection->eventTargetOpenedOn)) {
// Send the completion event.
if (mCallback) {
mozilla::Unused << mCallback->Complete(mStatus, nullptr);
}
return NS_OK;
}
// Ensure to invoke the callback regardless of errors.
auto guard = MakeScopeExit([&]() {
mConnection->mIsStatementOnHelperThreadInterruptible = false;
mozilla::Unused << mConnection->eventTargetOpenedOn->Dispatch(
this, NS_DISPATCH_NORMAL);
});
// Get list of attached databases.
nsCOMPtr<mozIStorageStatement> stmt;
nsresult rv = mConnection->CreateStatement(MOZ_STORAGE_UNIQUIFY_QUERY_STR
"PRAGMA database_list"_ns,
getter_AddRefs(stmt));
NS_ENSURE_SUCCESS(rv, rv);
// We must accumulate names and loop through them later, otherwise VACUUM
// will see an ongoing statement and bail out.
nsTArray<nsCString> schemaNames;
bool hasResult = false;
while (stmt && NS_SUCCEEDED(stmt->ExecuteStep(&hasResult)) && hasResult) {
nsAutoCString name;
rv = stmt->GetUTF8String(1, name);
if (NS_SUCCEEDED(rv) && !name.EqualsLiteral("temp")) {
schemaNames.AppendElement(name);
}
}
mStatus = NS_OK;
// Mark this vacuum as an interruptible operation, so it can be interrupted
// if the connection closes during shutdown.
mConnection->mIsStatementOnHelperThreadInterruptible = true;
for (const nsCString& schemaName : schemaNames) {
rv = this->Vacuum(schemaName);
if (NS_FAILED(rv)) {
// This is sub-optimal since it's only keeping the last error reason,
// but it will do for now.
mStatus = rv;
}
}
return mStatus;
}
nsresult Vacuum(const nsACString& aSchemaName) {
// Abort if we're in shutdown.
if (AppShutdown::IsInOrBeyond(ShutdownPhase::AppShutdownConfirmed)) {
return NS_ERROR_ABORT;
}
int32_t removablePages = mConnection->RemovablePagesInFreeList(aSchemaName);
if (!removablePages) {
// There's no empty pages to remove, so skip this vacuum for now.
return NS_OK;
}
nsresult rv;
bool needsFullVacuum = true;
if (mSetPageSize) {
nsAutoCString query(MOZ_STORAGE_UNIQUIFY_QUERY_STR "PRAGMA ");
query.Append(aSchemaName);
query.AppendLiteral(".page_size = ");
query.AppendInt(mSetPageSize);
nsCOMPtr<mozIStorageStatement> stmt;
rv = mConnection->ExecuteSimpleSQL(query);
NS_ENSURE_SUCCESS(rv, rv);
}
// Check auto_vacuum.
{
nsAutoCString query(MOZ_STORAGE_UNIQUIFY_QUERY_STR "PRAGMA ");
query.Append(aSchemaName);
query.AppendLiteral(".auto_vacuum");
nsCOMPtr<mozIStorageStatement> stmt;
rv = mConnection->CreateStatement(query, getter_AddRefs(stmt));
NS_ENSURE_SUCCESS(rv, rv);
bool hasResult = false;
bool changeAutoVacuum = false;
if (stmt && NS_SUCCEEDED(stmt->ExecuteStep(&hasResult)) && hasResult) {
bool isIncrementalVacuum = stmt->AsInt32(0) == 2;
changeAutoVacuum = isIncrementalVacuum != mUseIncremental;
if (isIncrementalVacuum && !changeAutoVacuum) {
needsFullVacuum = false;
}
}
// Changing auto_vacuum is only supported on the main schema.
if (aSchemaName.EqualsLiteral("main") && changeAutoVacuum) {
nsAutoCString query(MOZ_STORAGE_UNIQUIFY_QUERY_STR "PRAGMA ");
query.Append(aSchemaName);
query.AppendLiteral(".auto_vacuum = ");
query.AppendInt(mUseIncremental ? 2 : 0);
rv = mConnection->ExecuteSimpleSQL(query);
NS_ENSURE_SUCCESS(rv, rv);
}
}
if (needsFullVacuum) {
nsAutoCString query(MOZ_STORAGE_UNIQUIFY_QUERY_STR "VACUUM ");
query.Append(aSchemaName);
rv = mConnection->ExecuteSimpleSQL(query);
// TODO (Bug 1818039): Report failed vacuum telemetry.
NS_ENSURE_SUCCESS(rv, rv);
} else {
nsAutoCString query(MOZ_STORAGE_UNIQUIFY_QUERY_STR "PRAGMA ");
query.Append(aSchemaName);
query.AppendLiteral(".incremental_vacuum(");
query.AppendInt(removablePages);
query.AppendLiteral(")");
rv = mConnection->ExecuteSimpleSQL(query);
NS_ENSURE_SUCCESS(rv, rv);
}
return NS_OK;
}
~AsyncVacuumEvent() override {
NS_ReleaseOnMainThread("AsyncVacuum::mConnection", mConnection.forget());
NS_ReleaseOnMainThread("AsyncVacuum::mCallback", mCallback.forget());
}
private:
RefPtr<Connection> mConnection;
nsCOMPtr<mozIStorageCompletionCallback> mCallback;
bool mUseIncremental;
int32_t mSetPageSize;
Atomic<nsresult> mStatus;
};
} // namespace
////////////////////////////////////////////////////////////////////////////////
@ -430,6 +576,7 @@ Connection::Connection(Service* aService, int aFlags,
: sharedAsyncExecutionMutex("Connection::sharedAsyncExecutionMutex"),
sharedDBMutex("Connection::sharedDBMutex"),
eventTargetOpenedOn(WrapNotNull(GetCurrentSerialEventTarget())),
mIsStatementOnHelperThreadInterruptible(false),
mDBConn(nullptr),
mDefaultTransactionType(mozIStorageConnection::TRANSACTION_DEFERRED),
mDestroying(false),
@ -442,7 +589,8 @@ Connection::Connection(Service* aService, int aFlags,
aInterruptible),
mIgnoreLockingMode(aIgnoreLockingMode),
mAsyncExecutionThreadShuttingDown(false),
mConnectionClosed(false) {
mConnectionClosed(false),
mGrowthChunkSize(0) {
MOZ_ASSERT(!mIgnoreLockingMode || mFlags & SQLITE_OPEN_READONLY,
"Can't ignore locking for a non-readonly connection!");
mStorageService->registerConnection(this);
@ -1514,6 +1662,17 @@ Connection::AsyncClose(mozIStorageCompletionCallback* aCallback) {
return NS_OK;
}
// If we're closing the connection during shutdown, and there is an
// interruptible statement running on the helper thread, issue a
// sqlite3_interrupt() to avoid crashing when that statement takes a long
// time (for example a vacuum).
if (AppShutdown::IsInOrBeyond(ShutdownPhase::AppShutdownConfirmed) &&
mInterruptible && mIsStatementOnHelperThreadInterruptible) {
MOZ_ASSERT(!isClosing(), "Must not be closing, see Interrupt()");
DebugOnly<nsresult> rv2 = Interrupt();
MOZ_ASSERT(NS_SUCCEEDED(rv2));
}
// setClosedState nullifies our connection pointer, so we take a raw pointer
// off it, to pass it through the close procedure.
sqlite3* nativeConn = mDBConn;
@ -1762,6 +1921,36 @@ Connection::Interrupt() {
return NS_OK;
}
NS_IMETHODIMP
Connection::AsyncVacuum(mozIStorageCompletionCallback* aCallback,
bool aUseIncremental, int32_t aSetPageSize) {
NS_ENSURE_TRUE(NS_IsMainThread(), NS_ERROR_NOT_SAME_THREAD);
// Abort if we're shutting down.
if (AppShutdown::IsInOrBeyond(ShutdownPhase::AppShutdownConfirmed)) {
return NS_ERROR_ABORT;
}
// Check if AsyncClose or Close were already invoked.
if (!connectionReady()) {
return NS_ERROR_NOT_INITIALIZED;
}
nsresult rv = ensureOperationSupported(ASYNCHRONOUS);
if (NS_FAILED(rv)) {
return rv;
}
nsIEventTarget* asyncThread = getAsyncExecutionTarget();
if (!asyncThread) {
return NS_ERROR_NOT_INITIALIZED;
}
// Create and dispatch our vacuum event to the background thread.
nsCOMPtr<nsIRunnable> vacuumEvent =
new AsyncVacuumEvent(this, aCallback, aUseIncremental, aSetPageSize);
rv = asyncThread->Dispatch(vacuumEvent, NS_DISPATCH_NORMAL);
NS_ENSURE_SUCCESS(rv, rv);
return NS_OK;
}
NS_IMETHODIMP
Connection::GetDefaultPageSize(int32_t* _defaultPageSize) {
*_defaultPageSize = Service::kDefaultPageSize;
@ -2280,15 +2469,59 @@ Connection::SetGrowthIncrement(int32_t aChunkSize,
return NS_ERROR_FILE_TOO_BIG;
}
(void)::sqlite3_file_control(mDBConn,
aDatabaseName.Length()
? nsPromiseFlatCString(aDatabaseName).get()
: nullptr,
SQLITE_FCNTL_CHUNK_SIZE, &aChunkSize);
int srv = ::sqlite3_file_control(
mDBConn,
aDatabaseName.Length() ? nsPromiseFlatCString(aDatabaseName).get()
: nullptr,
SQLITE_FCNTL_CHUNK_SIZE, &aChunkSize);
if (srv == SQLITE_OK) {
mGrowthChunkSize = aChunkSize;
}
#endif
return NS_OK;
}
int32_t Connection::RemovablePagesInFreeList(const nsACString& aSchemaName) {
int32_t freeListPagesCount = 0;
if (!isConnectionReadyOnThisThread()) {
MOZ_ASSERT(false, "Database connection is not ready");
return freeListPagesCount;
}
{
nsAutoCString query(MOZ_STORAGE_UNIQUIFY_QUERY_STR "PRAGMA ");
query.Append(aSchemaName);
query.AppendLiteral(".freelist_count");
nsCOMPtr<mozIStorageStatement> stmt;
DebugOnly<nsresult> rv = CreateStatement(query, getter_AddRefs(stmt));
MOZ_ASSERT(NS_SUCCEEDED(rv));
bool hasResult = false;
if (stmt && NS_SUCCEEDED(stmt->ExecuteStep(&hasResult)) && hasResult) {
freeListPagesCount = stmt->AsInt32(0);
}
}
// If there's no chunk size set, any page is good to be removed.
if (mGrowthChunkSize == 0 || freeListPagesCount == 0) {
return freeListPagesCount;
}
int32_t pageSize;
{
nsAutoCString query(MOZ_STORAGE_UNIQUIFY_QUERY_STR "PRAGMA ");
query.Append(aSchemaName);
query.AppendLiteral(".page_size");
nsCOMPtr<mozIStorageStatement> stmt;
DebugOnly<nsresult> rv = CreateStatement(query, getter_AddRefs(stmt));
MOZ_ASSERT(NS_SUCCEEDED(rv));
bool hasResult = false;
if (stmt && NS_SUCCEEDED(stmt->ExecuteStep(&hasResult)) && hasResult) {
pageSize = stmt->AsInt32(0);
} else {
MOZ_ASSERT(false, "Couldn't get page_size");
return 0;
}
}
return std::max(0, freeListPagesCount - (mGrowthChunkSize / pageSize));
}
NS_IMETHODIMP
Connection::EnableModule(const nsACString& aModuleName) {
if (!connectionReady()) {

24
storage/mozStorageConnection.h
Просмотреть файл

@ -317,6 +317,25 @@ class Connection final : public mozIStorageConnection,
*/
void RecordQueryStatus(int srv);
/**
* Returns the number of pages in the free list that can be removed.
*
* A database may use chunked growth to reduce filesystem fragmentation, then
* Sqlite will allocate and release multiple pages in chunks. We want to
* preserve the chunked space to reduce the likelihood of fragmentation,
* releasing free pages only when there's a large amount of them. This can be
* used to decide if it's worth vacuuming the database and how many pages can
* be vacuumed in case of incremental vacuum.
* Note this returns 0, and asserts, in case of errors.
*/
int32_t RemovablePagesInFreeList(const nsACString& aSchemaName);
/**
* Whether the statement currently running on the helper thread can be
* interrupted.
*/
Atomic<bool> mIsStatementOnHelperThreadInterruptible;
private:
~Connection();
nsresult initializeInternal();
@ -483,6 +502,11 @@ class Connection final : public mozIStorageConnection,
* sharedAsyncExecutionMutex.
*/
bool mConnectionClosed;
/**
* Stores the growth increment chunk size, set through SetGrowthIncrement().
*/
Atomic<int32_t> mGrowthChunkSize;
};
/**

4
storage/test/moz.build
Просмотреть файл

@ -7,3 +7,7 @@
XPCSHELL_TESTS_MANIFESTS += ["unit/xpcshell.ini"]
TEST_DIRS += ["gtest"]
TESTING_JS_MODULES += [
"unit/VacuumParticipant.sys.mjs",
]

177
storage/test/unit/VacuumParticipant.sys.mjs
Просмотреть файл

@ -4,95 +4,106 @@
// This testing component is used in test_vacuum* tests.
/**
* Returns a new nsIFile reference for a profile database.
* @param filename for the database, excluded the .sqlite extension.
*/
function new_db_file(name) {
let file = Services.dirsvc.get("ProfD", Ci.nsIFile);
file.append(name + ".sqlite");
return file;
}
const CAT_NAME = "vacuum-participant";
const CONTRACT_ID = "@unit.test.com/test-vacuum-participant;1";
/**
* Opens and returns a connection to the provided database file.
* @param nsIFile interface to the database file.
*/
function getDatabase(aFile) {
return Services.storage.openDatabase(aFile);
}
import { MockRegistrar } from "resource://testing-common/MockRegistrar.sys.mjs";
import { TestUtils } from "resource://testing-common/TestUtils.sys.mjs";
export function VacuumParticipant() {
this._dbConn = getDatabase(new_db_file("testVacuum"));
Services.obs.addObserver(this, "test-options");
}
export class VacuumParticipant {
#dbConn;
#expectedPageSize = 0;
#useIncrementalVacuum = false;
#grant = false;
/**
* Build a VacuumParticipant instance.
* Note: After creation you must await instance.promiseRegistered() to ensure
* Category Caches have been updated.
*
* @param {mozIStorageAsyncConnection} databaseConnection
* The connection to be vacuumed.
* @param {Number} [expectedPageSize]
* Used to change the database page size.
* @param {boolean} [useIncrementalVacuum]
* Whether to enable incremental vacuum on the database.
* @param {boolean} [grant]
* Whether the vacuum operation should be granted.
*/
constructor(
databaseConnection,
{ expectedPageSize = 0, useIncrementalVacuum = false, grant = true } = {}
) {
this.#dbConn = databaseConnection;
// Register as the only participant.
this.#unregisterAllParticipants();
this.#registerAsParticipant();
this.#expectedPageSize = expectedPageSize;
this.#useIncrementalVacuum = useIncrementalVacuum;
this.#grant = grant;
this.QueryInterface = ChromeUtils.generateQI([
"mozIStorageVacuumParticipant",
]);
}
promiseRegistered() {
// The category manager dispatches change notifications to the main thread,
// so we must wait one tick.
return TestUtils.waitForTick();
}
#registerAsParticipant() {
MockRegistrar.register(CONTRACT_ID, this);
Services.catMan.addCategoryEntry(
CAT_NAME,
"vacuumParticipant",
CONTRACT_ID,
false,
false
);
}
#unregisterAllParticipants() {
// First unregister other participants.
for (let { data: entry } of Services.catMan.enumerateCategory(CAT_NAME)) {
Services.catMan.deleteCategoryEntry("vacuum-participant", entry, false);
}
}
async dispose() {
this.#unregisterAllParticipants();
MockRegistrar.unregister(CONTRACT_ID);
await new Promise(resolve => {
this.#dbConn.asyncClose(resolve);
});
}
VacuumParticipant.prototype = {
get expectedDatabasePageSize() {
return this._dbConn.defaultPageSize;
},
get databaseConnection() {
return this._dbConn;
},
return this.#expectedPageSize;
}
_grant: true,
onBeginVacuum: function TVP_onBeginVacuum() {
if (!this._grant) {
this._grant = true;
get useIncrementalVacuum() {
return this.#useIncrementalVacuum;
}
get databaseConnection() {
return this.#dbConn;
}
onBeginVacuum() {
if (!this.#grant) {
return false;
}
Services.obs.notifyObservers(null, "test-begin-vacuum");
return true;
},
onEndVacuum: function TVP_EndVacuum(aSucceeded) {
if (this._stmt) {
this._stmt.finalize();
}
Services.obs.notifyObservers(null, "test-end-vacuum", aSucceeded);
},
observe: function TVP_observe(aSubject, aTopic, aData) {
if (aData == "opt-out") {
this._grant = false;
} else if (aData == "wal") {
try {
this._dbConn.close();
} catch (e) {
// Do nothing.
}
this._dbConn = getDatabase(new_db_file("testVacuum2"));
} else if (aData == "wal-fail") {
try {
this._dbConn.close();
} catch (e) {
// Do nothing.
}
this._dbConn = getDatabase(new_db_file("testVacuum3"));
// Use WAL journal mode.
this._dbConn.executeSimpleSQL("PRAGMA journal_mode = WAL");
// Create a not finalized statement.
this._stmt = this._dbConn.createStatement("SELECT :test");
this._stmt.params.test = 1;
this._stmt.executeStep();
} else if (aData == "memory") {
try {
this._dbConn.asyncClose();
} catch (e) {
// Do nothing.
}
this._dbConn = Services.storage.openSpecialDatabase("memory");
} else if (aData == "dispose") {
Services.obs.removeObserver(this, "test-options");
try {
this._dbConn.asyncClose();
} catch (e) {
// Do nothing.
}
}
},
QueryInterface: ChromeUtils.generateQI([
"mozIStorageVacuumParticipant",
"nsIObserver",
]),
};
}
onEndVacuum(succeeded) {
Services.obs.notifyObservers(
null,
succeeded ? "test-end-vacuum-success" : "test-end-vacuum-failure"
);
}
}

6
storage/test/unit/head_storage.js
Просмотреть файл

@ -11,12 +11,10 @@ var { AppConstants } = ChromeUtils.importESModule(
ChromeUtils.defineESModuleGetters(this, {
Sqlite: "resource://gre/modules/Sqlite.sys.mjs",
TelemetryTestUtils: "resource://testing-common/TelemetryTestUtils.sys.mjs",
TestUtils: "resource://testing-common/TestUtils.sys.mjs",
});
const { TelemetryTestUtils } = ChromeUtils.importESModule(
"resource://testing-common/TelemetryTestUtils.sys.mjs"
);
const OPEN_HISTOGRAM = "SQLITE_STORE_OPEN";
const QUERY_HISTOGRAM = "SQLITE_STORE_QUERY";

642
storage/test/unit/test_vacuum.js
Просмотреть файл

@ -2,325 +2,371 @@
* http://creativecommons.org/publicdomain/zero/1.0/
*/
// This file tests the Vacuum Manager.
// This file tests the Vacuum Manager and asyncVacuum().
const { MockRegistrar } = ChromeUtils.importESModule(
"resource://testing-common/MockRegistrar.sys.mjs"
const { VacuumParticipant } = ChromeUtils.importESModule(
"resource://testing-common/VacuumParticipant.sys.mjs"
);
/**
* Loads a test component that will register as a vacuum-participant.
* If other participants are found they will be unregistered, to avoid conflicts
* with the test itself.
*/
function load_test_vacuum_component() {
const CATEGORY_NAME = "vacuum-participant";
const CONTRACT_ID = "@unit.test.com/test-vacuum-participant;1";
MockRegistrar.registerESM(
CONTRACT_ID,
"resource://test/VacuumParticipant.sys.mjs",
"VacuumParticipant"
);
let { catMan } = Services;
// Temporary unregister other participants for this test.
for (let { data: entry } of catMan.enumerateCategory(CATEGORY_NAME)) {
print("Check if the found category entry (" + entry + ") is expected.");
catMan.deleteCategoryEntry("vacuum-participant", entry, false);
}
catMan.addCategoryEntry(
CATEGORY_NAME,
"vacuumParticipant",
CONTRACT_ID,
false,
false
);
print("Check the test entry exists.");
}
/**
* Sends a fake idle-daily notification to the VACUUM Manager.
*/
function synthesize_idle_daily() {
let vm = Cc["@mozilla.org/storage/vacuum;1"].getService(Ci.nsIObserver);
vm.observe(null, "idle-daily", null);
Cc["@mozilla.org/storage/vacuum;1"]
.getService(Ci.nsIObserver)
.observe(null, "idle-daily", null);
}
/**
* Returns a new nsIFile reference for a profile database.
* @param filename for the database, excluded the .sqlite extension.
*/
function new_db_file(name) {
function new_db_file(name = "testVacuum") {
let file = Services.dirsvc.get("ProfD", Ci.nsIFile);
file.append(name + ".sqlite");
return file;
}
function run_test() {
do_test_pending();
function reset_vacuum_date(name = "testVacuum") {
let date = parseInt(Date.now() / 1000 - 31 * 86400);
// Set last VACUUM to a date in the past.
Services.prefs.setIntPref(`storage.vacuum.last.${name}.sqlite`, date);
return date;
}
// Change initial page size. Do it immediately since it would require an
// additional vacuum op to do it later. As a bonus this makes the page size
// change test really fast since it only has to check results.
let conn = getDatabase(new_db_file("testVacuum"));
conn.executeSimpleSQL("PRAGMA page_size = 1024");
print("Check current page size.");
function get_vacuum_date(name = "testVacuum") {
return Services.prefs.getIntPref(`storage.vacuum.last.${name}.sqlite`, 0);
}
add_setup(async function() {
// turn on Cu.isInAutomation
Services.prefs.setBoolPref(
"security.turn_off_all_security_so_that_viruses_can_take_over_this_computer",
true
);
});
add_task(async function test_common_vacuum() {
let last_vacuum_date = reset_vacuum_date();
info("Test that a VACUUM correctly happens and all notifications are fired.");
let promiseTestVacuumBegin = TestUtils.topicObserved("test-begin-vacuum");
let promiseTestVacuumEnd = TestUtils.topicObserved("test-end-vacuum-success");
let promiseVacuumBegin = TestUtils.topicObserved("vacuum-begin");
let promiseVacuumEnd = TestUtils.topicObserved("vacuum-end");
let participant = new VacuumParticipant(
Services.storage.openDatabase(new_db_file())
);
await participant.promiseRegistered();
synthesize_idle_daily();
// Wait for notifications.
await Promise.all([
promiseTestVacuumBegin,
promiseTestVacuumEnd,
promiseVacuumBegin,
promiseVacuumEnd,
]);
Assert.greater(get_vacuum_date(), last_vacuum_date);
await participant.dispose();
});
add_task(async function test_skipped_if_recent_vacuum() {
info("Test that a VACUUM is skipped if it was run recently.");
Services.prefs.setIntPref(
"storage.vacuum.last.testVacuum.sqlite",
parseInt(Date.now() / 1000)
);
// Wait for VACUUM skipped notification.
let promiseSkipped = TestUtils.topicObserved("vacuum-skip");
let participant = new VacuumParticipant(
Services.storage.openDatabase(new_db_file())
);
await participant.promiseRegistered();
synthesize_idle_daily();
// Check that VACUUM has been skipped.
await promiseSkipped;
await participant.dispose();
});
add_task(async function test_page_size_change() {
info("Test that a VACUUM changes page_size");
reset_vacuum_date();
let conn = Services.storage.openDatabase(new_db_file());
info("Check initial page size.");
let stmt = conn.createStatement("PRAGMA page_size");
try {
while (stmt.executeStep()) {
Assert.equal(stmt.row.page_size, 1024);
}
} finally {
Assert.ok(stmt.executeStep());
Assert.equal(stmt.row.page_size, conn.defaultPageSize);
stmt.finalize();
await populateFreeList(conn);
let participant = new VacuumParticipant(conn, { expectedPageSize: 1024 });
await participant.promiseRegistered();
let promiseVacuumEnd = TestUtils.topicObserved("test-end-vacuum-success");
synthesize_idle_daily();
await promiseVacuumEnd;
info("Check that page size was updated.");
stmt = conn.createStatement("PRAGMA page_size");
Assert.ok(stmt.executeStep());
Assert.equal(stmt.row.page_size, 1024);
stmt.finalize();
await participant.dispose();
});
add_task(async function test_skipped_optout_vacuum() {
info("Test that a VACUUM is skipped if the participant wants to opt-out.");
reset_vacuum_date();
let participant = new VacuumParticipant(
Services.storage.openDatabase(new_db_file()),
{ grant: false }
);
await participant.promiseRegistered();
// Wait for VACUUM skipped notification.
let promiseSkipped = TestUtils.topicObserved("vacuum-skip");
synthesize_idle_daily();
// Check that VACUUM has been skipped.
await promiseSkipped;
await participant.dispose();
});
add_task(async function test_memory_database_crash() {
info("Test that we don't crash trying to vacuum a memory database");
reset_vacuum_date();
let participant = new VacuumParticipant(
Services.storage.openSpecialDatabase("memory")
);
await participant.promiseRegistered();
// Wait for VACUUM skipped notification.
let promiseSkipped = TestUtils.topicObserved("vacuum-skip");
synthesize_idle_daily();
// Check that VACUUM has been skipped.
await promiseSkipped;
await participant.dispose();
});
add_task(async function test_async_connection() {
info("Test we can vacuum an async connection");
reset_vacuum_date();
let conn = await openAsyncDatabase(new_db_file());
await populateFreeList(conn);
let participant = new VacuumParticipant(conn);
await participant.promiseRegistered();
let promiseVacuumEnd = TestUtils.topicObserved("test-end-vacuum-success");
synthesize_idle_daily();
await promiseVacuumEnd;
await participant.dispose();
});
add_task(async function test_change_to_incremental_vacuum() {
info("Test we can change to incremental vacuum");
reset_vacuum_date();
let conn = Services.storage.openDatabase(new_db_file());
info("Check initial vacuum.");
let stmt = conn.createStatement("PRAGMA auto_vacuum");
Assert.ok(stmt.executeStep());
Assert.equal(stmt.row.auto_vacuum, 0);
stmt.finalize();
await populateFreeList(conn);
let participant = new VacuumParticipant(conn, { useIncrementalVacuum: true });
await participant.promiseRegistered();
let promiseVacuumEnd = TestUtils.topicObserved("test-end-vacuum-success");
synthesize_idle_daily();
await promiseVacuumEnd;
info("Check that auto_vacuum was updated.");
stmt = conn.createStatement("PRAGMA auto_vacuum");
Assert.ok(stmt.executeStep());
Assert.equal(stmt.row.auto_vacuum, 2);
stmt.finalize();
await participant.dispose();
});
add_task(async function test_change_from_incremental_vacuum() {
info("Test we can change from incremental vacuum");
reset_vacuum_date();
let conn = Services.storage.openDatabase(new_db_file());
conn.executeSimpleSQL("PRAGMA auto_vacuum = 2");
info("Check initial vacuum.");
let stmt = conn.createStatement("PRAGMA auto_vacuum");
Assert.ok(stmt.executeStep());
Assert.equal(stmt.row.auto_vacuum, 2);
stmt.finalize();
await populateFreeList(conn);
let participant = new VacuumParticipant(conn, {
useIncrementalVacuum: false,
});
await participant.promiseRegistered();
let promiseVacuumEnd = TestUtils.topicObserved("test-end-vacuum-success");
synthesize_idle_daily();
await promiseVacuumEnd;
info("Check that auto_vacuum was updated.");
stmt = conn.createStatement("PRAGMA auto_vacuum");
Assert.ok(stmt.executeStep());
Assert.equal(stmt.row.auto_vacuum, 0);
stmt.finalize();
await participant.dispose();
});
add_task(async function test_attached_vacuum() {
info("Test attached database is not a problem");
reset_vacuum_date();
let conn = Services.storage.openDatabase(new_db_file());
let conn2 = Services.storage.openDatabase(new_db_file("attached"));
info("Attach " + conn2.databaseFile.path);
conn.executeSimpleSQL(
`ATTACH DATABASE '${conn2.databaseFile.path}' AS attached`
);
await asyncClose(conn2);
let stmt = conn.createStatement("PRAGMA database_list");
let schemas = [];
while (stmt.executeStep()) {
schemas.push(stmt.row.name);
}
Assert.deepEqual(schemas, ["main", "attached"]);
stmt.finalize();
await populateFreeList(conn);
await populateFreeList(conn, "attached");
let participant = new VacuumParticipant(conn);
await participant.promiseRegistered();
let promiseVacuumEnd = TestUtils.topicObserved("test-end-vacuum-success");
synthesize_idle_daily();
await promiseVacuumEnd;
await participant.dispose();
});
add_task(async function test_vacuum_fail() {
info("Test a failed vacuum");
reset_vacuum_date();
let conn = Services.storage.openDatabase(new_db_file());
// Cannot vacuum in a transaction.
conn.beginTransaction();
await populateFreeList(conn);
let participant = new VacuumParticipant(conn);
await participant.promiseRegistered();
let promiseVacuumEnd = TestUtils.topicObserved("test-end-vacuum-failure");
synthesize_idle_daily();
await promiseVacuumEnd;
conn.commitTransaction();
await participant.dispose();
});
add_task(async function test_async_vacuum() {
// Since previous tests already go through most cases, this only checks
// the basics of directly calling asyncVacuum().
info("Test synchronous connection");
let conn = Services.storage.openDatabase(new_db_file());
await populateFreeList(conn);
let rv = await new Promise(resolve => {
conn.asyncVacuum(status => {
resolve(status);
});
});
Assert.ok(Components.isSuccessCode(rv));
await asyncClose(conn);
info("Test asynchronous connection");
conn = await openAsyncDatabase(new_db_file());
await populateFreeList(conn);
rv = await new Promise(resolve => {
conn.asyncVacuum(status => {
resolve(status);
});
});
Assert.ok(Components.isSuccessCode(rv));
await asyncClose(conn);
});
// Chunked growth is disabled on Android, so this test is pointless there.
add_task(
{ skip_if: () => AppConstants.platform == "android" },
async function test_vacuum_growth() {
// Tests vacuum doesn't nullify chunked growth.
let conn = Services.storage.openDatabase(new_db_file("incremental"));
conn.executeSimpleSQL("PRAGMA auto_vacuum = INCREMENTAL");
conn.setGrowthIncrement(2 * conn.defaultPageSize, "");
await populateFreeList(conn);
let stmt = conn.createStatement("PRAGMA freelist_count");
let count = 0;
Assert.ok(stmt.executeStep());
count = stmt.row.freelist_count;
stmt.reset();
Assert.greater(count, 2, "There's more than 2 page in freelist");
let rv = await new Promise(resolve => {
conn.asyncVacuum(status => {
resolve(status);
}, true);
});
Assert.ok(Components.isSuccessCode(rv));
Assert.ok(stmt.executeStep());
Assert.equal(
stmt.row.freelist_count,
2,
"chunked growth space was preserved"
);
stmt.reset();
// A full vacuuum should not be executed if there's less free pages than
// chunked growth.
rv = await new Promise(resolve => {
conn.asyncVacuum(status => {
resolve(status);
});
});
Assert.ok(Components.isSuccessCode(rv));
Assert.ok(stmt.executeStep());
Assert.equal(
stmt.row.freelist_count,
2,
"chunked growth space was preserved"
);
stmt.finalize();
await asyncClose(conn);
}
);
load_test_vacuum_component();
run_next_test();
}
const TESTS = [
function test_common_vacuum() {
print(
"\n*** Test that a VACUUM correctly happens and all notifications are fired."
);
// Wait for VACUUM begin.
let beginVacuumReceived = false;
Services.obs.addObserver(function onVacuum(aSubject, aTopic, aData) {
Services.obs.removeObserver(onVacuum, aTopic);
beginVacuumReceived = true;
}, "test-begin-vacuum");
// Wait for heavy IO notifications.
let heavyIOTaskBeginReceived = false;
let heavyIOTaskEndReceived = false;
Services.obs.addObserver(function onVacuum(aSubject, aTopic, aData) {
if (heavyIOTaskBeginReceived && heavyIOTaskEndReceived) {
Services.obs.removeObserver(onVacuum, aTopic);
}
if (aData == "vacuum-begin") {
heavyIOTaskBeginReceived = true;
} else if (aData == "vacuum-end") {
heavyIOTaskEndReceived = true;
}
}, "heavy-io-task");
// Wait for VACUUM end.
Services.obs.addObserver(function onVacuum(aSubject, aTopic, aData) {
Services.obs.removeObserver(onVacuum, aTopic);
print("Check we received onBeginVacuum");
Assert.ok(beginVacuumReceived);
print("Check we received heavy-io-task notifications");
Assert.ok(heavyIOTaskBeginReceived);
Assert.ok(heavyIOTaskEndReceived);
print("Received onEndVacuum");
run_next_test();
}, "test-end-vacuum");
synthesize_idle_daily();
},
function test_skipped_if_recent_vacuum() {
print("\n*** Test that a VACUUM is skipped if it was run recently.");
Services.prefs.setIntPref(
"storage.vacuum.last.testVacuum.sqlite",
parseInt(Date.now() / 1000)
);
// Wait for VACUUM begin.
let vacuumObserver = {
gotNotification: false,
observe: function VO_observe(aSubject, aTopic, aData) {
this.gotNotification = true;
},
QueryInterface: ChromeUtils.generateQI(["nsIObserver"]),
};
Services.obs.addObserver(vacuumObserver, "test-begin-vacuum");
// Check after a couple seconds that no VACUUM has been run.
do_timeout(2000, function() {
print("Check VACUUM did not run.");
Assert.ok(!vacuumObserver.gotNotification);
Services.obs.removeObserver(vacuumObserver, "test-begin-vacuum");
run_next_test();
});
synthesize_idle_daily();
},
function test_page_size_change() {
print("\n*** Test that a VACUUM changes page_size");
// We did setup the database with a small page size, the previous vacuum
// should have updated it.
print("Check that page size was updated.");
let conn = getDatabase(new_db_file("testVacuum"));
let stmt = conn.createStatement("PRAGMA page_size");
try {
while (stmt.executeStep()) {
Assert.equal(stmt.row.page_size, conn.defaultPageSize);
}
} finally {
stmt.finalize();
}
run_next_test();
},
function test_skipped_optout_vacuum() {
print(
"\n*** Test that a VACUUM is skipped if the participant wants to opt-out."
);
Services.obs.notifyObservers(null, "test-options", "opt-out");
// Wait for VACUUM begin.
let vacuumObserver = {
gotNotification: false,
observe: function VO_observe(aSubject, aTopic, aData) {
this.gotNotification = true;
},
QueryInterface: ChromeUtils.generateQI(["nsIObserver"]),
};
Services.obs.addObserver(vacuumObserver, "test-begin-vacuum");
// Check after a couple seconds that no VACUUM has been run.
do_timeout(2000, function() {
print("Check VACUUM did not run.");
Assert.ok(!vacuumObserver.gotNotification);
Services.obs.removeObserver(vacuumObserver, "test-begin-vacuum");
run_next_test();
});
synthesize_idle_daily();
},
/* Changing page size on WAL is not supported till Bug 634374 is properly fixed.
function test_page_size_change_with_wal()
{
print("\n*** Test that a VACUUM changes page_size with WAL mode");
Services.obs.notifyObservers(null, "test-options", "wal");
// Set a small page size.
let conn = getDatabase(new_db_file("testVacuum2"));
conn.executeSimpleSQL("PRAGMA page_size = 1024");
let stmt = conn.createStatement("PRAGMA page_size");
try {
while (stmt.executeStep()) {
do_check_eq(stmt.row.page_size, 1024);
}
}
finally {
stmt.finalize();
}
// Use WAL journal mode.
conn.executeSimpleSQL("PRAGMA journal_mode = WAL");
stmt = conn.createStatement("PRAGMA journal_mode");
try {
while (stmt.executeStep()) {
do_check_eq(stmt.row.journal_mode, "wal");
}
}
finally {
stmt.finalize();
}
// Wait for VACUUM end.
let vacuumObserver = {
observe: function VO_observe(aSubject, aTopic, aData) {
Services.obs.removeObserver(this, aTopic);
print("Check page size has been updated.");
let stmt = conn.createStatement("PRAGMA page_size");
try {
while (stmt.executeStep()) {
do_check_eq(stmt.row.page_size, Ci.mozIStorageConnection.DEFAULT_PAGE_SIZE);
}
}
finally {
stmt.finalize();
}
print("Check journal mode has been restored.");
stmt = conn.createStatement("PRAGMA journal_mode");
try {
while (stmt.executeStep()) {
do_check_eq(stmt.row.journal_mode, "wal");
}
}
finally {
stmt.finalize();
}
run_next_test();
},
QueryInterface: ChromeUtils.generateQI([Ci.nsIObserver])
}
Services.obs.addObserver(vacuumObserver, "test-end-vacuum", false);
synthesize_idle_daily();
},
*/
function test_memory_database_crash() {
print("\n*** Test that we don't crash trying to vacuum a memory database");
Services.obs.notifyObservers(null, "test-options", "memory");
// Wait for VACUUM begin.
let vacuumObserver = {
gotNotification: false,
observe: function VO_observe(aSubject, aTopic, aData) {
this.gotNotification = true;
},
QueryInterface: ChromeUtils.generateQI(["nsIObserver"]),
};
Services.obs.addObserver(vacuumObserver, "test-begin-vacuum");
// Check after a couple seconds that no VACUUM has been run.
do_timeout(2000, function() {
print("Check VACUUM did not run.");
Assert.ok(!vacuumObserver.gotNotification);
Services.obs.removeObserver(vacuumObserver, "test-begin-vacuum");
run_next_test();
});
synthesize_idle_daily();
},
/* Changing page size on WAL is not supported till Bug 634374 is properly fixed.
function test_wal_restore_fail()
{
print("\n*** Test that a failing WAL restoration notifies failure");
Services.obs.notifyObservers(null, "test-options", "wal-fail");
// Wait for VACUUM end.
let vacuumObserver = {
observe: function VO_observe(aSubject, aTopic, aData) {
Services.obs.removeObserver(vacuumObserver, "test-end-vacuum");
print("Check WAL restoration failed.");
do_check_false(aData);
run_next_test();
},
QueryInterface: ChromeUtils.generateQI([Ci.nsIObserver])
}
Services.obs.addObserver(vacuumObserver, "test-end-vacuum", false);
synthesize_idle_daily();
},
*/
];
function run_next_test() {
if (!TESTS.length) {
Services.obs.notifyObservers(null, "test-options", "dispose");
do_test_finished();
} else {
// Set last VACUUM to a date in the past.
Services.prefs.setIntPref(
"storage.vacuum.last.testVacuum.sqlite",
parseInt(Date.now() / 1000 - 31 * 86400)
);
executeSoon(TESTS.shift());
}
async function populateFreeList(conn, schema = "main") {
await executeSimpleSQLAsync(conn, `CREATE TABLE ${schema}.test (id TEXT)`);
await executeSimpleSQLAsync(
conn,
`INSERT INTO ${schema}.test
VALUES ${Array.from({ length: 3000 }, () => Math.random()).map(
v => "('" + v + "')"
)}`
);
await executeSimpleSQLAsync(conn, `DROP TABLE ${schema}.test`);
}

7
testing/modules/MockRegistrar.sys.mjs
Просмотреть файл

@ -90,13 +90,6 @@ export var MockRegistrar = Object.freeze({
return cid;
},
registerESM(contractID, esmPath, symbol) {
return this.register(contractID, () => {
let exports = ChromeUtils.importESModule(esmPath);
return new exports[symbol]();
});
},
/**
* Unregister the mock.
*

14
toolkit/components/places/nsNavHistory.cpp
Просмотреть файл

@ -1825,8 +1825,18 @@ nsNavHistory::SetShouldStartFrecencyRecalculation(bool aVal) {
//// mozIStorageVacuumParticipant
NS_IMETHODIMP
nsNavHistory::GetDatabaseConnection(mozIStorageConnection** _DBConnection) {
return GetDBConnection(_DBConnection);
nsNavHistory::GetDatabaseConnection(
mozIStorageAsyncConnection** _DBConnection) {
NS_ENSURE_ARG_POINTER(_DBConnection);
nsCOMPtr<mozIStorageAsyncConnection> connection = mDB->MainConn();
connection.forget(_DBConnection);
return NS_OK;
}
NS_IMETHODIMP
nsNavHistory::GetUseIncrementalVacuum(bool* _useIncremental) {
*_useIncremental = false;
return NS_OK;
}
NS_IMETHODIMP