From fff8403f8a62c970e6d94237c7c64ba73fec54e0 Mon Sep 17 00:00:00 2001 From: Gijs Kruitbosch Date: Mon, 18 Jul 2016 16:46:45 +0100 Subject: [PATCH] Bug 1285041 - ignore locking when trying to read chrome DB file, r=mak MozReview-Commit-ID: 89f0YCxxgC8 --HG-- extra : rebase_source : be3ee58b00b4c8ae6ec41db9459f50abdce816f6 --- .../migration/ChromeProfileMigrator.js | 52 +++++++++++++-- storage/mozStorageConnection.cpp | 16 ++++- storage/mozStorageConnection.h | 15 ++++- storage/mozStorageService.cpp | 65 +++++++++++++------ toolkit/modules/Sqlite.jsm | 24 ++++++- 5 files changed, 139 insertions(+), 33 deletions(-) diff --git a/browser/components/migration/ChromeProfileMigrator.js b/browser/components/migration/ChromeProfileMigrator.js index aa8fea0dc1ad..71d0b0f667c9 100644 --- a/browser/components/migration/ChromeProfileMigrator.js +++ b/browser/components/migration/ChromeProfileMigrator.js @@ -311,19 +311,59 @@ function GetHistoryResource(aProfileFolder) { if (!historyFile.exists()) return null; + function getRows(dbOptions) { + const RETRYLIMIT = 10; + const RETRYINTERVAL = 100; + return Task.spawn(function* innerGetRows() { + let rows = null; + for (let retryCount = RETRYLIMIT; retryCount && !rows; retryCount--) { + // Attempt to get the rows. If this succeeds, we will bail out of the loop, + // close the database in a failsafe way, and pass the rows back. + // If fetching the rows throws, we will wait RETRYINTERVAL ms + // and try again. This will repeat a maximum of RETRYLIMIT times. + let db; + let didOpen = false; + let exceptionSeen; + try { + db = yield Sqlite.openConnection(dbOptions); + didOpen = true; + rows = yield db.execute(`SELECT url, title, last_visit_time, typed_count + FROM urls WHERE hidden = 0`); + } catch (ex) { + if (!exceptionSeen) { + Cu.reportError(ex); + } + exceptionSeen = ex; + } finally { + try { + if (didOpen) { + yield db.close(); + } + } catch (ex) {} + } + if (exceptionSeen) { + yield new Promise(resolve => setTimeout(resolve, RETRYINTERVAL)); + } + } + if (!rows) { + throw new Error("Couldn't get rows from the Chrome history database."); + } + return rows; + }); + } + return { type: MigrationUtils.resourceTypes.HISTORY, migrate(aCallback) { Task.spawn(function* () { - let db = yield Sqlite.openConnection({ + let dbOptions = { + readOnly: true, + ignoreLockingMode: true, path: historyFile.path - }); - - let rows = yield db.execute(`SELECT url, title, last_visit_time, typed_count - FROM urls WHERE hidden = 0`); - yield db.close(); + }; + let rows = yield getRows(dbOptions); let places = []; for (let row of rows) { try { diff --git a/storage/mozStorageConnection.cpp b/storage/mozStorageConnection.cpp index 00e0f0bb8147..0da908a429bd 100644 --- a/storage/mozStorageConnection.cpp +++ b/storage/mozStorageConnection.cpp @@ -472,7 +472,8 @@ private: Connection::Connection(Service *aService, int aFlags, - bool aAsyncOnly) + bool aAsyncOnly, + bool aIgnoreLockingMode) : sharedAsyncExecutionMutex("Connection::sharedAsyncExecutionMutex") , sharedDBMutex("Connection::sharedDBMutex") , threadOpenedOn(do_GetCurrentThread()) @@ -485,9 +486,12 @@ Connection::Connection(Service *aService, , mTransactionInProgress(false) , mProgressHandler(nullptr) , mFlags(aFlags) +, mIgnoreLockingMode(aIgnoreLockingMode) , mStorageService(aService) , mAsyncOnly(aAsyncOnly) { + MOZ_ASSERT(!mIgnoreLockingMode || mFlags & SQLITE_OPEN_READONLY, + "Can't ignore locking for a non-readonly connection!"); mStorageService->registerConnection(this); } @@ -577,6 +581,7 @@ nsresult Connection::initialize() { NS_ASSERTION (!mDBConn, "Initialize called on already opened database!"); + MOZ_ASSERT(!mIgnoreLockingMode, "Can't ignore locking on an in-memory db."); PROFILER_LABEL("mozStorageConnection", "initialize", js::ProfileEntry::Category::STORAGE); @@ -610,8 +615,15 @@ Connection::initialize(nsIFile *aDatabaseFile) nsresult rv = aDatabaseFile->GetPath(path); NS_ENSURE_SUCCESS(rv, rv); +#ifdef XP_WIN + static const char* sIgnoreLockingVFS = "win32-none"; +#else + static const char* sIgnoreLockingVFS = "unix-none"; +#endif + const char* vfs = mIgnoreLockingMode ? sIgnoreLockingVFS : nullptr; + int srv = ::sqlite3_open_v2(NS_ConvertUTF16toUTF8(path).get(), &mDBConn, - mFlags, nullptr); + mFlags, vfs); if (srv != SQLITE_OK) { mDBConn = nullptr; return convertResultCode(srv); diff --git a/storage/mozStorageConnection.h b/storage/mozStorageConnection.h index bb1b6dd4cf8e..979ac6436e8d 100644 --- a/storage/mozStorageConnection.h +++ b/storage/mozStorageConnection.h @@ -69,8 +69,16 @@ public: * - |mozIStorageAsyncConnection|; * If |false|, the result also implements synchronous interface: * - |mozIStorageConnection|. + * @param aIgnoreLockingMode + * If |true|, ignore locks in force on the file. Only usable with + * read-only connections. Defaults to false. + * Use with extreme caution. If sqlite ignores locks, reads may fail + * indicating database corruption (the database won't actually be + * corrupt) or produce wrong results without any indication that has + * happened. */ - Connection(Service *aService, int aFlags, bool aAsyncOnly); + Connection(Service *aService, int aFlags, bool aAsyncOnly, + bool aIgnoreLockingMode = false); /** * Creates the connection to an in-memory database. @@ -356,6 +364,11 @@ private: */ const int mFlags; + /** + * Stores whether we should ask sqlite3_open_v2 to ignore locking. + */ + const bool mIgnoreLockingMode; + // This is here for two reasons: 1) It's used to make sure that the // connections do not outlive the service. 2) Our custom collating functions // call its localeCompareStrings() method. diff --git a/storage/mozStorageService.cpp b/storage/mozStorageService.cpp index 1d8f9905b924..5536d5b254dd 100644 --- a/storage/mozStorageService.cpp +++ b/storage/mozStorageService.cpp @@ -748,11 +748,43 @@ Service::OpenAsyncDatabase(nsIVariant *aDatabaseStore, NS_ENSURE_ARG(aDatabaseStore); NS_ENSURE_ARG(aCallback); - nsCOMPtr storageFile; - int flags = SQLITE_OPEN_READWRITE; + nsresult rv; + bool shared = false; + bool readOnly = false; + bool ignoreLockingMode = false; + int32_t growthIncrement = -1; +#define FAIL_IF_SET_BUT_INVALID(rv)\ + if (NS_FAILED(rv) && rv != NS_ERROR_NOT_AVAILABLE) { \ + return NS_ERROR_INVALID_ARG; \ + } + + // Deal with options first: + if (aOptions) { + rv = aOptions->GetPropertyAsBool(NS_LITERAL_STRING("readOnly"), &readOnly); + FAIL_IF_SET_BUT_INVALID(rv); + + rv = aOptions->GetPropertyAsBool(NS_LITERAL_STRING("ignoreLockingMode"), + &ignoreLockingMode); + FAIL_IF_SET_BUT_INVALID(rv); + // Specifying ignoreLockingMode will force use of the readOnly flag: + if (ignoreLockingMode) { + readOnly = true; + } + + rv = aOptions->GetPropertyAsBool(NS_LITERAL_STRING("shared"), &shared); + FAIL_IF_SET_BUT_INVALID(rv); + + // NB: we re-set to -1 if we don't have a storage file later on. + rv = aOptions->GetPropertyAsInt32(NS_LITERAL_STRING("growthIncrement"), + &growthIncrement); + FAIL_IF_SET_BUT_INVALID(rv); + } + int flags = readOnly ? SQLITE_OPEN_READONLY : SQLITE_OPEN_READWRITE; + + nsCOMPtr storageFile; nsCOMPtr dbStore; - nsresult rv = aDatabaseStore->GetAsISupports(getter_AddRefs(dbStore)); + rv = aDatabaseStore->GetAsISupports(getter_AddRefs(dbStore)); if (NS_SUCCEEDED(rv)) { // Generally, aDatabaseStore holds the database nsIFile. storageFile = do_QueryInterface(dbStore, &rv); @@ -763,17 +795,12 @@ Service::OpenAsyncDatabase(nsIVariant *aDatabaseStore, rv = storageFile->Clone(getter_AddRefs(storageFile)); MOZ_ASSERT(NS_SUCCEEDED(rv)); - // Ensure that SQLITE_OPEN_CREATE is passed in for compatibility reasons. - flags |= SQLITE_OPEN_CREATE; - - // Extract and apply the shared-cache option. - bool shared = false; - if (aOptions) { - rv = aOptions->GetPropertyAsBool(NS_LITERAL_STRING("shared"), &shared); - if (NS_FAILED(rv) && rv != NS_ERROR_NOT_AVAILABLE) { - return NS_ERROR_INVALID_ARG; - } + if (!readOnly) { + // Ensure that SQLITE_OPEN_CREATE is passed in for compatibility reasons. + flags |= SQLITE_OPEN_CREATE; } + + // Apply the shared-cache option. flags |= shared ? SQLITE_OPEN_SHAREDCACHE : SQLITE_OPEN_PRIVATECACHE; } else { // Sometimes, however, it's a special database name. @@ -787,17 +814,13 @@ Service::OpenAsyncDatabase(nsIVariant *aDatabaseStore, // connection to use a memory DB. } - int32_t growthIncrement = -1; - if (aOptions && storageFile) { - rv = aOptions->GetPropertyAsInt32(NS_LITERAL_STRING("growthIncrement"), - &growthIncrement); - if (NS_FAILED(rv) && rv != NS_ERROR_NOT_AVAILABLE) { - return NS_ERROR_INVALID_ARG; - } + if (!storageFile && growthIncrement >= 0) { + return NS_ERROR_INVALID_ARG; } // Create connection on this thread, but initialize it on its helper thread. - RefPtr msc = new Connection(this, flags, true); + RefPtr msc = new Connection(this, flags, true, + ignoreLockingMode); nsCOMPtr target = msc->getAsyncExecutionTarget(); MOZ_ASSERT(target, "Cannot initialize a connection that has been closed already"); diff --git a/toolkit/modules/Sqlite.jsm b/toolkit/modules/Sqlite.jsm index f024ec8bd195..e8d986c0e501 100644 --- a/toolkit/modules/Sqlite.jsm +++ b/toolkit/modules/Sqlite.jsm @@ -868,6 +868,15 @@ ConnectionData.prototype = Object.freeze({ * *not* a timer on the idle service and this could fire while the * application is active. * + * readOnly -- (bool) Whether to open the database with SQLITE_OPEN_READONLY + * set. If used, writing to the database will fail. Defaults to false. + * + * ignoreLockingMode -- (bool) Whether to ignore locks on the database held + * by other connections. If used, implies readOnly. Defaults to false. + * USE WITH EXTREME CAUTION. This mode WILL produce incorrect results or + * return "false positive" corruption errors if other connections write + * to the DB at the same time. + * * FUTURE options to control: * * special named databases @@ -915,12 +924,21 @@ function openConnection(options) { log.info("Opening database: " + path + " (" + identifier + ")"); return new Promise((resolve, reject) => { - let dbOptions = null; + let dbOptions = Cc["@mozilla.org/hash-property-bag;1"]. + createInstance(Ci.nsIWritablePropertyBag); if (!sharedMemoryCache) { - dbOptions = Cc["@mozilla.org/hash-property-bag;1"]. - createInstance(Ci.nsIWritablePropertyBag); dbOptions.setProperty("shared", false); } + if (options.readOnly) { + dbOptions.setProperty("readOnly", true); + } + if (options.ignoreLockingMode) { + dbOptions.setProperty("ignoreLockingMode", true); + dbOptions.setProperty("readOnly", true); + } + + dbOptions = dbOptions.enumerator.hasMoreElements() ? dbOptions : null; + Services.storage.openAsyncDatabase(file, dbOptions, (status, connection) => { if (!connection) { log.warn(`Could not open connection to ${path}: ${status}`);