Bug 866846 - Use WAL journal mode for IndexedDB databases, r=janv.

--HG--
extra : rebase_source : 6bd130ec0bdbeacde6b725ee16736083775f8ca7

dom/indexedDB/ActorsParent.cpp

@@ -87,13 +87,6 @@
 #include "snappy/snappy.h"
 #include "TransactionThreadPool.h"
 
-namespace mozilla {
-namespace dom {
-namespace indexedDB {
-
-using namespace mozilla::dom::quota;
-using namespace mozilla::ipc;
-
 #define DISABLE_ASSERTS_FOR_FUZZING 0
 
 #if DISABLE_ASSERTS_FOR_FUZZING
@@ -102,6 +95,17 @@ using namespace mozilla::ipc;
 #define ASSERT_UNLESS_FUZZING(...) MOZ_ASSERT(false, __VA_ARGS__)
 #endif
 
+#if defined(MOZ_WIDGET_ANDROID) || defined(MOZ_WIDGET_GONK)
+#define IDB_MOBILE
+#endif
+
+namespace mozilla {
+namespace dom {
+namespace indexedDB {
+
+using namespace mozilla::dom::quota;
+using namespace mozilla::ipc;
+
 namespace {
 
 class Cursor;
@@ -144,6 +148,34 @@ const int32_t kSQLiteSchemaVersion =
 
 const int32_t kStorageProgressGranularity = 1000;
 
+// Changing the value here will override the page size of new databases only.
+// A journal mode change and VACUUM are needed to change existing databases, so
+// the best way to do that is to use the schema version upgrade mechanism.
+const uint32_t kSQLitePageSizeOverride =
+#ifdef IDB_MOBILE
+  2048;
+#else
+  4096;
+#endif
+
+static_assert(kSQLitePageSizeOverride == /* mozStorage default */ 0 ||
+              (kSQLitePageSizeOverride % 2 == 0 &&
+               kSQLitePageSizeOverride >= 512  &&
+               kSQLitePageSizeOverride <= 65536),
+              "Must be 0 (disabled) or a power of 2 between 512 and 65536!");
+
+// Set to -1 to use SQLite's default, 0 to disable, or some positive number to
+// enforce a custom limit.
+const int32_t kMaxWALPages = 5000; // 20MB on desktop, 10MB on mobile.
+
+// Set to some multiple of the page size to grow the database in larger chunks.
+const uint32_t kSQLiteGrowthIncrement = kSQLitePageSizeOverride * 2;
+
+static_assert(kSQLiteGrowthIncrement >= 0 &&
+              kSQLiteGrowthIncrement % kSQLitePageSizeOverride == 0 &&
+              kSQLiteGrowthIncrement < uint32_t(INT32_MAX),
+              "Must be 0 (disabled) or a positive multiple of the page size!");
+
 const char kSavepointClause[] = "SAVEPOINT sp;";
 
 const uint32_t kFileCopyBufferSize = 32768;
@@ -151,6 +183,9 @@ const uint32_t kFileCopyBufferSize = 32768;
 const char kJournalDirectoryName[] = "journals";
 
 const char kFileManagerDirectoryNameSuffix[] = ".files";
+const char kSQLiteJournalSuffix[] = ".sqlite-journal";
+const char kSQLiteSHMSuffix[] = ".sqlite-shm";
+const char kSQLiteWALSuffix[] = ".sqlite-wal";
 
 const char kPrefIndexedDBEnabled[] = "dom.indexedDB.enabled";
 
@@ -178,6 +213,16 @@ const uint32_t kDEBUGThreadSleepMS = 0;
 
 #endif
 
+template <size_t N>
+MOZ_CONSTEXPR size_t
+LiteralStringLength(const char (&aArr)[N])
+{
+  static_assert(N, "Zero-length string literal?!");
+
+  // Don't include the null terminator.
+  return ArrayLength<const char, N>(aArr) - 1;
+}
+
 /*******************************************************************************
  * Metadata classes
  ******************************************************************************/
@@ -2063,7 +2108,7 @@ UpgradeSchemaFrom12_0To13_0(mozIStorageConnection* aConnection,
 
   nsresult rv;
 
-#if defined(MOZ_WIDGET_ANDROID) || defined(MOZ_WIDGET_GONK)
+#ifdef IDB_MOBILE
   int32_t defaultPageSize;
   rv = aConnection->GetDefaultPageSize(&defaultPageSize);
   if (NS_WARN_IF(NS_FAILED(rv))) {
@@ -2181,48 +2226,230 @@ GetDatabaseFileURL(nsIFile* aDatabaseFile,
 nsresult
 SetDefaultPragmas(mozIStorageConnection* aConnection)
 {
+  MOZ_ASSERT(!NS_IsMainThread());
   MOZ_ASSERT(aConnection);
 
-  static const char query[] =
+  static const char kBuiltInPragmas[] =
-#if defined(MOZ_WIDGET_ANDROID) || defined(MOZ_WIDGET_GONK)
-    // Switch the journaling mode to TRUNCATE to avoid changing the directory
-    // structure at the conclusion of every transaction for devices with slower
-    // file systems.
-    "PRAGMA journal_mode = TRUNCATE; "
-#endif
     // We use foreign keys in lots of places.
-    "PRAGMA foreign_keys = ON; "
+    "PRAGMA foreign_keys = ON;"
+
     // The "INSERT OR REPLACE" statement doesn't fire the update trigger,
     // instead it fires only the insert trigger. This confuses the update
     // refcount function. This behavior changes with enabled recursive triggers,
     // so the statement fires the delete trigger first and then the insert
     // trigger.
     "PRAGMA recursive_triggers = ON;"
-    // We don't need SQLite's table locks because we manage transaction ordering
+
-    // ourselves and we know we will never allow a write transaction to modify
+    // We aggressively truncate the database file when idle so don't bother
-    // an object store that a read transaction is in the process of using.
+    // overwriting the WAL with 0 during active periods.
-    "PRAGMA read_uncommitted = TRUE;"
+    "PRAGMA secure_delete = OFF;"
-    // No more PRAGMAs.
   ;
 
-  nsresult rv = aConnection->ExecuteSimpleSQL(NS_LITERAL_CSTRING(query));
+  nsresult rv =
+    aConnection->ExecuteSimpleSQL(
+      nsDependentCString(kBuiltInPragmas,
+                         LiteralStringLength(kBuiltInPragmas)));
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
+  nsAutoCString pragmaStmt;
+  pragmaStmt.AssignLiteral("PRAGMA synchronous = ");
+
   if (IndexedDatabaseManager::FullSynchronous()) {
-    rv = aConnection->ExecuteSimpleSQL(
+    pragmaStmt.AppendLiteral("FULL");
-                             NS_LITERAL_CSTRING("PRAGMA synchronous = FULL;"));
+  } else {
+    pragmaStmt.AppendLiteral("NORMAL");
+  }
+  pragmaStmt.Append(';');
+
+  rv = aConnection->ExecuteSimpleSQL(pragmaStmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+#ifndef IDB_MOBILE
+  if (kSQLiteGrowthIncrement) {
+    rv = aConnection->SetGrowthIncrement(kSQLiteGrowthIncrement,
+                                         EmptyCString());
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
   }
+#endif // IDB_MOBILE
 
   return NS_OK;
 }
 
 nsresult
-CreateDatabaseConnection(nsIFile* aDBFile,
+SetJournalMode(mozIStorageConnection* aConnection)
+{
+  MOZ_ASSERT(!NS_IsMainThread());
+  MOZ_ASSERT(aConnection);
+
+  // Try enabling WAL mode. This can fail in various circumstances so we have to
+  // check the results here.
+  NS_NAMED_LITERAL_CSTRING(journalModeQueryStart, "PRAGMA journal_mode = ");
+  NS_NAMED_LITERAL_CSTRING(journalModeWAL, "wal");
+
+  nsCOMPtr<mozIStorageStatement> stmt;
+  nsresult rv =
+    aConnection->CreateStatement(journalModeQueryStart + journalModeWAL,
+                                 getter_AddRefs(stmt));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  bool hasResult;
+  rv = stmt->ExecuteStep(&hasResult);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  MOZ_ASSERT(hasResult);
+
+  nsCString journalMode;
+  rv = stmt->GetUTF8String(0, journalMode);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  if (journalMode.Equals(journalModeWAL)) {
+    // WAL mode successfully enabled. Maybe set limits on its size here.
+    if (kMaxWALPages >= 0) {
+      nsAutoCString pageCount;
+      pageCount.AppendInt(kMaxWALPages);
+
+      rv = aConnection->ExecuteSimpleSQL(
+        NS_LITERAL_CSTRING("PRAGMA wal_autocheckpoint = ") + pageCount);
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+    }
+  } else {
+    NS_WARNING("Failed to set WAL mode, falling back to normal journal mode.");
+#ifdef IDB_MOBILE
+    rv = aConnection->ExecuteSimpleSQL(journalModeQueryStart +
+                                       NS_LITERAL_CSTRING("truncate"));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+#endif
+  }
+
+  return NS_OK;
+}
+
+template <class FileOrURLType>
+struct StorageOpenTraits;
+
+template <>
+struct StorageOpenTraits<nsIFileURL*>
+{
+  static nsresult
+  Open(mozIStorageService* aStorageService,
+       nsIFileURL* aFileURL,
+       mozIStorageConnection** aConnection)
+  {
+    return aStorageService->OpenDatabaseWithFileURL(aFileURL, aConnection);
+  }
+
+#ifdef DEBUG
+  static void
+  GetPath(nsIFileURL* aFileURL, nsCString& aPath)
+  {
+    MOZ_ALWAYS_TRUE(NS_SUCCEEDED(aFileURL->GetFileName(aPath)));
+  }
+#endif
+};
+
+template <>
+struct StorageOpenTraits<nsIFile*>
+{
+  static nsresult
+  Open(mozIStorageService* aStorageService,
+       nsIFile* aFile,
+       mozIStorageConnection** aConnection)
+  {
+    return aStorageService->OpenUnsharedDatabase(aFile, aConnection);
+  }
+
+#ifdef DEBUG
+  static void
+  GetPath(nsIFile* aFile, nsCString& aPath)
+  {
+    nsString path;
+    MOZ_ALWAYS_TRUE(NS_SUCCEEDED(aFile->GetPath(path)));
+
+    aPath.AssignWithConversion(path);
+  }
+#endif
+};
+
+template <template <class> class SmartPtr, class FileOrURLType>
+struct StorageOpenTraits<SmartPtr<FileOrURLType>>
+  : public StorageOpenTraits<FileOrURLType*>
+{ };
+
+template <class FileOrURLType>
+nsresult
+OpenDatabaseAndHandleBusy(mozIStorageService* aStorageService,
+                          FileOrURLType aFileOrURL,
+                          mozIStorageConnection** aConnection)
+{
+  MOZ_ASSERT(!NS_IsMainThread());
+  MOZ_ASSERT(!IsOnBackgroundThread());
+  MOZ_ASSERT(aStorageService);
+  MOZ_ASSERT(aFileOrURL);
+  MOZ_ASSERT(aConnection);
+
+  nsCOMPtr<mozIStorageConnection> connection;
+  nsresult rv =
+    StorageOpenTraits<FileOrURLType>::Open(aStorageService,
+                                           aFileOrURL,
+                                           getter_AddRefs(connection));
+
+  if (rv == NS_ERROR_STORAGE_BUSY) {
+#ifdef DEBUG
+    {
+      nsCString path;
+      StorageOpenTraits<FileOrURLType>::GetPath(aFileOrURL, path);
+
+      nsPrintfCString message("Received NS_ERROR_STORAGE_BUSY when attempting "
+                              "to open database '%s', retrying for up to 10 "
+                              "seconds",
+                              path.get());
+      NS_WARNING(message.get());
+    }
+#endif
+
+    // Another thread must be checkpointing the WAL. Wait up to 10 seconds for
+    // that to complete.
+    TimeStamp start = TimeStamp::NowLoRes();
+
+    while (true) {
+      PR_Sleep(PR_MillisecondsToInterval(100));
+
+      rv = StorageOpenTraits<FileOrURLType>::Open(aStorageService,
+                                                  aFileOrURL,
+                                                  getter_AddRefs(connection));
+      if (rv != NS_ERROR_STORAGE_BUSY ||
+          TimeStamp::NowLoRes() - start > TimeDuration::FromSeconds(10)) {
+        break;
+      }
+    }
+  }
+
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  connection.forget(aConnection);
+  return NS_OK;
+}
+
+nsresult
+CreateStorageConnection(nsIFile* aDBFile,
                         nsIFile* aFMDirectory,
                         const nsAString& aName,
                         PersistenceType aPersistenceType,
@@ -2236,7 +2463,7 @@ CreateDatabaseConnection(nsIFile* aDBFile,
   MOZ_ASSERT(aConnection);
 
   PROFILER_LABEL("IndexedDB",
-                 "CreateDatabaseConnection",
+                 "CreateStorageConnection",
                  js::ProfileEntry::Category::STORAGE);
 
   nsresult rv;
@@ -2268,7 +2495,7 @@ CreateDatabaseConnection(nsIFile* aDBFile,
   }
 
   nsCOMPtr<mozIStorageConnection> connection;
-  rv = ss->OpenDatabaseWithFileURL(dbFileUrl, getter_AddRefs(connection));
+  rv = OpenDatabaseAndHandleBusy(ss, dbFileUrl, getter_AddRefs(connection));
   if (rv == NS_ERROR_FILE_CORRUPTED) {
     // If we're just opening the database during origin initialization, then
     // we don't want to erase any files. The failure here will fail origin
@@ -2305,8 +2532,9 @@ CreateDatabaseConnection(nsIFile* aDBFile,
       }
     }
 
-    rv = ss->OpenDatabaseWithFileURL(dbFileUrl, getter_AddRefs(connection));
+    rv = OpenDatabaseAndHandleBusy(ss, dbFileUrl, getter_AddRefs(connection));
   }
+
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -2339,11 +2567,13 @@ CreateDatabaseConnection(nsIFile* aDBFile,
     return NS_ERROR_DOM_INDEXEDDB_UNKNOWN_ERR;
   }
 
-  bool vacuumNeeded = false;
+  bool journalModeSet = false;
 
   if (schemaVersion != kSQLiteSchemaVersion) {
-#if defined(MOZ_WIDGET_ANDROID) || defined(MOZ_WIDGET_GONK)
     if (!schemaVersion) {
+      // Brand new file.
+
+#ifdef IDB_MOBILE
       // Have to do this before opening a transaction.
       rv = connection->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
         // Turn on auto_vacuum mode to reclaim disk space on mobile devices.
@@ -2357,9 +2587,18 @@ CreateDatabaseConnection(nsIFile* aDBFile,
       if (NS_WARN_IF(NS_FAILED(rv))) {
         return rv;
       }
-    }
 #endif
 
+      rv = SetJournalMode(connection);
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+
+      journalModeSet = true;
+    }
+
+    bool vacuumNeeded = false;
+
     mozStorageTransaction transaction(connection, false,
                                   mozIStorageConnection::TRANSACTION_IMMEDIATE);
 
@@ -2452,10 +2691,17 @@ CreateDatabaseConnection(nsIFile* aDBFile,
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
-  }
 
     if (vacuumNeeded) {
-    rv = connection->ExecuteSimpleSQL(NS_LITERAL_CSTRING("VACUUM;"));
+      rv = connection->ExecuteSimpleSQL(NS_LITERAL_CSTRING("VACUUM"));
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+    }
+  }
+
+  if (!journalModeSet) {
+    rv = SetJournalMode(connection);
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
@@ -2483,7 +2729,7 @@ GetFileForPath(const nsAString& aPath)
 }
 
 nsresult
-GetDatabaseConnection(const nsAString& aDatabaseFilePath,
+GetStorageConnection(const nsAString& aDatabaseFilePath,
                      PersistenceType aPersistenceType,
                      const nsACString& aGroup,
                      const nsACString& aOrigin,
@@ -2496,7 +2742,7 @@ GetDatabaseConnection(const nsAString& aDatabaseFilePath,
   MOZ_ASSERT(aConnection);
 
   PROFILER_LABEL("IndexedDB",
-                 "GetDatabaseConnection",
+                 "GetStorageConnection",
                  js::ProfileEntry::Category::STORAGE);
 
   nsCOMPtr<nsIFile> dbFile = GetFileForPath(aDatabaseFilePath);
@@ -2530,7 +2776,7 @@ GetDatabaseConnection(const nsAString& aDatabaseFilePath,
   }
 
   nsCOMPtr<mozIStorageConnection> connection;
-  rv = ss->OpenDatabaseWithFileURL(dbFileUrl, getter_AddRefs(connection));
+  rv = OpenDatabaseAndHandleBusy(ss, dbFileUrl, getter_AddRefs(connection));
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -2540,6 +2786,11 @@ GetDatabaseConnection(const nsAString& aDatabaseFilePath,
     return rv;
   }
 
+  rv = SetJournalMode(connection);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
   connection.forget(aConnection);
   return NS_OK;
 }
@@ -4281,6 +4532,11 @@ private:
   void
   RunOnOwningThread();
 
+  nsresult
+  DeleteFile(nsIFile* aDirectory,
+             const nsAString& aFilename,
+             QuotaManager* aQuotaManager);
+
   NS_DECL_NSIRUNNABLE
 };
 
@@ -6690,22 +6946,26 @@ TransactionBase::EnsureConnection()
                  "TransactionBase::EnsureConnection",
                  js::ProfileEntry::Category::STORAGE);
 
+  const bool readOnly = mMode == IDBTransaction::READ_ONLY;
+
   if (!mConnection) {
     nsCOMPtr<mozIStorageConnection> connection;
     nsresult rv =
-      GetDatabaseConnection(mDatabase->FilePath(), mDatabase->Type(),
+      GetStorageConnection(mDatabase->FilePath(),
-                            mDatabase->Group(), mDatabase->Origin(),
+                           mDatabase->Type(),
+                           mDatabase->Group(),
+                           mDatabase->Origin(),
                            getter_AddRefs(connection));
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
 
     nsRefPtr<UpdateRefcountFunction> function;
-    nsCString beginTransaction;
 
-    if (mMode == IDBTransaction::READ_ONLY) {
+    nsAutoCString beginTransaction;
-      beginTransaction.AssignLiteral("BEGIN TRANSACTION;");
+    beginTransaction.AssignLiteral("BEGIN");
-    } else {
+
+    if (!readOnly) {
       function = new UpdateRefcountFunction(mDatabase->GetFileManager());
 
       rv = connection->CreateFunction(NS_LITERAL_CSTRING("update_refcount"), 2,
@@ -6714,16 +6974,10 @@ TransactionBase::EnsureConnection()
         return rv;
       }
 
-      beginTransaction.AssignLiteral("BEGIN IMMEDIATE TRANSACTION;");
+      beginTransaction.AppendLiteral(" IMMEDIATE");
     }
 
-    nsCOMPtr<mozIStorageStatement> stmt;
+    rv = connection->ExecuteSimpleSQL(beginTransaction);
-    rv = connection->CreateStatement(beginTransaction, getter_AddRefs(stmt));
-    if (NS_WARN_IF(NS_FAILED(rv))) {
-      return rv;
-    }
-
-    rv = stmt->Execute();
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
@@ -9069,7 +9323,7 @@ FileManager::InitDirectory(nsIFile* aDirectory,
 
     if (hasElements) {
       nsCOMPtr<mozIStorageConnection> connection;
-      rv = CreateDatabaseConnection(aDatabaseFile,
+      rv = CreateStorageConnection(aDatabaseFile,
                                    aDirectory,
                                    NullString(),
                                    aPersistenceType,
@@ -9279,6 +9533,7 @@ struct FileManagerInitInfo
 {
   nsCOMPtr<nsIFile> mDirectory;
   nsCOMPtr<nsIFile> mDatabaseFile;
+  nsCOMPtr<nsIFile> mDatabaseWALFile;
 };
 
 nsresult
@@ -9311,7 +9566,17 @@ QuotaClient::InitOrigin(PersistenceType aPersistenceType,
     return rv;
   }
 
-  const NS_ConvertASCIItoUTF16 filesSuffix(kFileManagerDirectoryNameSuffix);
+  const NS_ConvertASCIItoUTF16 filesSuffix(
+    kFileManagerDirectoryNameSuffix,
+    LiteralStringLength(kFileManagerDirectoryNameSuffix));
+
+  const NS_ConvertASCIItoUTF16 journalSuffix(
+    kSQLiteJournalSuffix,
+    LiteralStringLength(kSQLiteJournalSuffix));
+  const NS_ConvertASCIItoUTF16 shmSuffix(kSQLiteSHMSuffix,
+                                         LiteralStringLength(kSQLiteSHMSuffix));
+  const NS_ConvertASCIItoUTF16 walSuffix(kSQLiteWALSuffix,
+                                         LiteralStringLength(kSQLiteWALSuffix));
 
   bool hasMore;
   while (NS_SUCCEEDED((rv = entries->HasMoreElements(&hasMore))) &&
@@ -9332,7 +9597,6 @@ QuotaClient::InitOrigin(PersistenceType aPersistenceType,
       return rv;
     }
 
-
     bool isDirectory;
     rv = file->IsDirectory(&isDirectory);
     if (NS_WARN_IF(NS_FAILED(rv))) {
@@ -9347,12 +9611,24 @@ QuotaClient::InitOrigin(PersistenceType aPersistenceType,
       continue;
     }
 
-    // Skip SQLite and Desktop Service Store (.DS_Store) files.
+    // Skip Desktop Service Store (.DS_Store) files. These files are only used
-    // Desktop Service Store file is only used on Mac OS X, but the profile
+    // on Mac OS X, but the profile can be shared across different operating
-    // can be shared across different operating systems, so we check it on
+    // systems, so we check it on all platforms.
-    // all platforms.
+    if (leafName.EqualsLiteral(DSSTORE_FILE_NAME)) {
-    if (StringEndsWith(leafName, NS_LITERAL_STRING(".sqlite-journal")) ||
+      continue;
-        leafName.EqualsLiteral(DSSTORE_FILE_NAME)) {
+    }
+
+    // Skip SQLite temporary files. These files take up space on disk but will
+    // be deleted as soon as the database is opened, so we don't count them
+    // towards quota.
+    if (StringEndsWith(leafName, journalSuffix) ||
+        StringEndsWith(leafName, shmSuffix)) {
+      continue;
+    }
+
+    // The SQLite WAL file does count towards quota, but it is handled below
+    // once we find the actual database file.
+    if (StringEndsWith(leafName, walSuffix)) {
       continue;
     }
 
@@ -9362,22 +9638,36 @@ QuotaClient::InitOrigin(PersistenceType aPersistenceType,
       continue;
     }
 
-    nsString fmDirectoryBaseName = dbBaseFilename + filesSuffix;
-
     nsCOMPtr<nsIFile> fmDirectory;
     rv = directory->Clone(getter_AddRefs(fmDirectory));
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
 
+    nsString fmDirectoryBaseName = dbBaseFilename + filesSuffix;
+
     rv = fmDirectory->Append(fmDirectoryBaseName);
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
 
+    nsCOMPtr<nsIFile> walFile;
+    if (aUsageInfo) {
+      rv = directory->Clone(getter_AddRefs(walFile));
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+
+      rv = walFile->Append(dbBaseFilename + walSuffix);
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+    }
+
     FileManagerInitInfo* initInfo = initInfos.AppendElement();
     initInfo->mDirectory.swap(fmDirectory);
     initInfo->mDatabaseFile.swap(file);
+    initInfo->mDatabaseWALFile.swap(walFile);
 
     validSubdirs.PutEntry(fmDirectoryBaseName);
   }
@@ -9462,6 +9752,7 @@ QuotaClient::InitOrigin(PersistenceType aPersistenceType,
     FileManagerInitInfo& initInfo = initInfos[i];
     MOZ_ASSERT(initInfo.mDirectory);
     MOZ_ASSERT(initInfo.mDatabaseFile);
+    MOZ_ASSERT_IF(aUsageInfo, initInfo.mDatabaseWALFile);
 
     rv = FileManager::InitDirectory(initInfo.mDirectory,
                                     initInfo.mDatabaseFile,
@@ -9483,6 +9774,15 @@ QuotaClient::InitOrigin(PersistenceType aPersistenceType,
 
       aUsageInfo->AppendToDatabaseUsage(uint64_t(fileSize));
 
+      rv = initInfo.mDatabaseWALFile->GetFileSize(&fileSize);
+      if (NS_SUCCEEDED(rv)) {
+        MOZ_ASSERT(fileSize >= 0);
+        aUsageInfo->AppendToDatabaseUsage(uint64_t(fileSize));
+      } else if (NS_WARN_IF(rv != NS_ERROR_FILE_NOT_FOUND &&
+                            rv != NS_ERROR_FILE_TARGET_DOES_NOT_EXIST)) {
+        return rv;
+      }
+
       uint64_t usage;
       rv = FileManager::GetUsage(initInfo.mDirectory, &usage);
       if (NS_WARN_IF(NS_FAILED(rv))) {
@@ -9494,20 +9794,31 @@ QuotaClient::InitOrigin(PersistenceType aPersistenceType,
   }
 
   // We have to do this after file manager initialization.
+  if (!unknownFiles.IsEmpty()) {
+#ifdef DEBUG
     for (uint32_t count = unknownFiles.Length(), i = 0; i < count; i++) {
       nsCOMPtr<nsIFile>& unknownFile = unknownFiles[i];
 
-    // Some temporary SQLite files could disappear during file manager
+      nsString leafName;
-    // initialization, so we have to check if the unknown file still exists.
+      MOZ_ALWAYS_TRUE(NS_SUCCEEDED(unknownFile->GetLeafName(leafName)));
-    bool exists;
-    rv = unknownFile->Exists(&exists);
-    if (NS_WARN_IF(NS_FAILED(rv))) {
-      return rv;
-    }
 
-    if (exists) {
+      MOZ_ASSERT(!StringEndsWith(leafName, journalSuffix));
-      return NS_ERROR_UNEXPECTED;
+      MOZ_ASSERT(!StringEndsWith(leafName, shmSuffix));
+      MOZ_ASSERT(!StringEndsWith(leafName, walSuffix));
+
+      nsString path;
+      MOZ_ALWAYS_TRUE(NS_SUCCEEDED(unknownFile->GetPath(path)));
+      MOZ_ASSERT(!path.IsEmpty());
+
+      nsPrintfCString warning("Refusing to open databases for \"%s\" because "
+                              "an unexpected file exists in the storage "
+                              "area: \"%s\"",
+                              PromiseFlatCString(aOrigin).get(),
+                              NS_ConvertUTF16toUTF8(path).get());
+      NS_WARNING(warning.get());
     }
+#endif
+    return NS_ERROR_UNEXPECTED;
   }
 
   return NS_OK;
@@ -9696,6 +10007,9 @@ QuotaClient::GetUsageForDirectoryInternal(nsIFile* aDirectory,
     return NS_OK;
   }
 
+  const NS_ConvertASCIItoUTF16 shmSuffix(kSQLiteSHMSuffix,
+                                         LiteralStringLength(kSQLiteSHMSuffix));
+
   bool hasMore;
   while (NS_SUCCEEDED((rv = entries->HasMoreElements(&hasMore))) &&
          hasMore &&
@@ -9709,6 +10023,16 @@ QuotaClient::GetUsageForDirectoryInternal(nsIFile* aDirectory,
     nsCOMPtr<nsIFile> file = do_QueryInterface(entry);
     MOZ_ASSERT(file);
 
+    nsString leafName;
+    rv = file->GetLeafName(leafName);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    if (StringEndsWith(leafName, shmSuffix)) {
+      continue;
+    }
+
     bool isDirectory;
     rv = file->IsDirectory(&isDirectory);
     if (NS_WARN_IF(NS_FAILED(rv))) {
@@ -10845,7 +11169,7 @@ FactoryOp::CheckPermission(ContentParent* aContentParent,
     return rv;
   }
 
-#if defined(MOZ_WIDGET_ANDROID) || defined(MOZ_WIDGET_GONK)
+#ifdef IDB_MOBILE
   if (persistenceType == PERSISTENCE_TYPE_PERSISTENT &&
       !QuotaManager::IsOriginWhitelistedForPersistentStorage(origin) &&
       !isApp) {
@@ -11342,7 +11666,7 @@ OpenDatabaseOp::DoDatabaseWork()
   }
 
   nsCOMPtr<mozIStorageConnection> connection;
-  rv = CreateDatabaseConnection(dbFile,
+  rv = CreateStorageConnection(dbFile,
                                fmDirectory,
                                databaseName,
                                persistenceType,
@@ -12411,7 +12735,7 @@ DeleteDatabaseOp::LoadPreviousVersion(nsIFile* aDatabaseFile)
   }
 
   nsCOMPtr<mozIStorageConnection> connection;
-  rv = ss->OpenDatabase(aDatabaseFile, getter_AddRefs(connection));
+  rv = OpenDatabaseAndHandleBusy(ss, aDatabaseFile, getter_AddRefs(connection));
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return;
   }
@@ -12730,6 +13054,74 @@ VersionChangeOp::RunOnMainThread()
   return NS_OK;
 }
 
+
+nsresult
+DeleteDatabaseOp::
+VersionChangeOp::DeleteFile(nsIFile* aDirectory,
+                            const nsAString& aFilename,
+                            QuotaManager* aQuotaManager)
+{
+  AssertIsOnIOThread();
+  MOZ_ASSERT(aDirectory);
+  MOZ_ASSERT(!aFilename.IsEmpty());
+  MOZ_ASSERT_IF(aQuotaManager, mDeleteDatabaseOp->mEnforcingQuota);
+
+  MOZ_ASSERT(mDeleteDatabaseOp->mState == State_DatabaseWorkVersionChange);
+
+  PROFILER_LABEL("IndexedDB",
+                 "DeleteDatabaseOp::VersionChangeOp::DeleteFile",
+                 js::ProfileEntry::Category::STORAGE);
+
+  nsCOMPtr<nsIFile> file;
+  nsresult rv = aDirectory->Clone(getter_AddRefs(file));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = file->Append(aFilename);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  int64_t fileSize;
+
+  if (aQuotaManager) {
+    rv = file->GetFileSize(&fileSize);
+    if (rv == NS_ERROR_FILE_NOT_FOUND ||
+        rv == NS_ERROR_FILE_TARGET_DOES_NOT_EXIST) {
+      return NS_OK;
+    }
+
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    MOZ_ASSERT(fileSize >= 0);
+  }
+
+  rv = file->Remove(false);
+  if (rv == NS_ERROR_FILE_NOT_FOUND ||
+      rv == NS_ERROR_FILE_TARGET_DOES_NOT_EXIST) {
+    return NS_OK;
+  }
+
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  if (aQuotaManager && fileSize > 0) {
+    const PersistenceType& persistenceType =
+      mDeleteDatabaseOp->mCommonParams.metadata().persistenceType();
+
+    aQuotaManager->DecreaseUsageForOrigin(persistenceType,
+                                          mDeleteDatabaseOp->mGroup,
+                                          mDeleteDatabaseOp->mOrigin,
+                                          fileSize);
+  }
+
+  return NS_OK;
+}
+
 nsresult
 DeleteDatabaseOp::
 VersionChangeOp::RunOnIOThread()
@@ -12747,6 +13139,16 @@ VersionChangeOp::RunOnIOThread()
     return NS_ERROR_DOM_INDEXEDDB_UNKNOWN_ERR;
   }
 
+  const PersistenceType& persistenceType =
+    mDeleteDatabaseOp->mCommonParams.metadata().persistenceType();
+
+  QuotaManager* quotaManager =
+    mDeleteDatabaseOp->mEnforcingQuota ?
+    QuotaManager::Get() :
+    nullptr;
+
+  MOZ_ASSERT_IF(mDeleteDatabaseOp->mEnforcingQuota, quotaManager);
+
   nsCOMPtr<nsIFile> directory =
     GetFileForPath(mDeleteDatabaseOp->mDatabaseDirectoryPath);
   if (NS_WARN_IF(!directory)) {
@@ -12754,78 +13156,48 @@ VersionChangeOp::RunOnIOThread()
     return NS_ERROR_DOM_INDEXEDDB_UNKNOWN_ERR;
   }
 
-  nsCOMPtr<nsIFile> dbFile;
+  // The database file counts towards quota.
-  nsresult rv = directory->Clone(getter_AddRefs(dbFile));
+  nsAutoString filename =
+    mDeleteDatabaseOp->mDatabaseFilenameBase + NS_LITERAL_STRING(".sqlite");
+
+  nsresult rv = DeleteFile(directory, filename, quotaManager);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
-  rv = dbFile->Append(mDeleteDatabaseOp->mDatabaseFilenameBase +
+  // .sqlite-journal files don't count towards quota.
-                      NS_LITERAL_STRING(".sqlite"));
+  const NS_ConvertASCIItoUTF16 journalSuffix(
+    kSQLiteJournalSuffix,
+    LiteralStringLength(kSQLiteJournalSuffix));
+
+  filename = mDeleteDatabaseOp->mDatabaseFilenameBase + journalSuffix;
+
+  rv = DeleteFile(directory, filename, /* doesn't count */ nullptr);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
-  bool exists;
+  // .sqlite-shm files don't count towards quota.
-  rv = dbFile->Exists(&exists);
+  const NS_ConvertASCIItoUTF16 shmSuffix(kSQLiteSHMSuffix,
+                                         LiteralStringLength(kSQLiteSHMSuffix));
+
+  filename = mDeleteDatabaseOp->mDatabaseFilenameBase + shmSuffix;
+
+  rv = DeleteFile(directory, filename, /* doesn't count */ nullptr);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
-  const nsString& databaseName =
+  // .sqlite-wal files do count towards quota.
-    mDeleteDatabaseOp->mCommonParams.metadata().name();
+  const NS_ConvertASCIItoUTF16 walSuffix(kSQLiteWALSuffix,
-  PersistenceType persistenceType =
+                                         LiteralStringLength(kSQLiteWALSuffix));
-    mDeleteDatabaseOp->mCommonParams.metadata().persistenceType();
 
-  QuotaManager* quotaManager = QuotaManager::Get();
+  filename = mDeleteDatabaseOp->mDatabaseFilenameBase + walSuffix;
-  MOZ_ASSERT(quotaManager);
 
-  if (exists) {
+  rv = DeleteFile(directory, filename, quotaManager);
-    int64_t fileSize;
-
-    if (mDeleteDatabaseOp->mEnforcingQuota) {
-      rv = dbFile->GetFileSize(&fileSize);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
-    }
-
-    rv = dbFile->Remove(false);
-    if (NS_WARN_IF(NS_FAILED(rv))) {
-      return rv;
-    }
-
-    if (mDeleteDatabaseOp->mEnforcingQuota) {
-      quotaManager->DecreaseUsageForOrigin(persistenceType,
-                                           mDeleteDatabaseOp->mGroup,
-                                           mDeleteDatabaseOp->mOrigin,
-                                           fileSize);
-    }
-  }
-
-  nsCOMPtr<nsIFile> dbJournalFile;
-  rv = directory->Clone(getter_AddRefs(dbJournalFile));
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  rv = dbJournalFile->Append(mDeleteDatabaseOp->mDatabaseFilenameBase +
-                             NS_LITERAL_STRING(".sqlite-journal"));
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  rv = dbJournalFile->Exists(&exists);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  if (exists) {
-    rv = dbJournalFile->Remove(false);
-    if (NS_WARN_IF(NS_FAILED(rv))) {
-      return rv;
-    }
-  }
 
   nsCOMPtr<nsIFile> fmDirectory;
   rv = directory->Clone(getter_AddRefs(fmDirectory));
@@ -12833,7 +13205,10 @@ VersionChangeOp::RunOnIOThread()
     return rv;
   }
 
-  const NS_ConvertASCIItoUTF16 filesSuffix(kFileManagerDirectoryNameSuffix);
+  // The files directory counts towards quota.
+  const NS_ConvertASCIItoUTF16 filesSuffix(
+    kFileManagerDirectoryNameSuffix,
+    LiteralStringLength(kFileManagerDirectoryNameSuffix));
 
   rv = fmDirectory->Append(mDeleteDatabaseOp->mDatabaseFilenameBase +
                            filesSuffix);
@@ -12841,6 +13216,7 @@ VersionChangeOp::RunOnIOThread()
     return rv;
   }
 
+  bool exists;
   rv = fmDirectory->Exists(&exists);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
@@ -12869,20 +13245,35 @@ VersionChangeOp::RunOnIOThread()
 
     rv = fmDirectory->Remove(true);
     if (NS_WARN_IF(NS_FAILED(rv))) {
-      return rv;
+      // We may have deleted some files, check if we can and update quota
+      // information before returning the error.
+      if (mDeleteDatabaseOp->mEnforcingQuota) {
+        uint64_t newUsage;
+        if (NS_SUCCEEDED(FileManager::GetUsage(fmDirectory, &newUsage))) {
+          MOZ_ASSERT(newUsage <= usage);
+          usage = usage - newUsage;
+        }
+      }
     }
 
-    if (mDeleteDatabaseOp->mEnforcingQuota) {
+    if (mDeleteDatabaseOp->mEnforcingQuota && usage) {
       quotaManager->DecreaseUsageForOrigin(persistenceType,
                                            mDeleteDatabaseOp->mGroup,
                                            mDeleteDatabaseOp->mOrigin,
                                            usage);
     }
+
+    if (NS_FAILED(rv)) {
+      return rv;
+    }
   }
 
   IndexedDatabaseManager* mgr = IndexedDatabaseManager::Get();
   MOZ_ASSERT(mgr);
 
+  const nsString& databaseName =
+    mDeleteDatabaseOp->mCommonParams.metadata().name();
+
   mgr->InvalidateFileManager(persistenceType,
                              mDeleteDatabaseOp->mOrigin,
                              databaseName);
@@ -17213,3 +17604,7 @@ DEBUGThreadSlower::AfterProcessNextEvent(nsIThreadInternal* /* aThread */,
 } // namespace indexedDB
 } // namespace dom
 } // namespace mozilla
+
+#undef IDB_MOBILE
+#undef ASSERT_UNLESS_FUZZING
+#undef DISABLE_ASSERTS_FOR_FUZZING

dom/quota/QuotaObject.cpp

@@ -6,11 +6,62 @@
 
 #include "QuotaObject.h"
 
+#include "mozilla/TypeTraits.h"
 #include "QuotaManager.h"
 #include "Utilities.h"
 
+#ifdef DEBUG
+#include "nsComponentManagerUtils.h"
+#include "nsIFile.h"
+#include "nsXPCOMCID.h"
+#endif
+
 USING_QUOTA_NAMESPACE
 
+namespace {
+
+template <typename T, typename U>
+void
+AssertPositiveIntegers(T aOne, U aTwo)
+{
+  static_assert(mozilla::IsIntegral<T>::value, "Not an integer!");
+  static_assert(mozilla::IsIntegral<U>::value, "Not an integer!");
+  MOZ_ASSERT(aOne >= 0);
+  MOZ_ASSERT(aTwo >= 0);
+}
+
+template <typename T, typename U>
+void
+AssertNoOverflow(T aOne, U aTwo)
+{
+  AssertPositiveIntegers(aOne, aTwo);
+  AssertNoOverflow(uint64_t(aOne), uint64_t(aTwo));
+}
+
+template <>
+void
+AssertNoOverflow<uint64_t, uint64_t>(uint64_t aOne, uint64_t aTwo)
+{
+  MOZ_ASSERT(UINT64_MAX - aOne >= aTwo);
+}
+
+template <typename T, typename U>
+void
+AssertNoUnderflow(T aOne, U aTwo)
+{
+  AssertPositiveIntegers(aOne, aTwo);
+  AssertNoUnderflow(uint64_t(aOne), uint64_t(aTwo));
+}
+
+template <>
+void
+AssertNoUnderflow<uint64_t, uint64_t>(uint64_t aOne, uint64_t aTwo)
+{
+  MOZ_ASSERT(aOne >= aTwo);
+}
+
+} // anonymous namespace
+
 void
 QuotaObject::AddRef()
 {
@@ -64,31 +115,65 @@ QuotaObject::Release()
 void
 QuotaObject::UpdateSize(int64_t aSize)
 {
+  MOZ_ASSERT(aSize >= 0);
+
+#ifdef DEBUG
+  {
+    nsCOMPtr<nsIFile> file = do_CreateInstance(NS_LOCAL_FILE_CONTRACTID);
+    MOZ_ASSERT(file);
+
+    MOZ_ASSERT(NS_SUCCEEDED(file->InitWithPath(mPath)));
+
+    bool exists;
+    MOZ_ASSERT(NS_SUCCEEDED(file->Exists(&exists)));
+
+    if (exists) {
+      int64_t fileSize;
+      MOZ_ASSERT(NS_SUCCEEDED(file->GetFileSize(&fileSize)));
+
+      MOZ_ASSERT(aSize == fileSize);
+    } else {
+      MOZ_ASSERT(!aSize);
+    }
+  }
+#endif
+
   QuotaManager* quotaManager = QuotaManager::Get();
   NS_ASSERTION(quotaManager, "Shouldn't be null!");
 
   MutexAutoLock lock(quotaManager->mQuotaMutex);
 
-  if (!mOriginInfo) {
+  if (!mOriginInfo || mSize == aSize) {
     return;
   }
 
+  AssertNoUnderflow(quotaManager->mTemporaryStorageUsage, mSize);
+  quotaManager->mTemporaryStorageUsage -= mSize;
+
   GroupInfo* groupInfo = mOriginInfo->mGroupInfo;
 
-  quotaManager->mTemporaryStorageUsage -= mSize;
+  AssertNoUnderflow(groupInfo->mUsage, mSize);
   groupInfo->mUsage -= mSize;
+
+  AssertNoUnderflow(mOriginInfo->mUsage, mSize);
   mOriginInfo->mUsage -= mSize;
 
   mSize = aSize;
 
+  AssertNoOverflow(mOriginInfo->mUsage, mSize);
   mOriginInfo->mUsage += mSize;
+
+  AssertNoOverflow(groupInfo->mUsage, mSize);
   groupInfo->mUsage += mSize;
+
+  AssertNoOverflow(quotaManager->mTemporaryStorageUsage, mSize);
   quotaManager->mTemporaryStorageUsage += mSize;
 }
 
 bool
 QuotaObject::MaybeAllocateMoreSpace(int64_t aOffset, int32_t aCount)
 {
+  AssertNoOverflow(aOffset, aCount);
   int64_t end = aOffset + aCount;
 
   QuotaManager* quotaManager = QuotaManager::Get();
@@ -106,26 +191,32 @@ QuotaObject::MaybeAllocateMoreSpace(int64_t aOffset, int32_t aCount)
     groupInfo->mGroupInfoPair->LockedGetGroupInfo(
       ComplementaryPersistenceType(groupInfo->mPersistenceType));
 
+  AssertNoUnderflow(end, mSize);
   uint64_t delta = end - mSize;
 
+  AssertNoOverflow(mOriginInfo->mUsage, delta);
   uint64_t newUsage = mOriginInfo->mUsage + delta;
 
   // Temporary storage has no limit for origin usage (there's a group and the
   // global limit though).
 
+  AssertNoOverflow(groupInfo->mUsage, delta);
   uint64_t newGroupUsage = groupInfo->mUsage + delta;
 
   uint64_t groupUsage = groupInfo->mUsage;
   if (complementaryGroupInfo) {
+    AssertNoOverflow(groupUsage, complementaryGroupInfo->mUsage);
     groupUsage += complementaryGroupInfo->mUsage;
   }
 
   // Temporary storage has a hard limit for group usage (20 % of the global
   // limit).
+  AssertNoOverflow(groupUsage, delta);
   if (groupUsage + delta > quotaManager->GetGroupLimit()) {
     return false;
   }
 
+  AssertNoOverflow(quotaManager->mTemporaryStorageUsage, delta);
   uint64_t newTemporaryStorageUsage = quotaManager->mTemporaryStorageUsage +
                                       delta;
 
@@ -181,17 +272,22 @@ QuotaObject::MaybeAllocateMoreSpace(int64_t aOffset, int32_t aCount)
     // We unlocked and relocked several times so we need to recompute all the
     // essential variables and recheck the group limit.
 
+    AssertNoUnderflow(end, mSize);
     delta = end - mSize;
 
+    AssertNoOverflow(mOriginInfo->mUsage, delta);
     newUsage = mOriginInfo->mUsage + delta;
 
+    AssertNoOverflow(groupInfo->mUsage, delta);
     newGroupUsage = groupInfo->mUsage + delta;
 
     groupUsage = groupInfo->mUsage;
     if (complementaryGroupInfo) {
+      AssertNoOverflow(groupUsage, complementaryGroupInfo->mUsage);
       groupUsage += complementaryGroupInfo->mUsage;
     }
 
+    AssertNoOverflow(groupUsage, delta);
     if (groupUsage + delta > quotaManager->GetGroupLimit()) {
       // Unfortunately some other thread increased the group usage in the
       // meantime and we are not below the group limit anymore.
@@ -204,6 +300,7 @@ QuotaObject::MaybeAllocateMoreSpace(int64_t aOffset, int32_t aCount)
       return false;
     }
 
+    AssertNoOverflow(quotaManager->mTemporaryStorageUsage, delta);
     newTemporaryStorageUsage = quotaManager->mTemporaryStorageUsage + delta;
 
     NS_ASSERTION(newTemporaryStorageUsage <=
@@ -211,14 +308,13 @@ QuotaObject::MaybeAllocateMoreSpace(int64_t aOffset, int32_t aCount)
 
     // Ok, we successfully freed enough space and the operation can continue
     // without throwing the quota error.
-
     mOriginInfo->mUsage = newUsage;
     groupInfo->mUsage = newGroupUsage;
     quotaManager->mTemporaryStorageUsage = newTemporaryStorageUsage;;
 
     // Some other thread could increase the size in the meantime, but no more
     // than this one.
-    NS_ASSERTION(mSize < end, "This shouldn't happen!");
+    MOZ_ASSERT(mSize < end);
     mSize = end;
 
     // Finally, release IO thread only objects and allow next synchronized
@@ -244,13 +340,16 @@ OriginInfo::LockedDecreaseUsage(int64_t aSize)
 {
   AssertCurrentThreadOwnsQuotaMutex();
 
+  AssertNoUnderflow(mUsage, aSize);
   mUsage -= aSize;
 
+  AssertNoUnderflow(mGroupInfo->mUsage, aSize);
   mGroupInfo->mUsage -= aSize;
 
   QuotaManager* quotaManager = QuotaManager::Get();
   MOZ_ASSERT(quotaManager);
 
+  AssertNoUnderflow(quotaManager->mTemporaryStorageUsage, aSize);
   quotaManager->mTemporaryStorageUsage -= aSize;
 }
 
@@ -294,11 +393,13 @@ GroupInfo::LockedAddOriginInfo(OriginInfo* aOriginInfo)
                "Replacing an existing entry!");
   mOriginInfos.AppendElement(aOriginInfo);
 
+  AssertNoOverflow(mUsage, aOriginInfo->mUsage);
   mUsage += aOriginInfo->mUsage;
 
   QuotaManager* quotaManager = QuotaManager::Get();
   MOZ_ASSERT(quotaManager);
 
+  AssertNoOverflow(quotaManager->mTemporaryStorageUsage, aOriginInfo->mUsage);
   quotaManager->mTemporaryStorageUsage += aOriginInfo->mUsage;
 }
 
@@ -309,13 +410,13 @@ GroupInfo::LockedRemoveOriginInfo(const nsACString& aOrigin)
 
   for (uint32_t index = 0; index < mOriginInfos.Length(); index++) {
     if (mOriginInfos[index]->mOrigin == aOrigin) {
-      MOZ_ASSERT(mUsage >= mOriginInfos[index]->mUsage);
+      AssertNoUnderflow(mUsage, mOriginInfos[index]->mUsage);
       mUsage -= mOriginInfos[index]->mUsage;
 
       QuotaManager* quotaManager = QuotaManager::Get();
       MOZ_ASSERT(quotaManager);
 
-      MOZ_ASSERT(quotaManager->mTemporaryStorageUsage >=
+      AssertNoUnderflow(quotaManager->mTemporaryStorageUsage,
                         mOriginInfos[index]->mUsage);
       quotaManager->mTemporaryStorageUsage -= mOriginInfos[index]->mUsage;
 
@@ -337,10 +438,10 @@ GroupInfo::LockedRemoveOriginInfos()
   for (uint32_t index = mOriginInfos.Length(); index > 0; index--) {
     OriginInfo* originInfo = mOriginInfos[index - 1];
 
-    MOZ_ASSERT(mUsage >= originInfo->mUsage);
+    AssertNoUnderflow(mUsage, originInfo->mUsage);
     mUsage -= originInfo->mUsage;
 
-    MOZ_ASSERT(quotaManager->mTemporaryStorageUsage >= originInfo->mUsage);
+    AssertNoUnderflow(quotaManager->mTemporaryStorageUsage, originInfo->mUsage);
     quotaManager->mTemporaryStorageUsage -= originInfo->mUsage;
 
     mOriginInfos.RemoveElementAt(index - 1);

storage/src/TelemetryVFS.cpp

@@ -144,6 +144,198 @@ struct telemetry_file {
   sqlite3_file pReal[1];
 };
 
+const char*
+DatabasePathFromWALPath(const char *zWALName)
+{
+  /**
+   * Do some sketchy pointer arithmetic to find the parameter key. The WAL
+   * filename is in the middle of a big allocated block that contains:
+   *
+   *   - Random Values
+   *   - Main Database Path
+   *   - \0
+   *   - Multiple URI components consisting of:
+   *     - Key
+   *     - \0
+   *     - Value
+   *     - \0
+   *   - \0
+   *   - Journal Path
+   *   - \0
+   *   - WAL Path (zWALName)
+   *   - \0
+   *
+   * Because the main database path is preceded by a random value we have to be
+   * careful when trying to figure out when we should terminate this loop.
+   */
+  MOZ_ASSERT(zWALName);
+
+  nsDependentCSubstring dbPath(zWALName, strlen(zWALName));
+
+  // Chop off the "-wal" suffix.
+  NS_NAMED_LITERAL_CSTRING(kWALSuffix, "-wal");
+  MOZ_ASSERT(StringEndsWith(dbPath, kWALSuffix));
+
+  dbPath.Rebind(zWALName, dbPath.Length() - kWALSuffix.Length());
+  MOZ_ASSERT(!dbPath.IsEmpty());
+
+  // We want to scan to the end of the key/value URI pairs. Skip the preceding
+  // null and go to the last char of the journal path.
+  const char* cursor = zWALName - 2;
+
+  // Make sure we just skipped a null.
+  MOZ_ASSERT(!*(cursor + 1));
+
+  // Walk backwards over the journal path.
+  while (*cursor) {
+    cursor--;
+  }
+
+  // There should be another null here.
+  cursor--;
+  MOZ_ASSERT(!*cursor);
+
+  // Back up one more char to the last char of the previous string. It may be
+  // the database path or it may be a key/value URI pair.
+  cursor--;
+
+#ifdef DEBUG
+  {
+    // Verify that we just walked over the journal path. Account for the two
+    // nulls we just skipped.
+    const char *journalStart = cursor + 3;
+
+    nsDependentCSubstring journalPath(journalStart,
+                                      strlen(journalStart));
+
+    // Chop off the "-journal" suffix.
+    NS_NAMED_LITERAL_CSTRING(kJournalSuffix, "-journal");
+    MOZ_ASSERT(StringEndsWith(journalPath, kJournalSuffix));
+
+    journalPath.Rebind(journalStart,
+                        journalPath.Length() - kJournalSuffix.Length());
+    MOZ_ASSERT(!journalPath.IsEmpty());
+
+    // Make sure that the database name is a substring of the journal name.
+    MOZ_ASSERT(journalPath == dbPath);
+  }
+#endif
+
+  // Now we're either at the end of the key/value URI pairs or we're at the
+  // end of the database path. Carefully walk backwards one character at a
+  // time to do this safely without running past the beginning of the database
+  // path.
+  const char *const dbPathStart = dbPath.BeginReading();
+  const char *dbPathCursor = dbPath.EndReading() - 1;
+  bool isDBPath = true;
+
+  while (true) {
+    MOZ_ASSERT(*dbPathCursor, "dbPathCursor should never see a null char!");
+
+    if (isDBPath) {
+      isDBPath = dbPathStart <= dbPathCursor &&
+                 *dbPathCursor == *cursor &&
+                 *cursor;
+    }
+
+    if (!isDBPath) {
+      // This isn't the database path so it must be a value. Scan past it and
+      // the key also.
+      for (size_t stringCount = 0; stringCount < 2; stringCount++) {
+        // Scan past the string to the preceding null character.
+        while (*cursor) {
+          cursor--;
+        }
+
+        // Back up one more char to the last char of preceding string.
+        cursor--;
+      }
+
+      // Reset and start again.
+      dbPathCursor = dbPath.EndReading() - 1;
+      isDBPath = true;
+
+      continue;
+    }
+
+    MOZ_ASSERT(isDBPath);
+    MOZ_ASSERT(*cursor);
+
+    if (dbPathStart == dbPathCursor) {
+      // Found the full database path, we're all done.
+      MOZ_ASSERT(nsDependentCString(cursor) == dbPath);
+      return cursor;
+    }
+
+    // Change the cursors and go through the loop again.
+    cursor--;
+    dbPathCursor--;
+  }
+
+  MOZ_CRASH("Should never get here!");
+}
+
+already_AddRefed<QuotaObject>
+GetQuotaObjectFromNameAndParameters(const char *zName,
+                                    const char *zURIParameterKey)
+{
+  MOZ_ASSERT(zName);
+  MOZ_ASSERT(zURIParameterKey);
+
+  const char *persistenceType =
+    persistenceType = sqlite3_uri_parameter(zURIParameterKey,
+                                            "persistenceType");
+  if (!persistenceType) {
+    return nullptr;
+  }
+
+  const char *group = sqlite3_uri_parameter(zURIParameterKey, "group");
+  if (!group) {
+    NS_WARNING("SQLite URI had 'persistenceType' but not 'group'?!");
+    return nullptr;
+  }
+
+  const char *origin = sqlite3_uri_parameter(zURIParameterKey, "origin");
+  if (!origin) {
+    NS_WARNING("SQLite URI had 'persistenceType' and 'group' but not "
+               "'origin'?!");
+    return nullptr;
+  }
+
+  QuotaManager *quotaManager = QuotaManager::Get();
+  MOZ_ASSERT(quotaManager);
+
+  return quotaManager->GetQuotaObject(
+    PersistenceTypeFromText(nsDependentCString(persistenceType)),
+    nsDependentCString(group),
+    nsDependentCString(origin),
+    NS_ConvertUTF8toUTF16(zName));
+}
+
+void
+MaybeEstablishQuotaControl(const char *zName,
+                           telemetry_file *pFile,
+                           int flags)
+{
+  MOZ_ASSERT(pFile);
+  MOZ_ASSERT(!pFile->quotaObject);
+
+  if (!(flags & (SQLITE_OPEN_URI | SQLITE_OPEN_WAL))) {
+    return;
+  }
+
+  MOZ_ASSERT(zName);
+
+  const char *zURIParameterKey = (flags & SQLITE_OPEN_WAL) ?
+                                 DatabasePathFromWALPath(zName) :
+                                 zName;
+
+  MOZ_ASSERT(zURIParameterKey);
+
+  pFile->quotaObject =
+    GetQuotaObjectFromNameAndParameters(zName, zURIParameterKey);
+}
+
 /*
 ** Close a telemetry_file.
 */
@@ -197,6 +389,19 @@ xWrite(sqlite3_file *pFile, const void *zBuf, int iAmt, sqlite_int64 iOfst)
   return rc;
 }
 
+/*
+** Return the current file-size of a telemetry_file.
+*/
+int
+xFileSize(sqlite3_file *pFile, sqlite_int64 *pSize)
+{
+  IOThreadAutoTimer ioTimer(IOInterposeObserver::OpStat);
+  telemetry_file *p = (telemetry_file *)pFile;
+  int rc;
+  rc = p->pReal->pMethods->xFileSize(p->pReal, pSize);
+  return rc;
+}
+
 /*
 ** Truncate a telemetry_file.
 */
@@ -209,7 +414,15 @@ xTruncate(sqlite3_file *pFile, sqlite_int64 size)
   Telemetry::AutoTimer<Telemetry::MOZ_SQLITE_TRUNCATE_MS> timer;
   rc = p->pReal->pMethods->xTruncate(p->pReal, size);
   if (rc == SQLITE_OK && p->quotaObject) {
-    p->quotaObject->UpdateSize(size);
+    // xTruncate doesn't always set the size of the file to the exact size
+    // requested (e.g. if a growth increment has been specified it will round up
+    // to the next multiple of the chunk size). Use xFileSize to see what the
+    // real size is.
+    sqlite_int64 newSize;
+    rc = xFileSize(pFile, &newSize);
+    if (rc == SQLITE_OK) {
+      p->quotaObject->UpdateSize(newSize);
+    }
   }
   return rc;
 }
@@ -225,19 +438,6 @@ xSync(sqlite3_file *pFile, int flags)
   return p->pReal->pMethods->xSync(p->pReal, flags);
 }
 
-/*
-** Return the current file-size of a telemetry_file.
-*/
-int
-xFileSize(sqlite3_file *pFile, sqlite_int64 *pSize)
-{
-  IOThreadAutoTimer ioTimer(IOInterposeObserver::OpStat);
-  telemetry_file *p = (telemetry_file *)pFile;
-  int rc;
-  rc = p->pReal->pMethods->xFileSize(p->pReal, pSize);
-  return rc;
-}
-
 /*
 ** Lock a telemetry_file.
 */
@@ -386,20 +586,7 @@ xOpen(sqlite3_vfs* vfs, const char *zName, sqlite3_file* pFile,
   }
   p->histograms = h;
 
-  const char* persistenceType;
+  MaybeEstablishQuotaControl(zName, p, flags);
-  const char* group;
-  const char* origin;
-  if ((flags & SQLITE_OPEN_URI) &&
-      (persistenceType = sqlite3_uri_parameter(zName, "persistenceType")) &&
-      (group = sqlite3_uri_parameter(zName, "group")) &&
-      (origin = sqlite3_uri_parameter(zName, "origin"))) {
-    QuotaManager* quotaManager = QuotaManager::Get();
-    MOZ_ASSERT(quotaManager);
-
-    p->quotaObject = quotaManager->GetQuotaObject(PersistenceTypeFromText(
-      nsDependentCString(persistenceType)), nsDependentCString(group),
-      nsDependentCString(origin), NS_ConvertUTF8toUTF16(zName));
-  }
 
   rc = orig_vfs->xOpen(orig_vfs, zName, p->pReal, flags, pOutFlags);
   if( rc != SQLITE_OK )
@@ -451,7 +638,22 @@ int
 xDelete(sqlite3_vfs* vfs, const char *zName, int syncDir)
 {
   sqlite3_vfs *orig_vfs = static_cast<sqlite3_vfs*>(vfs->pAppData);
-  return orig_vfs->xDelete(orig_vfs, zName, syncDir);
+  int rc;
+  nsRefPtr<QuotaObject> quotaObject;
+
+  if (StringEndsWith(nsDependentCString(zName), NS_LITERAL_CSTRING("-wal"))) {
+    const char *zURIParameterKey = DatabasePathFromWALPath(zName);
+    MOZ_ASSERT(zURIParameterKey);
+
+    quotaObject = GetQuotaObjectFromNameAndParameters(zName, zURIParameterKey);
+  }
+
+  rc = orig_vfs->xDelete(orig_vfs, zName, syncDir);
+  if (rc == SQLITE_OK && quotaObject) {
+    quotaObject->UpdateSize(0);
+  }
+
+  return rc;
 }
 
 int