Bug 1548788 - LSNG: Datastore preparation doesn't have to create new origin directories if they don't exist; r=asuth

Differential Revision: https://phabricator.services.mozilla.com/D29822

dom/localstorage/ActorsParent.cpp

@@ -1488,6 +1488,8 @@ class Connection final {
   class CachedStatement;
 
  private:
+  class InitOriginHelper;
+
   class FlushOp;
   class CloseOp;
 
@@ -1499,8 +1501,10 @@ class Connection final {
   nsInterfaceHashtable<nsCStringHashKey, mozIStorageStatement>
       mCachedStatements;
   WriteOptimizer mWriteOptimizer;
+  const nsCString mSuffix;
+  const nsCString mGroup;
   const nsCString mOrigin;
-  const nsString mDirectoryPath;
+  nsString mDirectoryPath;
   /**
    * Propagated from PrepareDatastoreOp. PrepareDatastoreOp may defer the
    * creation of the localstorage client directory and database on the
@@ -1569,8 +1573,8 @@ class Connection final {
 
  private:
   // Only created by ConnectionThread.
-  Connection(ConnectionThread* aConnectionThread, const nsACString& aOrigin,
-             const nsAString& aDirectoryPath,
+  Connection(ConnectionThread* aConnectionThread, const nsACString& aSuffix,
+             const nsACString& aGroup, const nsACString& aOrigin,
              nsAutoPtr<ArchivedOriginScope>&& aArchivedOriginScope,
              bool aDatabaseNotAvailable);
 
@@ -1607,6 +1611,48 @@ class Connection::CachedStatement final {
   CachedStatement& operator=(const CachedStatement&) = delete;
 };
 
+/**
+ * Helper to invoke EnsureOriginIsInitialized and InitUsageForOrigin on the
+ * QuotaManager IO thread from the LocalStorage connection thread when creating
+ * a database connection on demand. This is necessary because we attempt to
+ * defer the creation of the origin directory and the database until absolutely
+ * needed, but the directory creation and origin initialization must happen on
+ * the QM IO thread for invariant reasons. (We can't just use a mutex because
+ * there could be logic on the IO thread that also wants to deal with the same
+ * origin, so we need to queue a runnable and wait our turn.)
+ */
+class Connection::InitOriginHelper final : public Runnable {
+  mozilla::Monitor mMonitor;
+  const nsCString mSuffix;
+  const nsCString mGroup;
+  const nsCString mOrigin;
+  nsString mOriginDirectoryPath;
+  nsresult mIOThreadResultCode;
+  bool mWaiting;
+
+ public:
+  InitOriginHelper(const nsACString& aSuffix, const nsACString& aGroup,
+                   const nsACString& aOrigin)
+      : Runnable("dom::localstorage::Connection::InitOriginHelper"),
+        mMonitor("InitOriginHelper::mMonitor"),
+        mSuffix(aSuffix),
+        mGroup(aGroup),
+        mOrigin(aOrigin),
+        mIOThreadResultCode(NS_OK),
+        mWaiting(true) {
+    AssertIsOnConnectionThread();
+  }
+
+  nsresult BlockAndReturnOriginDirectoryPath(nsAString& aOriginDirectoryPath);
+
+ private:
+  ~InitOriginHelper() {}
+
+  nsresult RunOnIOThread();
+
+  NS_DECL_NSIRUNNABLE
+};
+
 class Connection::FlushOp final : public ConnectionDatastoreOperationBase {
   WriteOptimizer mWriteOptimizer;
   bool mShadowWrites;
@@ -1651,7 +1697,8 @@ class ConnectionThread final {
   void AssertIsOnConnectionThread();
 
   already_AddRefed<Connection> CreateConnection(
-      const nsACString& aOrigin, const nsAString& aDirectoryPath,
+      const nsACString& aSuffix, const nsACString& aGroup,
+      const nsACString& aOrigin,
       nsAutoPtr<ArchivedOriginScope>&& aArchivedOriginScope,
       bool aDatabaseNotAvailable);
 
@@ -2363,7 +2410,6 @@ class PrepareDatastoreOp
   nsCString mGroup;
   nsCString mMainThreadOrigin;
   nsCString mOrigin;
-  nsString mDirectoryPath;
   nsString mDatabaseFilePath;
   uint32_t mPrivateBrowsingId;
   int64_t mUsage;
@@ -4131,15 +4177,16 @@ ConnectionDatastoreOperationBase::Run() {
  ******************************************************************************/
 
 Connection::Connection(ConnectionThread* aConnectionThread,
+                       const nsACString& aSuffix, const nsACString& aGroup,
                        const nsACString& aOrigin,
-                       const nsAString& aDirectoryPath,
                        nsAutoPtr<ArchivedOriginScope>&& aArchivedOriginScope,
                        bool aDatabaseNotAvailable)
     : mConnectionThread(aConnectionThread),
       mQuotaClient(QuotaClient::GetInstance()),
       mArchivedOriginScope(std::move(aArchivedOriginScope)),
+      mSuffix(aSuffix),
+      mGroup(aGroup),
       mOrigin(aOrigin),
-      mDirectoryPath(aDirectoryPath),
       mDatabaseNotAvailable(aDatabaseNotAvailable),
       mFlushScheduled(false)
 #ifdef DEBUG
@@ -4148,8 +4195,8 @@ Connection::Connection(ConnectionThread* aConnectionThread,
 #endif
 {
   AssertIsOnOwningThread();
+  MOZ_ASSERT(!aGroup.IsEmpty());
   MOZ_ASSERT(!aOrigin.IsEmpty());
-  MOZ_ASSERT(!aDirectoryPath.IsEmpty());
 }
 
 Connection::~Connection() {
@@ -4241,95 +4288,136 @@ void Connection::EndUpdateBatch() {
 nsresult Connection::EnsureStorageConnection() {
   AssertIsOnConnectionThread();
 
-  if (!mStorageConnection) {
-    nsCOMPtr<nsIFile> file;
-    nsresult rv = NS_NewLocalFile(mDirectoryPath, false, getter_AddRefs(file));
+  if (mStorageConnection) {
+    return NS_OK;
+  }
+
+  nsresult rv;
+
+  QuotaManager* quotaManager = QuotaManager::Get();
+  MOZ_ASSERT(quotaManager);
+
+  if (!mDatabaseNotAvailable) {
+    nsCOMPtr<nsIFile> directoryEntry;
+    rv = quotaManager->GetDirectoryForOrigin(PERSISTENCE_TYPE_DEFAULT, mOrigin,
+                                             getter_AddRefs(directoryEntry));
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
 
-    if (mDatabaseNotAvailable) {
-      bool exists;
-      rv = file->Exists(&exists);
-      if (NS_WARN_IF(NS_FAILED(rv))) {
-        return rv;
-      }
-
-      if (!exists) {
-        rv = file->Create(nsIFile::DIRECTORY_TYPE, 0755);
-        if (NS_WARN_IF(NS_FAILED(rv))) {
-          return rv;
-        }
-      }
+    rv = directoryEntry->Append(NS_LITERAL_STRING(LS_DIRECTORY_NAME));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
     }
 
-    rv = file->Append(NS_LITERAL_STRING(DATA_FILE_NAME));
+    rv = directoryEntry->GetPath(mDirectoryPath);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = directoryEntry->Append(NS_LITERAL_STRING(DATA_FILE_NAME));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    nsString databaseFilePath;
+    rv = directoryEntry->GetPath(databaseFilePath);
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
 
     nsCOMPtr<mozIStorageConnection> storageConnection;
-
-    if (mDatabaseNotAvailable) {
-      QuotaManager* quotaManager = QuotaManager::Get();
-      MOZ_ASSERT(quotaManager);
-
-      RefPtr<Runnable> runnable =
-          NS_NewRunnableFunction("dom::localstorage::InitUsageRunnable",
-                                 [origin = mOrigin]() {
-                                   InitUsageForOrigin(origin, 0);
-                                 });
-
-      MOZ_ALWAYS_SUCCEEDS(
-          quotaManager->IOThread()->Dispatch(runnable, NS_DISPATCH_NORMAL));
-
-      nsCOMPtr<nsIFile> usageFile;
-      rv = GetUsageFile(mDirectoryPath, getter_AddRefs(usageFile));
-      if (NS_WARN_IF(NS_FAILED(rv))) {
-        return rv;
-      }
-
-      bool removedUsageFile;
-      rv = CreateStorageConnection(file, usageFile, mOrigin,
-                                   getter_AddRefs(storageConnection),
-                                   &removedUsageFile);
-
-      MOZ_ASSERT(!removedUsageFile);
-
-      if (NS_WARN_IF(NS_FAILED(rv))) {
-        return rv;
-      }
-
-      MOZ_ASSERT(mQuotaClient);
-
-      MutexAutoLock shadowDatabaseLock(mQuotaClient->ShadowDatabaseMutex());
-
-      nsCOMPtr<mozIStorageConnection> shadowConnection;
-      if (!gInitializedShadowStorage) {
-        rv = CreateShadowStorageConnection(quotaManager->GetBasePath(),
-                                           getter_AddRefs(shadowConnection));
-        if (NS_WARN_IF(NS_FAILED(rv))) {
-          return rv;
-        }
-
-        gInitializedShadowStorage = true;
-      }
-    } else {
-      nsString filePath;
-      rv = file->GetPath(filePath);
-      if (NS_WARN_IF(NS_FAILED(rv))) {
-        return rv;
-      }
-
-      rv = GetStorageConnection(filePath, getter_AddRefs(storageConnection));
-      if (NS_WARN_IF(NS_FAILED(rv))) {
-        return rv;
-      }
+    rv = GetStorageConnection(databaseFilePath,
+                              getter_AddRefs(storageConnection));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
     }
 
     mStorageConnection = storageConnection;
+
+    return NS_OK;
   }
 
+  RefPtr<InitOriginHelper> helper =
+      new InitOriginHelper(mSuffix, mGroup, mOrigin);
+
+  nsString originDirectoryPath;
+  rv = helper->BlockAndReturnOriginDirectoryPath(originDirectoryPath);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  nsCOMPtr<nsIFile> directoryEntry;
+  rv = NS_NewLocalFile(originDirectoryPath, false,
+                       getter_AddRefs(directoryEntry));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = directoryEntry->Append(NS_LITERAL_STRING(LS_DIRECTORY_NAME));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = directoryEntry->GetPath(mDirectoryPath);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  bool exists;
+  rv = directoryEntry->Exists(&exists);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  if (!exists) {
+    rv = directoryEntry->Create(nsIFile::DIRECTORY_TYPE, 0755);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+  }
+
+  rv = directoryEntry->Append(NS_LITERAL_STRING(DATA_FILE_NAME));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  nsCOMPtr<nsIFile> usageFile;
+  rv = GetUsageFile(mDirectoryPath, getter_AddRefs(usageFile));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  nsCOMPtr<mozIStorageConnection> storageConnection;
+  bool removedUsageFile;
+
+  rv = CreateStorageConnection(directoryEntry, usageFile, mOrigin,
+                               getter_AddRefs(storageConnection),
+                               &removedUsageFile);
+
+  MOZ_ASSERT(!removedUsageFile);
+
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  MOZ_ASSERT(mQuotaClient);
+
+  MutexAutoLock shadowDatabaseLock(mQuotaClient->ShadowDatabaseMutex());
+
+  nsCOMPtr<mozIStorageConnection> shadowConnection;
+  if (!gInitializedShadowStorage) {
+    rv = CreateShadowStorageConnection(quotaManager->GetBasePath(),
+                                       getter_AddRefs(shadowConnection));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    gInitializedShadowStorage = true;
+  }
+
+  mStorageConnection = storageConnection;
+
   return NS_OK;
 }
 
@@ -4457,6 +4545,71 @@ void Connection::CachedStatement::Assign(
   }
 }
 
+nsresult Connection::InitOriginHelper::BlockAndReturnOriginDirectoryPath(
+    nsAString& aOriginDirectoryPath) {
+  AssertIsOnConnectionThread();
+
+  QuotaManager* quotaManager = QuotaManager::Get();
+  MOZ_ASSERT(quotaManager);
+
+  MOZ_ALWAYS_SUCCEEDS(
+      quotaManager->IOThread()->Dispatch(this, NS_DISPATCH_NORMAL));
+
+  mozilla::MonitorAutoLock lock(mMonitor);
+  while (mWaiting) {
+    lock.Wait();
+  }
+
+  if (NS_WARN_IF(NS_FAILED(mIOThreadResultCode))) {
+    return mIOThreadResultCode;
+  }
+
+  aOriginDirectoryPath = mOriginDirectoryPath;
+  return NS_OK;
+}
+
+nsresult Connection::InitOriginHelper::RunOnIOThread() {
+  AssertIsOnIOThread();
+
+  QuotaManager* quotaManager = QuotaManager::Get();
+  MOZ_ASSERT(quotaManager);
+
+  nsCOMPtr<nsIFile> directoryEntry;
+  nsresult rv = quotaManager->EnsureOriginIsInitialized(
+      PERSISTENCE_TYPE_DEFAULT, mSuffix, mGroup, mOrigin,
+      getter_AddRefs(directoryEntry));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = directoryEntry->GetPath(mOriginDirectoryPath);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  InitUsageForOrigin(mOrigin, 0);
+
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+Connection::InitOriginHelper::Run() {
+  AssertIsOnIOThread();
+
+  nsresult rv = RunOnIOThread();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    mIOThreadResultCode = rv;
+  }
+
+  mozilla::MonitorAutoLock lock(mMonitor);
+  MOZ_ASSERT(mWaiting);
+
+  mWaiting = false;
+  lock.Notify();
+
+  return NS_OK;
+}
+
 Connection::FlushOp::FlushOp(Connection* aConnection,
                              WriteOptimizer&& aWriteOptimizer)
     : ConnectionDatastoreOperationBase(aConnection),
@@ -4615,7 +4768,8 @@ void ConnectionThread::AssertIsOnConnectionThread() {
 }
 
 already_AddRefed<Connection> ConnectionThread::CreateConnection(
-    const nsACString& aOrigin, const nsAString& aDirectoryPath,
+    const nsACString& aSuffix, const nsACString& aGroup,
+    const nsACString& aOrigin,
     nsAutoPtr<ArchivedOriginScope>&& aArchivedOriginScope,
     bool aDatabaseNotAvailable) {
   AssertIsOnOwningThread();
@@ -4623,7 +4777,7 @@ already_AddRefed<Connection> ConnectionThread::CreateConnection(
   MOZ_ASSERT(!mConnections.GetWeak(aOrigin));
 
   RefPtr<Connection> connection =
-      new Connection(this, aOrigin, aDirectoryPath,
+      new Connection(this, aSuffix, aGroup, aOrigin,
                      std::move(aArchivedOriginScope), aDatabaseNotAvailable);
   mConnections.Put(aOrigin, connection);
 
@@ -6745,15 +6899,28 @@ nsresult PrepareDatastoreOp::DatabaseWork() {
     return DatabaseNotAvailable();
   }
 
-  // Initialize the origin even when the origin directory doesn't exist and we
-  // don't have data for migration. GetQuotaObject in GetResponse would fail
-  // otherwise.
+  // The origin directory doesn't need to be created when we don't have data for
+  // migration. It will be created on the connection thread in
+  // Connection::EnsureStorageConnection.
+  // However, origin quota must be initialized, GetQuotaObject in GetResponse
+  // would fail otherwise.
   nsCOMPtr<nsIFile> directoryEntry;
-  rv = quotaManager->EnsureOriginIsInitialized(PERSISTENCE_TYPE_DEFAULT,
-                                               mSuffix, mGroup, mOrigin,
-                                               getter_AddRefs(directoryEntry));
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
+  if (hasDataForMigration) {
+    rv = quotaManager->EnsureOriginIsInitialized(
+        PERSISTENCE_TYPE_DEFAULT, mSuffix, mGroup, mOrigin,
+        getter_AddRefs(directoryEntry));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+  } else {
+    rv = quotaManager->GetDirectoryForOrigin(PERSISTENCE_TYPE_DEFAULT, mOrigin,
+                                             getter_AddRefs(directoryEntry));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    quotaManager->EnsureQuotaForOrigin(PERSISTENCE_TYPE_DEFAULT, mGroup,
+                                       mOrigin);
   }
 
   rv = directoryEntry->Append(NS_LITERAL_STRING(LS_DIRECTORY_NAME));
@@ -6761,7 +6928,8 @@ nsresult PrepareDatastoreOp::DatabaseWork() {
     return rv;
   }
 
-  rv = directoryEntry->GetPath(mDirectoryPath);
+  nsString directoryPath;
+  rv = directoryEntry->GetPath(directoryPath);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -6817,7 +6985,7 @@ nsresult PrepareDatastoreOp::DatabaseWork() {
   }
 
   nsCOMPtr<nsIFile> usageFile;
-  rv = GetUsageFile(mDirectoryPath, getter_AddRefs(usageFile));
+  rv = GetUsageFile(directoryPath, getter_AddRefs(usageFile));
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -7106,7 +7274,7 @@ nsresult PrepareDatastoreOp::BeginLoadData() {
   }
 
   mConnection = gConnectionThread->CreateConnection(
-      mOrigin, mDirectoryPath, std::move(mArchivedOriginScope),
+      mSuffix, mGroup, mOrigin, std::move(mArchivedOriginScope),
       /* aDatabaseNotAvailable */ false);
   MOZ_ASSERT(mConnection);
 
@@ -7231,7 +7399,7 @@ void PrepareDatastoreOp::GetResponse(LSRequestResponse& aResponse) {
         }
 
         mConnection = gConnectionThread->CreateConnection(
-            mOrigin, mDirectoryPath, std::move(mArchivedOriginScope),
+            mSuffix, mGroup, mOrigin, std::move(mArchivedOriginScope),
             /* aDatabaseNotAvailable */ true);
         MOZ_ASSERT(mConnection);
       }

dom/quota/ActorsParent.cpp

@@ -613,10 +613,20 @@ class OriginInfo final {
 
  public:
   OriginInfo(GroupInfo* aGroupInfo, const nsACString& aOrigin, uint64_t aUsage,
-             int64_t aAccessTime, bool aPersisted);
+             int64_t aAccessTime, bool aPersisted, bool aDirectoryExists);
 
   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(OriginInfo)
 
+  GroupInfo* GetGroupInfo() const { return mGroupInfo; }
+
+  const nsCString& Origin() const { return mOrigin; }
+
+  int64_t LockedUsage() const {
+    AssertCurrentThreadOwnsQuotaMutex();
+
+    return mUsage;
+  }
+
   int64_t LockedAccessTime() const {
     AssertCurrentThreadOwnsQuotaMutex();
 
@@ -654,6 +664,19 @@ class OriginInfo final {
   uint64_t mUsage;
   int64_t mAccessTime;
   bool mPersisted;
+  /**
+   * In some special cases like the LocalStorage client where it's possible to
+   * create a Quota-using representation but not actually write any data, we
+   * want to be able to track quota for an origin without creating its origin
+   * directory or the per-client files until they are actually needed to store
+   * data. In those cases, the OriginInfo will be created by
+   * EnsureQuotaForOrigin and the resulting mDirectoryExists will be false until
+   * the origin actually needs to be created. It is possible for mUsage to be
+   * greater than zero while mDirectoryExists is false, representing a state
+   * where a client like LocalStorage has reserved quota for disk writes, but
+   * has not yet flushed the data to disk.
+   */
+  bool mDirectoryExists;
 };
 
 class OriginInfoLRUComparator {
@@ -689,6 +712,8 @@ class GroupInfo final {
 
   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(GroupInfo)
 
+  PersistenceType GetPersistenceType() const { return mPersistenceType; }
+
  private:
   // Private destructor, to discourage deletion outside of Release():
   ~GroupInfo() { MOZ_COUNT_DTOR(GroupInfo); }
@@ -1124,6 +1149,12 @@ class GetUsageOp final : public QuotaUsageRequestBase,
  private:
   ~GetUsageOp() {}
 
+  void ProcessOriginInternal(QuotaManager* aQuotaManager,
+                             const PersistenceType aPersistenceType,
+                             const nsACString& aOrigin,
+                             const int64_t aTimestamp, const bool aPersisted,
+                             const uint64_t aUsage);
+
   nsresult DoDirectoryWork(QuotaManager* aQuotaManager) override;
 
   bool IsCanceled() override;
@@ -2224,26 +2255,21 @@ nsresult CreateDirectoryMetadataFiles(nsIFile* aDirectory, bool aPersisted,
                                       const nsACString& aSuffix,
                                       const nsACString& aGroup,
                                       const nsACString& aOrigin,
-                                      int64_t* aTimestamp) {
+                                      int64_t aTimestamp) {
   AssertIsOnIOThread();
 
-  int64_t timestamp = PR_Now();
-
   nsresult rv =
-      CreateDirectoryMetadata(aDirectory, timestamp, aSuffix, aGroup, aOrigin);
+      CreateDirectoryMetadata(aDirectory, aTimestamp, aSuffix, aGroup, aOrigin);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
-  rv = CreateDirectoryMetadata2(aDirectory, timestamp, aPersisted, aSuffix,
+  rv = CreateDirectoryMetadata2(aDirectory, aTimestamp, aPersisted, aSuffix,
                                 aGroup, aOrigin);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
-  if (aTimestamp) {
-    *aTimestamp = timestamp;
-  }
   return NS_OK;
 }
 
@@ -3385,6 +3411,28 @@ uint64_t QuotaManager::CollectOriginsForEviction(
   return 0;
 }
 
+template <typename P>
+void QuotaManager::CollectPendingOriginsForListing(P aPredicate) {
+  MutexAutoLock lock(mQuotaMutex);
+
+  for (auto iter = mGroupInfoPairs.Iter(); !iter.Done(); iter.Next()) {
+    GroupInfoPair* pair = iter.UserData();
+
+    MOZ_ASSERT(!iter.Key().IsEmpty());
+    MOZ_ASSERT(pair);
+
+    RefPtr<GroupInfo> groupInfo =
+        pair->LockedGetGroupInfo(PERSISTENCE_TYPE_DEFAULT);
+    if (groupInfo) {
+      for (RefPtr<OriginInfo>& originInfo : groupInfo->mOriginInfos) {
+        if (!originInfo->mDirectoryExists) {
+          aPredicate(originInfo);
+        }
+      }
+    }
+  }
+}
+
 nsresult QuotaManager::Init(const nsAString& aBasePath) {
   mBasePath = aBasePath;
 
@@ -3521,24 +3569,66 @@ void QuotaManager::InitQuotaForOrigin(PersistenceType aPersistenceType,
 
   MutexAutoLock lock(mQuotaMutex);
 
-  GroupInfoPair* pair;
-  if (!mGroupInfoPairs.Get(aGroup, &pair)) {
-    pair = new GroupInfoPair();
-    mGroupInfoPairs.Put(aGroup, pair);
-    // The hashtable is now responsible to delete the GroupInfoPair.
-  }
-
-  RefPtr<GroupInfo> groupInfo = pair->LockedGetGroupInfo(aPersistenceType);
-  if (!groupInfo) {
-    groupInfo = new GroupInfo(pair, aPersistenceType, aGroup);
-    pair->LockedSetGroupInfo(aPersistenceType, groupInfo);
-  }
+  RefPtr<GroupInfo> groupInfo =
+      LockedGetOrCreateGroupInfo(aPersistenceType, aGroup);
 
   RefPtr<OriginInfo> originInfo =
-      new OriginInfo(groupInfo, aOrigin, aUsageBytes, aAccessTime, aPersisted);
+      new OriginInfo(groupInfo, aOrigin, aUsageBytes, aAccessTime, aPersisted,
+                     /* aDirectoryExists */ true);
   groupInfo->LockedAddOriginInfo(originInfo);
 }
 
+void QuotaManager::EnsureQuotaForOrigin(PersistenceType aPersistenceType,
+                                        const nsACString& aGroup,
+                                        const nsACString& aOrigin) {
+  AssertIsOnIOThread();
+  MOZ_ASSERT(aPersistenceType != PERSISTENCE_TYPE_PERSISTENT);
+
+  MutexAutoLock lock(mQuotaMutex);
+
+  RefPtr<GroupInfo> groupInfo =
+      LockedGetOrCreateGroupInfo(aPersistenceType, aGroup);
+
+  RefPtr<OriginInfo> originInfo = groupInfo->LockedGetOriginInfo(aOrigin);
+  if (!originInfo) {
+    originInfo = new OriginInfo(
+        groupInfo, aOrigin, /* aUsageBytes */ 0, /* aAccessTime */ PR_Now(),
+        /* aPersisted */ false, /* aDirectoryExists */ false);
+    groupInfo->LockedAddOriginInfo(originInfo);
+  }
+}
+
+void QuotaManager::NoteOriginDirectoryCreated(PersistenceType aPersistenceType,
+                                              const nsACString& aGroup,
+                                              const nsACString& aOrigin,
+                                              bool aPersisted,
+                                              int64_t& aTimestamp) {
+  AssertIsOnIOThread();
+  MOZ_ASSERT(aPersistenceType != PERSISTENCE_TYPE_PERSISTENT);
+
+  int64_t timestamp;
+
+  MutexAutoLock lock(mQuotaMutex);
+
+  RefPtr<GroupInfo> groupInfo =
+      LockedGetOrCreateGroupInfo(aPersistenceType, aGroup);
+
+  RefPtr<OriginInfo> originInfo = groupInfo->LockedGetOriginInfo(aOrigin);
+  if (originInfo) {
+    originInfo->mPersisted = aPersisted;
+    originInfo->mDirectoryExists = true;
+    timestamp = originInfo->LockedAccessTime();
+  } else {
+    timestamp = PR_Now();
+    RefPtr<OriginInfo> originInfo = new OriginInfo(
+        groupInfo, aOrigin, /* aUsageBytes */ 0, /* aAccessTime */ timestamp,
+        aPersisted, /* aDirectoryExists */ true);
+    groupInfo->LockedAddOriginInfo(originInfo);
+  }
+
+  aTimestamp = timestamp;
+}
+
 void QuotaManager::DecreaseUsageForOrigin(PersistenceType aPersistenceType,
                                           const nsACString& aGroup,
                                           const nsACString& aOrigin,
@@ -5685,9 +5775,11 @@ nsresult QuotaManager::EnsureOriginIsInitializedInternal(
   int64_t timestamp;
   if (aPersistenceType == PERSISTENCE_TYPE_PERSISTENT) {
     if (created) {
+      timestamp = PR_Now();
+
       rv = CreateDirectoryMetadataFiles(directory,
                                         /* aPersisted */ true, aSuffix, aGroup,
-                                        aOrigin, &timestamp);
+                                        aOrigin, timestamp);
       if (NS_WARN_IF(NS_FAILED(rv))) {
         return rv;
       }
@@ -5708,17 +5800,15 @@ nsresult QuotaManager::EnsureOriginIsInitializedInternal(
 
     mInitializedOrigins.AppendElement(aOrigin);
   } else if (created) {
+    NoteOriginDirectoryCreated(aPersistenceType, aGroup, aOrigin,
+                               /* aPersisted */ false, timestamp);
+
     rv = CreateDirectoryMetadataFiles(directory,
                                       /* aPersisted */ false, aSuffix, aGroup,
-                                      aOrigin, &timestamp);
+                                      aOrigin, timestamp);
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
-
-    // Don't need to traverse the directory, since it's empty.
-    InitQuotaForOrigin(aPersistenceType, aGroup, aOrigin,
-                       /* aUsageBytes */ 0, timestamp,
-                       /* aPersisted */ false);
   }
 
   directory.forget(aDirectory);
@@ -6256,6 +6346,27 @@ void QuotaManager::LockedRemoveQuotaForOrigin(PersistenceType aPersistenceType,
   }
 }
 
+already_AddRefed<GroupInfo> QuotaManager::LockedGetOrCreateGroupInfo(
+    PersistenceType aPersistenceType, const nsACString& aGroup) {
+  mQuotaMutex.AssertCurrentThreadOwns();
+  MOZ_ASSERT(aPersistenceType != PERSISTENCE_TYPE_PERSISTENT);
+
+  GroupInfoPair* pair;
+  if (!mGroupInfoPairs.Get(aGroup, &pair)) {
+    pair = new GroupInfoPair();
+    mGroupInfoPairs.Put(aGroup, pair);
+    // The hashtable is now responsible to delete the GroupInfoPair.
+  }
+
+  RefPtr<GroupInfo> groupInfo = pair->LockedGetGroupInfo(aPersistenceType);
+  if (!groupInfo) {
+    groupInfo = new GroupInfo(pair, aPersistenceType, aGroup);
+    pair->LockedSetGroupInfo(aPersistenceType, groupInfo);
+  }
+
+  return groupInfo.forget();
+}
+
 already_AddRefed<OriginInfo> QuotaManager::LockedGetOriginInfo(
     PersistenceType aPersistenceType, const nsACString& aGroup,
     const nsACString& aOrigin) {
@@ -6511,12 +6622,14 @@ bool QuotaManager::IsSanitizedOriginValid(const nsACString& aSanitizedOrigin) {
  ******************************************************************************/
 
 OriginInfo::OriginInfo(GroupInfo* aGroupInfo, const nsACString& aOrigin,
-                       uint64_t aUsage, int64_t aAccessTime, bool aPersisted)
+                       uint64_t aUsage, int64_t aAccessTime, bool aPersisted,
+                       bool aDirectoryExists)
     : mGroupInfo(aGroupInfo),
       mOrigin(aOrigin),
       mUsage(aUsage),
       mAccessTime(aAccessTime),
-      mPersisted(aPersisted) {
+      mPersisted(aPersisted),
+      mDirectoryExists(aDirectoryExists) {
   MOZ_ASSERT(aGroupInfo);
   MOZ_ASSERT_IF(aPersisted,
                 aGroupInfo->mPersistenceType == PERSISTENCE_TYPE_DEFAULT);
@@ -7619,6 +7732,47 @@ GetUsageOp::GetUsageOp(const UsageRequestParams& aParams)
   MOZ_ASSERT(aParams.type() == UsageRequestParams::TAllUsageParams);
 }
 
+void GetUsageOp::ProcessOriginInternal(QuotaManager* aQuotaManager,
+                                       const PersistenceType aPersistenceType,
+                                       const nsACString& aOrigin,
+                                       const int64_t aTimestamp,
+                                       const bool aPersisted,
+                                       const uint64_t aUsage) {
+  if (!mGetAll && aQuotaManager->IsOriginInternal(aOrigin)) {
+    return;
+  }
+
+  OriginUsage* originUsage;
+
+  // We can't store pointers to OriginUsage objects in the hashtable
+  // since AppendElement() reallocates its internal array buffer as number
+  // of elements grows.
+  uint32_t index;
+  if (mOriginUsagesIndex.Get(aOrigin, &index)) {
+    originUsage = &mOriginUsages[index];
+  } else {
+    index = mOriginUsages.Length();
+
+    originUsage = mOriginUsages.AppendElement();
+
+    originUsage->origin() = aOrigin;
+    originUsage->persisted() = false;
+    originUsage->usage() = 0;
+    originUsage->lastAccessed() = 0;
+
+    mOriginUsagesIndex.Put(aOrigin, index);
+  }
+
+  if (aPersistenceType == PERSISTENCE_TYPE_DEFAULT) {
+    originUsage->persisted() = aPersisted;
+  }
+
+  originUsage->usage() = originUsage->usage() + aUsage;
+
+  originUsage->lastAccessed() =
+      std::max<int64_t>(originUsage->lastAccessed(), aTimestamp);
+}
+
 bool GetUsageOp::IsCanceled() {
   AssertIsOnIOThread();
 
@@ -7643,36 +7797,6 @@ nsresult GetUsageOp::ProcessOrigin(QuotaManager* aQuotaManager,
     return rv;
   }
 
-  if (!mGetAll && aQuotaManager->IsOriginInternal(origin)) {
-    return NS_OK;
-  }
-
-  OriginUsage* originUsage;
-
-  // We can't store pointers to OriginUsage objects in the hashtable
-  // since AppendElement() reallocates its internal array buffer as number
-  // of elements grows.
-  uint32_t index;
-  if (mOriginUsagesIndex.Get(origin, &index)) {
-    originUsage = &mOriginUsages[index];
-  } else {
-    index = mOriginUsages.Length();
-
-    originUsage = mOriginUsages.AppendElement();
-
-    originUsage->origin() = origin;
-    originUsage->persisted() = false;
-    originUsage->usage() = 0;
-
-    mOriginUsagesIndex.Put(origin, index);
-  }
-
-  if (aPersistenceType == PERSISTENCE_TYPE_DEFAULT) {
-    originUsage->persisted() = persisted;
-  }
-
-  originUsage->lastAccessed() = timestamp;
-
   UsageInfo usageInfo;
   rv = GetUsageForOrigin(aQuotaManager, aPersistenceType, group, origin,
                          &usageInfo);
@@ -7680,7 +7804,8 @@ nsresult GetUsageOp::ProcessOrigin(QuotaManager* aQuotaManager,
     return rv;
   }
 
-  originUsage->usage() = originUsage->usage() + usageInfo.TotalUsage();
+  ProcessOriginInternal(aQuotaManager, aPersistenceType, origin, timestamp,
+                        persisted, usageInfo.TotalUsage());
 
   return NS_OK;
 }
@@ -7699,6 +7824,18 @@ nsresult GetUsageOp::DoDirectoryWork(QuotaManager* aQuotaManager) {
     }
   }
 
+  // TraverseRepository above only consulted the filesystem. We also need to
+  // consider origins which may have pending quota usage, such as buffered
+  // LocalStorage writes for an origin which didn't previously have any
+  // LocalStorage data.
+
+  aQuotaManager->CollectPendingOriginsForListing([&](OriginInfo* aOriginInfo) {
+    ProcessOriginInternal(
+        aQuotaManager, aOriginInfo->GetGroupInfo()->GetPersistenceType(),
+        aOriginInfo->Origin(), aOriginInfo->LockedAccessTime(),
+        aOriginInfo->LockedPersisted(), aOriginInfo->LockedUsage());
+  });
+
   return NS_OK;
 }
 
@@ -8434,20 +8571,22 @@ nsresult PersistOp::DoDirectoryWork(QuotaManager* aQuotaManager) {
 
   if (created) {
     int64_t timestamp;
-    rv = CreateDirectoryMetadataFiles(directory,
-                                      /* aPersisted */ true, mSuffix, mGroup,
-                                      mOriginScope.GetOrigin(), &timestamp);
-    if (NS_WARN_IF(NS_FAILED(rv))) {
-      return rv;
-    }
 
-    // Directory metadata has been successfully created.
+    // Origin directory has been successfully created.
     // Create OriginInfo too if temporary storage was already initialized.
     if (aQuotaManager->IsTemporaryStorageInitialized()) {
-      aQuotaManager->InitQuotaForOrigin(mPersistenceType.Value(), mGroup,
-                                        mOriginScope.GetOrigin(),
-                                        /* aUsageBytes */ 0, timestamp,
-                                        /* aPersisted */ true);
+      aQuotaManager->NoteOriginDirectoryCreated(
+          mPersistenceType.Value(), mGroup, mOriginScope.GetOrigin(),
+          /* aPersisted */ true, timestamp);
+    } else {
+      timestamp = PR_Now();
+    }
+
+    rv = CreateDirectoryMetadataFiles(directory,
+                                      /* aPersisted */ true, mSuffix, mGroup,
+                                      mOriginScope.GetOrigin(), timestamp);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
     }
   } else {
     // Get the persisted flag (restore the metadata file if necessary).
@@ -8542,6 +8681,14 @@ nsresult ListInitializedOriginsOp::DoDirectoryWork(
     }
   }
 
+  // TraverseRepository above only consulted the file-system to get a list of
+  // known origins, but we also need to include origins that have pending quota
+  // usage.
+
+  aQuotaManager->CollectPendingOriginsForListing([&](OriginInfo* aOriginInfo) {
+    mOrigins.AppendElement(aOriginInfo->Origin());
+  });
+
   return NS_OK;
 }
 

dom/quota/QuotaManager.h

@@ -145,11 +145,41 @@ class QuotaManager final : public BackgroundThreadObject {
     return mTemporaryStorageInitialized;
   }
 
+  /**
+   * For initialization of an origin where the directory already exists. This is
+   * used by EnsureTemporaryStorageIsInitialized/InitializeRepository once it
+   * has tallied origin usage by calling each of the QuotaClient InitOrigin
+   * methods.
+   */
   void InitQuotaForOrigin(PersistenceType aPersistenceType,
                           const nsACString& aGroup, const nsACString& aOrigin,
                           uint64_t aUsageBytes, int64_t aAccessTime,
                           bool aPersisted);
 
+  /**
+   * For use in special-cases like LSNG where we need to be able to know that
+   * there is no data stored for an origin. LSNG knows that there is 0 usage for
+   * its storage of an origin and wants to make sure there is a QuotaObject
+   * tracking this. This method will create a non-persisted, 0-usage,
+   * mDirectoryExists=false OriginInfo if there isn't already an OriginInfo. If
+   * an OriginInfo already exists, it will be left as-is, because that implies a
+   * different client has usages for the origin (and there's no need to add
+   * LSNG's 0 usage to the QuotaObject).
+   */
+  void EnsureQuotaForOrigin(PersistenceType aPersistenceType,
+                            const nsACString& aGroup,
+                            const nsACString& aOrigin);
+
+  /**
+   * For use when creating an origin directory. It's possible that origin usage
+   * is already being tracked due to a call to EnsureQuotaForOrigin, and in that
+   * case we need to update the existing OriginInfo rather than create a new one.
+   */
+  void NoteOriginDirectoryCreated(PersistenceType aPersistenceType,
+                                  const nsACString& aGroup,
+                                  const nsACString& aOrigin, bool aPersisted,
+                                  int64_t& aTimestamp);
+
   void DecreaseUsageForOrigin(PersistenceType aPersistenceType,
                               const nsACString& aGroup,
                               const nsACString& aOrigin, int64_t aSize);
@@ -248,6 +278,19 @@ class QuotaManager final : public BackgroundThreadObject {
   uint64_t CollectOriginsForEviction(
       uint64_t aMinSizeToBeFreed, nsTArray<RefPtr<DirectoryLockImpl>>& aLocks);
 
+  /**
+   * Helper method to invoke the provided predicate on all "pending" OriginInfo
+   * instances. These are origins for which the origin directory has not yet
+   * been created but for which quota is already being tracked. This happens,
+   * for example, for the LocalStorage client where an origin that previously
+   * was not using LocalStorage can start issuing writes which it buffers until
+   * eventually flushing them. We defer creating the origin directory for as
+   * long as possible in that case, so the directory won't exist. Logic that
+   * would otherwise only consult the filesystem also needs to use this method.
+   */
+  template <typename P>
+  void CollectPendingOriginsForListing(P aPredicate);
+
   void AssertStorageIsInitialized() const
 #ifdef DEBUG
       ;
@@ -399,6 +442,9 @@ class QuotaManager final : public BackgroundThreadObject {
                                   const nsACString& aGroup,
                                   const nsACString& aOrigin);
 
+  already_AddRefed<GroupInfo> LockedGetOrCreateGroupInfo(
+      PersistenceType aPersistenceType, const nsACString& aGroup);
+
   already_AddRefed<OriginInfo> LockedGetOriginInfo(
       PersistenceType aPersistenceType, const nsACString& aGroup,
       const nsACString& aOrigin);