Backed out 2 changesets (bug 1729717) for causing assertion failures on CacheObserver.cpp. CLOSED TREE

Backed out changeset 7123efe825f0 (bug 1729717)
Backed out changeset 92e06bd625c9 (bug 1729717)

netwerk/cache/moz.build

@@ -7,8 +7,13 @@
 with Files("**"):
     BUG_COMPONENT = ("Core", "Networking: Cache")
 
+EXPORTS += [
+    "nsDeleteDir.h",
+]
+
 UNIFIED_SOURCES += [
     "nsCacheUtils.cpp",
+    "nsDeleteDir.cpp",
 ]
 
 FINAL_LIBRARY = "xul"

netwerk/cache/nsDeleteDir.cpp

@@ -0,0 +1,360 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsDeleteDir.h"
+#include "nsIFile.h"
+#include "nsString.h"
+#include "mozilla/Telemetry.h"
+#include "nsITimer.h"
+#include "nsThreadUtils.h"
+#include "nsISupportsPriority.h"
+#include "nsCacheUtils.h"
+#include "prtime.h"
+#include <time.h>
+
+using namespace mozilla;
+
+class nsBlockOnBackgroundThreadEvent : public Runnable {
+ public:
+  nsBlockOnBackgroundThreadEvent()
+      : mozilla::Runnable("nsBlockOnBackgroundThreadEvent") {}
+  NS_IMETHOD Run() override {
+    MutexAutoLock lock(nsDeleteDir::gInstance->mLock);
+    nsDeleteDir::gInstance->mNotified = true;
+    nsDeleteDir::gInstance->mCondVar.Notify();
+    return NS_OK;
+  }
+};
+
+nsDeleteDir* nsDeleteDir::gInstance = nullptr;
+
+nsDeleteDir::nsDeleteDir()
+    : mLock("nsDeleteDir.mLock"), mCondVar(mLock, "nsDeleteDir.mCondVar") {
+  NS_ASSERTION(gInstance == nullptr, "multiple nsCacheService instances!");
+}
+
+nsDeleteDir::~nsDeleteDir() { gInstance = nullptr; }
+
+nsresult nsDeleteDir::Init() {
+  if (gInstance) return NS_ERROR_ALREADY_INITIALIZED;
+
+  gInstance = new nsDeleteDir();
+  return NS_OK;
+}
+
+nsresult nsDeleteDir::Shutdown(bool finishDeleting) {
+  if (!gInstance) return NS_ERROR_NOT_INITIALIZED;
+
+  nsCOMArray<nsIFile> dirsToRemove;
+  nsCOMPtr<nsISerialEventTarget> eventTarget;
+  {
+    MutexAutoLock lock(gInstance->mLock);
+    NS_ASSERTION(!gInstance->mShutdownPending,
+                 "Unexpected state in nsDeleteDir::Shutdown()");
+    gInstance->mShutdownPending = true;
+
+    if (!finishDeleting) gInstance->mStopDeleting = true;
+
+    // remove all pending timers
+    for (int32_t i = gInstance->mTimers.Count(); i > 0; i--) {
+      nsCOMPtr<nsITimer> timer = gInstance->mTimers[i - 1];
+      gInstance->mTimers.RemoveObjectAt(i - 1);
+
+      nsCOMArray<nsIFile>* arg;
+      timer->GetClosure((reinterpret_cast<void**>(&arg)));
+      timer->Cancel();
+
+      if (finishDeleting) dirsToRemove.AppendObjects(*arg);
+
+      // delete argument passed to the timer
+      delete arg;
+    }
+
+    eventTarget.swap(gInstance->mBackgroundET);
+    if (eventTarget) {
+      // dispatch event and wait for it to run and notify us, so we know thread
+      // has completed all work and can be shutdown
+      nsCOMPtr<nsIRunnable> event = new nsBlockOnBackgroundThreadEvent();
+      nsresult rv = eventTarget->Dispatch(event, NS_DISPATCH_EVENT_MAY_BLOCK);
+      if (NS_FAILED(rv)) {
+        NS_WARNING("Failed dispatching block-event");
+        return NS_ERROR_UNEXPECTED;
+      }
+
+      gInstance->mNotified = false;
+      while (!gInstance->mNotified) {
+        gInstance->mCondVar.Wait();
+      }
+    }
+  }
+
+  delete gInstance;
+
+  for (int32_t i = 0; i < dirsToRemove.Count(); i++) {
+    dirsToRemove[i]->Remove(true);
+  }
+
+  return NS_OK;
+}
+
+nsresult nsDeleteDir::InitThread() {
+  if (mBackgroundET) return NS_OK;
+
+  nsresult rv = NS_CreateBackgroundTaskQueue("Cache Deleter",
+                                             getter_AddRefs(mBackgroundET));
+  if (NS_FAILED(rv)) {
+    NS_WARNING("Can't create background task queue");
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+void nsDeleteDir::DestroyThread() {
+  if (!mBackgroundET) return;
+
+  if (mTimers.Count()) {
+    // more work to do, so don't delete thread.
+    return;
+  }
+
+  mBackgroundET = nullptr;
+}
+
+void nsDeleteDir::TimerCallback(nsITimer* aTimer, void* arg) {
+  Telemetry::AutoTimer<Telemetry::NETWORK_DISK_CACHE_DELETEDIR> timer;
+  {
+    MutexAutoLock lock(gInstance->mLock);
+
+    int32_t idx = gInstance->mTimers.IndexOf(aTimer);
+    if (idx == -1) {
+      // Timer was canceled and removed during shutdown.
+      return;
+    }
+
+    gInstance->mTimers.RemoveObjectAt(idx);
+  }
+
+  UniquePtr<nsCOMArray<nsIFile>> dirList;
+  dirList.reset(static_cast<nsCOMArray<nsIFile>*>(arg));
+
+  bool shuttingDown = false;
+
+  // Intentional extra braces to control variable sope.
+  {
+    // Low IO priority can only be set when running in the context of the
+    // current thread.  So this shouldn't be moved to where we set the priority
+    // of the Cache deleter thread using the nsThread's NSPR priority constants.
+    nsAutoLowPriorityIO autoLowPriority;
+    for (int32_t i = 0; i < dirList->Count() && !shuttingDown; i++) {
+      gInstance->RemoveDir((*dirList)[i], &shuttingDown);
+    }
+  }
+
+  {
+    MutexAutoLock lock(gInstance->mLock);
+    gInstance->DestroyThread();
+  }
+}
+
+nsresult nsDeleteDir::DeleteDir(nsIFile* dirIn, bool moveToTrash,
+                                uint32_t delay) {
+  Telemetry::AutoTimer<Telemetry::NETWORK_DISK_CACHE_TRASHRENAME> timer;
+
+  if (!gInstance) return NS_ERROR_NOT_INITIALIZED;
+
+  nsresult rv;
+  nsCOMPtr<nsIFile> trash, dir;
+
+  // Need to make a clone of this since we don't want to modify the input
+  // file object.
+  rv = dirIn->Clone(getter_AddRefs(dir));
+  if (NS_FAILED(rv)) return rv;
+
+  if (moveToTrash) {
+    rv = GetTrashDir(dir, &trash);
+    if (NS_FAILED(rv)) return rv;
+    nsAutoCString origLeaf;
+    rv = trash->GetNativeLeafName(origLeaf);
+    if (NS_FAILED(rv)) return rv;
+
+    // Append random number to the trash directory and check if it exists.
+    srand(static_cast<unsigned>(PR_Now()));
+    nsAutoCString leaf;
+    for (int32_t i = 0; i < 10; i++) {
+      leaf = origLeaf;
+      leaf.AppendInt(rand());
+      rv = trash->SetNativeLeafName(leaf);
+      if (NS_FAILED(rv)) return rv;
+
+      bool exists;
+      if (NS_SUCCEEDED(trash->Exists(&exists)) && !exists) {
+        break;
+      }
+
+      leaf.Truncate();
+    }
+
+    // Fail if we didn't find unused trash directory within the limit
+    if (!leaf.Length()) return NS_ERROR_FAILURE;
+
+#if defined(MOZ_WIDGET_ANDROID)
+    nsCOMPtr<nsIFile> parent;
+    rv = trash->GetParent(getter_AddRefs(parent));
+    if (NS_FAILED(rv)) return rv;
+    rv = dir->MoveToNative(parent, leaf);
+#else
+    // Important: must rename directory w/o changing parent directory: else on
+    // NTFS we'll wait (with cache lock) while nsIFile's ACL reset walks file
+    // tree: was hanging GUI for *minutes* on large cache dirs.
+    rv = dir->MoveToNative(nullptr, leaf);
+#endif
+    if (NS_FAILED(rv)) return rv;
+  } else {
+    // we want to pass a clone of the original off to the worker thread.
+    trash.swap(dir);
+  }
+
+  UniquePtr<nsCOMArray<nsIFile>> arg(new nsCOMArray<nsIFile>);
+  arg->AppendObject(trash);
+
+  rv = gInstance->PostTimer(arg.get(), delay);
+  if (NS_FAILED(rv)) return rv;
+
+  Unused << arg.release();
+  return NS_OK;
+}
+
+nsresult nsDeleteDir::GetTrashDir(nsIFile* target, nsCOMPtr<nsIFile>* result) {
+  nsresult rv;
+#if defined(MOZ_WIDGET_ANDROID)
+  // Try to use the app cache folder for cache trash on Android
+  char* cachePath = getenv("CACHE_DIRECTORY");
+  if (cachePath) {
+    rv = NS_NewNativeLocalFile(nsDependentCString(cachePath), true,
+                               getter_AddRefs(*result));
+    if (NS_FAILED(rv)) return rv;
+
+    // Add a sub folder with the cache folder name
+    nsAutoCString leaf;
+    rv = target->GetNativeLeafName(leaf);
+    (*result)->AppendNative(leaf);
+  } else
+#endif
+  {
+    rv = target->Clone(getter_AddRefs(*result));
+  }
+  if (NS_FAILED(rv)) return rv;
+
+  nsAutoCString leaf;
+  rv = (*result)->GetNativeLeafName(leaf);
+  if (NS_FAILED(rv)) return rv;
+  leaf.AppendLiteral(".Trash");
+
+  return (*result)->SetNativeLeafName(leaf);
+}
+
+nsresult nsDeleteDir::RemoveOldTrashes(nsIFile* cacheDir) {
+  if (!gInstance) return NS_ERROR_NOT_INITIALIZED;
+
+  nsresult rv;
+
+  nsCOMPtr<nsIFile> trash;
+  rv = GetTrashDir(cacheDir, &trash);
+  if (NS_FAILED(rv)) return rv;
+
+  nsAutoString trashName;
+  rv = trash->GetLeafName(trashName);
+  if (NS_FAILED(rv)) return rv;
+
+  nsCOMPtr<nsIFile> parent;
+#if defined(MOZ_WIDGET_ANDROID)
+  rv = trash->GetParent(getter_AddRefs(parent));
+#else
+  rv = cacheDir->GetParent(getter_AddRefs(parent));
+#endif
+  if (NS_FAILED(rv)) return rv;
+
+  nsCOMPtr<nsIDirectoryEnumerator> iter;
+  rv = parent->GetDirectoryEntries(getter_AddRefs(iter));
+  if (NS_FAILED(rv)) return rv;
+
+  UniquePtr<nsCOMArray<nsIFile>> dirList;
+
+  nsCOMPtr<nsIFile> file;
+  while (NS_SUCCEEDED(iter->GetNextFile(getter_AddRefs(file))) && file) {
+    nsAutoString leafName;
+    rv = file->GetLeafName(leafName);
+    if (NS_FAILED(rv)) continue;
+
+    // match all names that begin with the trash name (i.e. "Cache.Trash")
+    if (Substring(leafName, 0, trashName.Length()).Equals(trashName)) {
+      if (!dirList) dirList = MakeUnique<nsCOMArray<nsIFile>>();
+      dirList->AppendObject(file);
+    }
+  }
+
+  if (dirList) {
+    rv = gInstance->PostTimer(dirList.get(), 90000);
+    if (NS_FAILED(rv)) return rv;
+
+    Unused << dirList.release();
+  }
+
+  return NS_OK;
+}
+
+nsresult nsDeleteDir::PostTimer(void* arg, uint32_t delay) {
+  nsresult rv;
+
+  MutexAutoLock lock(mLock);
+
+  rv = InitThread();
+  if (NS_FAILED(rv)) return rv;
+
+  nsCOMPtr<nsITimer> timer;
+  rv = NS_NewTimerWithFuncCallback(getter_AddRefs(timer), TimerCallback, arg,
+                                   delay, nsITimer::TYPE_ONE_SHOT,
+                                   "nsDeleteDir::PostTimer", mBackgroundET);
+  if (NS_FAILED(rv)) return rv;
+
+  mTimers.AppendObject(timer);
+  return NS_OK;
+}
+
+nsresult nsDeleteDir::RemoveDir(nsIFile* file, bool* stopDeleting) {
+  nsresult rv;
+  bool isLink;
+
+  rv = file->IsSymlink(&isLink);
+  if (NS_FAILED(rv) || isLink) return NS_ERROR_UNEXPECTED;
+
+  bool isDir;
+  rv = file->IsDirectory(&isDir);
+  if (NS_FAILED(rv)) return rv;
+
+  if (isDir) {
+    nsCOMPtr<nsIDirectoryEnumerator> iter;
+    rv = file->GetDirectoryEntries(getter_AddRefs(iter));
+    if (NS_FAILED(rv)) return rv;
+
+    nsCOMPtr<nsIFile> file2;
+    while (NS_SUCCEEDED(iter->GetNextFile(getter_AddRefs(file2))) && file2) {
+      RemoveDir(file2, stopDeleting);
+      // No check for errors to remove as much as possible
+
+      if (*stopDeleting) return NS_OK;
+    }
+  }
+
+  file->Remove(false);
+  // No check for errors to remove as much as possible
+
+  MutexAutoLock lock(mLock);
+  if (mStopDeleting) *stopDeleting = true;
+
+  return NS_OK;
+}

netwerk/cache/nsDeleteDir.h

@@ -0,0 +1,79 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef nsDeleteDir_h__
+#define nsDeleteDir_h__
+
+#include "nsCOMPtr.h"
+#include "nsCOMArray.h"
+#include "mozilla/Mutex.h"
+#include "mozilla/CondVar.h"
+
+class nsIFile;
+class nsISerialEventTarget;
+class nsITimer;
+
+class nsDeleteDir {
+ public:
+  nsDeleteDir();
+  ~nsDeleteDir();
+
+  static nsresult Init();
+  static nsresult Shutdown(bool finishDeleting);
+
+  /**
+   * This routine attempts to delete a directory that may contain some files
+   * that are still in use. This latter point is only an issue on Windows and
+   * a few other systems.
+   *
+   * If the moveToTrash parameter is true we first rename the given directory
+   * "foo.Trash123" (where "foo" is the original directory name, and "123" is
+   * a random number, in order to support multiple concurrent deletes). The
+   * directory is then deleted, file-by-file, on a background thread.
+   *
+   * If the moveToTrash parameter is false, then the given directory is deleted
+   * directly.
+   *
+   * If 'delay' is non-zero, the directory will not be deleted until the
+   * specified number of milliseconds have passed. (The directory is still
+   * renamed immediately if 'moveToTrash' is passed, so upon return it is safe
+   * to create a directory with the same name).
+   */
+  static nsresult DeleteDir(nsIFile* dir, bool moveToTrash, uint32_t delay = 0);
+
+  /**
+   * Returns the trash directory corresponding to the given directory.
+   */
+  static nsresult GetTrashDir(nsIFile* target, nsCOMPtr<nsIFile>* result);
+
+  /**
+   * Remove all trashes left from previous run. This function does nothing when
+   * called second and more times.
+   */
+  static nsresult RemoveOldTrashes(nsIFile* cacheDir);
+
+  static void TimerCallback(nsITimer* aTimer, void* arg);
+
+ private:
+  friend class nsBlockOnBackgroundThreadEvent;
+  friend class nsDestroyThreadEvent;
+
+  nsresult InitThread();
+  void DestroyThread();
+  nsresult PostTimer(void* arg, uint32_t delay);
+  nsresult RemoveDir(nsIFile* file, bool* stopDeleting);
+
+  static nsDeleteDir* gInstance;
+  mozilla::Mutex mLock;
+  mozilla::CondVar mCondVar;
+  bool mNotified{false};
+  nsCOMArray<nsITimer> mTimers;
+  nsCOMPtr<nsISerialEventTarget> mBackgroundET;
+  bool mShutdownPending{false};
+  bool mStopDeleting{false};
+};
+
+#endif  // nsDeleteDir_h__

netwerk/cache2/CacheObserver.cpp

@@ -208,6 +208,11 @@ CacheObserver::Observe(nsISupports* aSubject, const char* aTopic,
     return NS_OK;
   }
 
+  if (!strcmp(aTopic, "browser-delayed-startup-finished")) {
+    CacheStorageService::CleaupCacheDirectories();
+    return NS_OK;
+  }
+
   if (!strcmp(aTopic, "profile-change-net-teardown") ||
       !strcmp(aTopic, "profile-before-change") ||
       !strcmp(aTopic, "xpcom-shutdown")) {

netwerk/cache2/CacheStorageService.cpp

@@ -14,6 +14,8 @@
 #include "CacheEntry.h"
 #include "CacheFileUtils.h"
 
+#include "nsDeleteDir.h"
+
 #include "nsICacheStorageVisitor.h"
 #include "nsIObserverService.h"
 #include "nsIFile.h"
@@ -551,6 +553,72 @@ void CacheStorageService::DropPrivateBrowsingEntries() {
   }
 }
 
+namespace {
+
+class CleaupCacheDirectoriesRunnable : public Runnable {
+ public:
+  NS_DECL_NSIRUNNABLE
+  static bool Post();
+
+ private:
+  CleaupCacheDirectoriesRunnable()
+      : Runnable("net::CleaupCacheDirectoriesRunnable") {
+    CacheFileIOManager::GetCacheDirectory(getter_AddRefs(mCache2Dir));
+#if defined(MOZ_WIDGET_ANDROID)
+    CacheFileIOManager::GetProfilelessCacheDirectory(
+        getter_AddRefs(mCache2Profileless));
+#endif
+  }
+
+  virtual ~CleaupCacheDirectoriesRunnable() = default;
+  nsCOMPtr<nsIFile> mCache1Dir, mCache2Dir;
+#if defined(MOZ_WIDGET_ANDROID)
+  nsCOMPtr<nsIFile> mCache2Profileless;
+#endif
+};
+
+// static
+bool CleaupCacheDirectoriesRunnable::Post() {
+  // TODO: initialize nsDeleteDir
+  return true;
+}
+
+NS_IMETHODIMP CleaupCacheDirectoriesRunnable::Run() {
+  MOZ_ASSERT(!NS_IsMainThread());
+
+  if (mCache1Dir) {
+    nsDeleteDir::RemoveOldTrashes(mCache1Dir);
+  }
+  if (mCache2Dir) {
+    nsDeleteDir::RemoveOldTrashes(mCache2Dir);
+  }
+#if defined(MOZ_WIDGET_ANDROID)
+  if (mCache2Profileless) {
+    nsDeleteDir::RemoveOldTrashes(mCache2Profileless);
+    // Always delete the profileless cache on Android
+    nsDeleteDir::DeleteDir(mCache2Profileless, true, 30000);
+  }
+#endif
+
+  if (mCache1Dir) {
+    nsDeleteDir::DeleteDir(mCache1Dir, true, 30000);
+  }
+
+  return NS_OK;
+}
+
+}  // namespace
+
+// static
+void CacheStorageService::CleaupCacheDirectories() {
+  // Make sure we schedule just once in case CleaupCacheDirectories gets called
+  // multiple times from some reason.
+  static bool runOnce = CleaupCacheDirectoriesRunnable::Post();
+  if (!runOnce) {
+    NS_WARNING("Could not start cache trashes cleanup");
+  }
+}
+
 // Helper methods
 
 // static

netwerk/cache2/CacheStorageService.h

@@ -90,6 +90,10 @@ class CacheStorageService final : public nsICacheStorageService,
   void Shutdown();
   void DropPrivateBrowsingEntries();
 
+  // Takes care of deleting any pending trashes for both cache1 and cache2
+  // as well as old cache directory.
+  static void CleaupCacheDirectories();
+
   static CacheStorageService* Self() { return sSelf; }
   static nsISupports* SelfISupports() {
     return static_cast<nsICacheStorageService*>(Self());