Bug 1878541 - Add migration code to delete nstmp files left over from failed folder compactions. r=darktrojan,freaktechnik

Differential Revision: https://phabricator.services.mozilla.com/D214916

--HG--
extra : moz-landing-system : lando

mail/components/MailGlue.sys.mjs

@@ -890,6 +890,8 @@ MailGlue.prototype = {
    */
   _scheduleBestEffortUserIdleTasks() {
     const idleTasks = [
+      // Migration work that needs happen after we're up and running.
+      () => lazy.MailMigrator.migrateAfterStartupComplete(),
       // Certificates revocation list, etc.
       () => lazy.RemoteSecuritySettings.init(),
       // If we haven't already, ensure the address book manager is ready.

mail/modules/MailMigrator.sys.mjs

@@ -211,6 +211,78 @@ export var MailMigrator = {
     }
   },
 
+  /**
+   * Scan through a profile, removing 'nstmp' / 'nstmp-N'
+   * files left over from failed folder compactions.
+   * See Bug 1878541.
+   */
+  async _nstmpCleanup() {
+    // Latch to ensure this only ever runs once.
+    if (Services.prefs.getBoolPref("mail.nstmp_cleanup_completed", false)) {
+      return;
+    }
+
+    const logger = console.createInstance({
+      prefix: "nstmp cleanup",
+      maxLogLevel: "Log",
+    });
+    logger.log("Looking for left-over nstmp files to remove...");
+
+    // Go through all known folders, building up a list of the directories
+    // and all the potential mbox files in those directories.
+    // Each entry is a set of the potential mbox filenames in the dir.
+    const dirs = {};
+    for (const s of MailServices.accounts.allServers) {
+      if (s.msgStore.storeType != "mbox") {
+        continue;
+      }
+      // Don't process the root folder here (it shouldn't have an mbox).
+      for (const child of s.rootFolder.descendants) {
+        const mbox = child.filePath.path;
+        const d = PathUtils.parent(mbox);
+        if (!Object.hasOwn(dirs, d)) {
+          dirs[d] = new Set();
+        }
+        // We'll be doing case-insensitive compares.
+        dirs[d].add(PathUtils.filename(mbox).toLowerCase());
+      }
+    }
+
+    // For each directory, find nstmp files, excluding names of known folders.
+    const doomed = [];
+    for (const [dir, mboxes] of Object.entries(dirs)) {
+      const files = await IOUtils.getChildren(dir, { ignoreAbsent: true });
+      for (const file of files) {
+        // Skip anything that isn't a regular file.
+        const info = await IOUtils.stat(file);
+        if (info.type != "regular") {
+          continue;
+        }
+
+        // Looks like an nstmp file? (as created by createUnique()).
+        const bare = PathUtils.filename(file);
+        if (/^nstmp(-[0-9]{1,4})?$/.test(bare)) {
+          // Make sure it doesn't match any of the potential mbox files (case
+          // insensitive).
+          if (mboxes.has(bare.toLowerCase())) {
+            continue;
+          }
+          doomed.push(file);
+        }
+      }
+    }
+
+    if (doomed.length > 0) {
+      logger.log("Found left-over nstmp files to remove:", doomed);
+    }
+    for (const f of doomed) {
+      await IOUtils.remove(f);
+    }
+
+    Services.prefs.setBoolPref("mail.nstmp_cleanup_completed", true);
+    logger.log(`nstmp cleanup completed: ${doomed.length} files removed.`);
+  },
+
   /**
    * Perform any migration work that needs to occur once the user profile has
    * been loaded.
@@ -218,6 +290,14 @@ export var MailMigrator = {
   migrateAtProfileStartup() {
     this._migrateUI();
   },
+
+  /**
+   * Perform any migration work that needs to occur once everything is up and
+   * running.
+   */
+  async migrateAfterStartupComplete() {
+    await this._nstmpCleanup();
+  },
 };
 
 /**

mailnews/imap/test/unit/test_nstmpCleanup.js

@@ -0,0 +1,154 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * Test that nstmpCleanup() works. It's an idle task (see MailGlue.sys.mjs)
+ * which removes nstmp (and nstmp-NNNN) files left over from old, failed
+ * folder compaction operations.
+ */
+
+const { MessageGenerator } = ChromeUtils.importESModule(
+  "resource://testing-common/mailnews/MessageGenerator.sys.mjs"
+);
+
+const { MailMigrator } = ChromeUtils.importESModule(
+  "resource:///modules/MailMigrator.sys.mjs"
+);
+
+add_task(async function testNstmpCleanup() {
+  // Part 1
+  //
+  // Setup an IMAP server and some folders, including ones with
+  // potentially troublesome names (nstmp*).
+
+  setupIMAPPump();
+
+  const inbox = IMAPPump.inbox;
+  Assert.ok(inbox.msgStore.supportsCompaction, "test only applies to mbox");
+
+  // Create some mailboxes and populate with messages.
+  const mailboxes = [
+    "INBOX/foo",
+    "INBOX/foo/bar",
+    "INBOX/nstmp-42",
+    "INBOX/nstmp",
+    "INBOX/nstmp/wibble",
+    "INBOX/nstmp/nstmp-123",
+  ];
+  const generator = new MessageGenerator();
+  for (const name of mailboxes) {
+    IMAPPump.daemon.createMailbox(name, { subscribed: true });
+    const mailbox = IMAPPump.daemon.getMailbox(name);
+    generator.makeMessages({ count: 10 }).forEach(m => {
+      mailbox.addMessage(
+        new ImapMessage(
+          "data:text/plain;base64," + btoa(m.toMessageString()),
+          mailbox.uidnext++,
+          []
+        )
+      );
+    });
+  }
+
+  // These hacks are required because we've created the inbox before
+  // running initial folder discovery, and adding the folder bails
+  // out before we set it as verified online, so we bail out, and
+  // then remove the INBOX folder since it's not verified.
+  inbox.hierarchyDelimiter = "/";
+  inbox.verifiedAsOnlineFolder = true;
+
+  // Select the inbox to force folder discovery.
+  const listener = new PromiseTestUtils.PromiseUrlListener();
+  inbox.updateFolderWithListener(null, listener);
+  await listener.promise;
+
+  // These correspond to the mailboxes we set up on the IMAP server.
+  const folders = [
+    inbox,
+    inbox.getChildNamed("foo"),
+    inbox.getChildNamed("foo").getChildNamed("bar"),
+    inbox.getChildNamed("nstmp-42"),
+    inbox.getChildNamed("nstmp"),
+    inbox.getChildNamed("nstmp").getChildNamed("wibble"),
+    inbox.getChildNamed("nstmp").getChildNamed("nstmp-123"),
+  ];
+
+  // Attempt to mark all folders as offline and download them locally.
+  const expectToKeep = [];
+  for (const folder of folders) {
+    folder.setFlag(Ci.nsMsgFolderFlags.Offline);
+    const promiseUrlListener = new PromiseTestUtils.PromiseUrlListener();
+    folder.downloadAllForOffline(promiseUrlListener, null);
+    await promiseUrlListener.promise;
+    const file = folder.filePath.path;
+    // Note all the files we expect to remain untouched.
+    expectToKeep.push(file);
+  }
+
+  // HACKHACKHACK: DownloadAllForOffline() doesn't seem have created the mbox
+  // files by the time we get here, despite waiting for the listener.
+  // Investigation required.
+  // For now, if file doesn't exists, we'll fudge it by creating a stand-in
+  // one.
+  for (const file of expectToKeep) {
+    if (!(await IOUtils.exists(file))) {
+      await IOUtils.writeUTF8(file, "Pretending to be an mbox file.");
+    }
+  }
+
+  // Part 2
+  //
+  // Make sure nstmpCleanup() removes leftover nstmp files,
+  // and doesn't delete anything it shouldn't.
+
+  // Install extra nstmp files we expect nstmpCleanup() to delete.
+  const inboxDir = inbox.filePath.path + ".sbd";
+  const fooDir = inbox.getChildNamed("foo").filePath.path + ".sbd";
+  const expectToDelete = [
+    PathUtils.join(fooDir, "nstmp"),
+    PathUtils.join(fooDir, "nstmp-1"),
+    PathUtils.join(inboxDir, "nstmp-99"),
+    PathUtils.join(inboxDir, "nstmp-999"),
+  ];
+  for (const f of expectToDelete) {
+    await IOUtils.writeUTF8(f, "leftover cruft to delete");
+  }
+
+  // GO!
+  await MailMigrator._nstmpCleanup();
+
+  // Make sure we didn't delete anything we shouldn't have.
+  for (const f of expectToKeep) {
+    Assert.ok(await IOUtils.exists(f), `Kept ${f}`);
+  }
+
+  // Make sure we did delete rogue nstmp files.
+  for (const f of expectToDelete) {
+    Assert.ok(!(await IOUtils.exists(f)), `Deleted ${f}`);
+  }
+
+  // Part 3
+  //
+  // Lastly, make sure that nstmpCleanup() only runs once.
+
+  // Reinstall all the leftover nstmp files.
+  for (const f of expectToDelete) {
+    await IOUtils.writeUTF8(f, "more leftover cruft to delete");
+  }
+
+  // GO (again)!
+  await MailMigrator._nstmpCleanup();
+
+  // Make sure we didn't delete anything we shouldn't have.
+  for (const f of expectToKeep) {
+    Assert.ok(await IOUtils.exists(f), `Kept ${f}`);
+  }
+  // Make sure we didn't even delete the nstmp files that.
+  // were deleted on the first run!
+  for (const f of expectToDelete) {
+    Assert.ok(await IOUtils.exists(f), `Didn't delete ${f}`);
+  }
+
+  teardownIMAPPump();
+});

mailnews/imap/test/unit/xpcshell-cpp.ini

@@ -10,5 +10,6 @@ prefs =
 [test_customCommandReturnsFetchResponse.js]
 [test_imapChunks.js]
 [test_imapHdrChunking.js]
+[test_nstmpCleanup.js]
 
 [include:xpcshell-shared.ini]