Bug 1801495 - Add a shim no-lock VFS that implements immutable behavior. r=asuth

When ignoreLockingMode option is used, we default to the unix-none or win32-none
VFS, and enforce the connection to be read-only, because it's trivial to
corrupt data using a no-lock VFS.
Unfortunately read-only mode doesn't allow to open a WAL database if the journal
files are missing, unless the device is marked as "immutable".
There is no SQLITE_OPEN_* flag for immutable, and adding one could easily be
misinterpreted since it only makes sense for read-only.

This implement a readonly-immutable-nolock VFS shim on top of unix-none or
win32-none that overrides device characteristics with immutable. When
ignoreLockingMode option is specified, we use this VFS. It can only be used on
read-only connections, as enforced both by Storage service and a MOZ_ASSERT.

Differential Revision: https://phabricator.services.mozilla.com/D162492

storage/ReadOnlyNoLockVFS.cpp

@@ -0,0 +1,130 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ :
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * This VFS is built on top of the (unix|win32)-none, but it additionally
+ * sets any opened file as immutable, that allows to also open in read-only
+ * mode databases using WAL, or other journals that need auxiliary files, when
+ * such files cannot be created.
+ * This is useful when trying to read from third-party databases, avoiding any
+ * risk of creating auxiliary files (e.g. journals).
+ * It can only be used on read-only connections, because being a no-lock VFS
+ * it would be trivial to corrupt the data.
+ */
+
+#include "nsDebug.h"
+#include "sqlite3.h"
+
+#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData))
+
+#if defined(XP_WIN)
+#  define BASE_VFS "win32-none"
+#else
+#  define BASE_VFS "unix-none"
+#endif
+
+#define VFS_NAME "readonly-immutable-nolock"
+
+namespace {
+
+static int vfsOpen(sqlite3_vfs* vfs, const char* zName, sqlite3_file* pFile,
+                   int flags, int* pOutFlags) {
+  if ((flags & SQLITE_OPEN_READONLY) == 0) {
+    // This is not done to be used in readwrite connections.
+    return SQLITE_CANTOPEN;
+  }
+
+  sqlite3_vfs* pOrigVfs = ORIGVFS(vfs);
+  int rc = pOrigVfs->xOpen(pOrigVfs, zName, pFile, flags, pOutFlags);
+  if (rc != SQLITE_OK) {
+    return rc;
+  }
+
+  const sqlite3_io_methods* pOrigMethods = pFile->pMethods;
+
+  // If the IO version is higher than the last known one, you should update
+  // this IO adding appropriate methods for any methods added in the version
+  // change.
+  MOZ_ASSERT(pOrigMethods->iVersion <= 3);
+
+  static const sqlite3_io_methods vfs_io_methods = {
+      pOrigMethods->iVersion,           /* iVersion */
+      pOrigMethods->xClose,             /* xClose */
+      pOrigMethods->xRead,              /* xRead */
+      pOrigMethods->xWrite,             /* xWrite */
+      pOrigMethods->xTruncate,          /* xTruncate */
+      pOrigMethods->xSync,              /* xSync */
+      pOrigMethods->xFileSize,          /* xFileSize */
+      pOrigMethods->xLock,              /* xLock */
+      pOrigMethods->xUnlock,            /* xUnlock */
+      pOrigMethods->xCheckReservedLock, /* xCheckReservedLock */
+      pOrigMethods->xFileControl,       /* xFileControl */
+      pOrigMethods->xSectorSize,        /* xSectorSize */
+      [](sqlite3_file*) {
+        return SQLITE_IOCAP_IMMUTABLE;
+      },                         /* xDeviceCharacteristics */
+      pOrigMethods->xShmMap,     /* xShmMap */
+      pOrigMethods->xShmLock,    /* xShmLock */
+      pOrigMethods->xShmBarrier, /* xShmBarrier */
+      pOrigMethods->xShmUnmap,   /* xShmUnmap */
+      pOrigMethods->xFetch,      /* xFetch */
+      pOrigMethods->xUnfetch     /* xUnfetch */
+  };
+  pFile->pMethods = &vfs_io_methods;
+  if (pOutFlags) {
+    *pOutFlags = flags;
+  }
+
+  return SQLITE_OK;
+}
+
+}  // namespace
+
+namespace mozilla::storage {
+
+UniquePtr<sqlite3_vfs> ConstructReadOnlyNoLockVFS() {
+  if (sqlite3_vfs_find(VFS_NAME) != nullptr) {
+    return nullptr;
+  }
+  sqlite3_vfs* pOrigVfs = sqlite3_vfs_find(BASE_VFS);
+  if (!pOrigVfs) {
+    return nullptr;
+  }
+
+  // If the VFS version is higher than the last known one, you should update
+  // this VFS adding appropriate methods for any methods added in the version
+  // change.
+  MOZ_ASSERT(pOrigVfs->iVersion <= 3);
+
+  static const sqlite3_vfs vfs = {
+      pOrigVfs->iVersion,          /* iVersion  */
+      pOrigVfs->szOsFile,          /* szOsFile */
+      pOrigVfs->mxPathname,        /* mxPathname */
+      nullptr,                     /* pNext */
+      VFS_NAME,                    /* zName */
+      pOrigVfs,                    /* pAppData */
+      vfsOpen,                     /* xOpen */
+      pOrigVfs->xDelete,           /* xDelete */
+      pOrigVfs->xAccess,           /* xAccess */
+      pOrigVfs->xFullPathname,     /* xFullPathname */
+      pOrigVfs->xDlOpen,           /* xDlOpen */
+      pOrigVfs->xDlError,          /* xDlError */
+      pOrigVfs->xDlSym,            /* xDlSym */
+      pOrigVfs->xDlClose,          /* xDlClose */
+      pOrigVfs->xRandomness,       /* xRandomness */
+      pOrigVfs->xSleep,            /* xSleep */
+      pOrigVfs->xCurrentTime,      /* xCurrentTime */
+      pOrigVfs->xGetLastError,     /* xGetLastError */
+      pOrigVfs->xCurrentTimeInt64, /* xCurrentTimeInt64 */
+      pOrigVfs->xSetSystemCall,    /* xSetSystemCall */
+      pOrigVfs->xGetSystemCall,    /* xGetSystemCall */
+      pOrigVfs->xNextSystemCall    /* xNextSystemCall */
+  };
+
+  return MakeUnique<sqlite3_vfs>(vfs);
+}
+
+}  // namespace mozilla::storage

storage/moz.build

@@ -74,6 +74,7 @@ UNIFIED_SOURCES += [
     "mozStorageStatementParams.cpp",
     "mozStorageStatementRow.cpp",
     "ObfuscatingVFS.cpp",
+    "ReadOnlyNoLockVFS.cpp",
     "SQLCollations.cpp",
     "StorageBaseStatementInternal.cpp",
     "TelemetryVFS.cpp",

storage/mozStorageConnection.cpp

@@ -717,18 +717,12 @@ nsresult Connection::initialize(nsIFile* aDatabaseFile) {
   nsresult rv = aDatabaseFile->GetPath(path);
   NS_ENSURE_SUCCESS(rv, rv);
 
-#ifdef XP_WIN
-  static const char* sIgnoreLockingVFS = "win32-none";
-#else
-  static const char* sIgnoreLockingVFS = "unix-none";
-#endif
-
   bool exclusive = StaticPrefs::storage_sqlite_exclusiveLock_enabled();
   int srv;
   if (mIgnoreLockingMode) {
     exclusive = false;
     srv = ::sqlite3_open_v2(NS_ConvertUTF16toUTF8(path).get(), &mDBConn, mFlags,
-                            sIgnoreLockingVFS);
+                            "readonly-immutable-nolock");
   } else {
     srv = ::sqlite3_open_v2(NS_ConvertUTF16toUTF8(path).get(), &mDBConn, mFlags,
                             GetTelemetryVFSName(exclusive));

storage/mozStorageService.cpp

@@ -310,6 +310,8 @@ const char* GetTelemetryVFSName(bool);
 
 UniquePtr<sqlite3_vfs> ConstructObfuscatingVFS(const char* aBaseVFSName);
 
+UniquePtr<sqlite3_vfs> ConstructReadOnlyNoLockVFS();
+
 static const char* sObserverTopics[] = {"memory-pressure",
                                         "xpcom-shutdown-threads"};
 
@@ -337,6 +339,11 @@ nsresult Service::initialize() {
     return convertResultCode(rc);
   }
 
+  rc = mReadOnlyNoLockSqliteVFS.Init(ConstructReadOnlyNoLockVFS());
+  if (rc != SQLITE_OK) {
+    return convertResultCode(rc);
+  }
+
   nsCOMPtr<nsIObserverService> os = mozilla::services::GetObserverService();
   NS_ENSURE_TRUE(os, NS_ERROR_FAILURE);
 

storage/mozStorageService.h

@@ -132,6 +132,7 @@ class Service : public mozIStorageService,
   AutoVFSRegistration mTelemetrySqliteVFS;
   AutoVFSRegistration mTelemetryExclSqliteVFS;
   AutoVFSRegistration mObfuscatingSqliteVFS;
+  AutoVFSRegistration mReadOnlyNoLockSqliteVFS;
 
   /**
    * Protects mConnections.

storage/test/unit/head_storage.js

@@ -9,6 +9,10 @@ var { AppConstants } = ChromeUtils.importESModule(
   "resource://gre/modules/AppConstants.sys.mjs"
 );
 
+ChromeUtils.defineESModuleGetters(this, {
+  Sqlite: "resource://gre/modules/Sqlite.sys.mjs",
+});
+
 const { TelemetryTestUtils } = ChromeUtils.import(
   "resource://testing-common/TelemetryTestUtils.jsm"
 );

storage/test/unit/test_readonly-immutable-nolock_vfs.js

@@ -0,0 +1,47 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// This file tests the readonly-immutable-nolock VFS.
+
+add_task(async function test() {
+  const path = PathUtils.join(PathUtils.profileDir, "ro");
+  await IOUtils.makeDirectory(path);
+  const dbpath = PathUtils.join(path, "test-immutable.sqlite");
+
+  let conn = await Sqlite.openConnection({ path: dbpath });
+  await conn.execute("PRAGMA journal_mode = WAL");
+  await conn.execute("CREATE TABLE test (id INTEGER PRIMARY KEY)");
+  Assert.ok(await IOUtils.exists(dbpath + "-wal"), "wal journal exists");
+  await conn.close();
+
+  // The wal should have been merged at this point, but just in case...
+  info("Remove auxiliary files and set the folder as readonly");
+  await IOUtils.remove(dbpath + "-wal", { ignoreAbsent: true });
+  await IOUtils.setPermissions(path, 0o555);
+  registerCleanupFunction(async () => {
+    await IOUtils.setPermissions(path, 0o777);
+    await IOUtils.remove(path, { recursive: true });
+  });
+
+  // Windows doesn't disallow creating files in read only folders.
+  if (AppConstants.platform == "macosx" || AppConstants.platform == "linux") {
+    await Assert.rejects(
+      Sqlite.openConnection({ path: dbpath, readOnly: true }),
+      /NS_ERROR_FILE/,
+      "Should not be able to open the db because it can't create a wal journal"
+    );
+  }
+
+  // Open the database with ignoreLockingMode.
+  let conn2 = await Sqlite.openConnection({
+    path: dbpath,
+    ignoreLockingMode: true,
+  });
+  await conn2.execute("SELECT * FROM sqlite_master");
+  Assert.ok(
+    !(await IOUtils.exists(dbpath + "-wal")),
+    "wal journal was not created"
+  );
+  await conn2.close();
+});

storage/test/unit/xpcshell.ini

@@ -32,6 +32,7 @@ skip-if = debug
 [test_locale_collation.js]
 [test_minimizeMemory.js]
 [test_page_size_is_32k.js]
+[test_readonly-immutable-nolock_vfs.js]
 [test_sqlite_secure_delete.js]
 [test_statement_executeAsync.js]
 [test_statement_wrapper_automatically.js]