Bug 1166038 De-duplicate security info stored in the Cache API. r=ehsan

2015-05-21 18:16:36 -07:00 · 2015-05-21 18:16:36 -07:00 · 4a673bf1ef
--- a/dom/cache/DBSchema.cpp
+++ b/dom/cache/DBSchema.cpp
@ -18,6 +18,7 @@
 #include "nsTArray.h"
 #include "nsCRT.h"
 #include "nsHttp.h"
+#include "nsICryptoHash.h"
 #include "mozilla/dom/HeadersBinding.h"
 #include "mozilla/dom/RequestBinding.h"
 #include "mozilla/dom/ResponseBinding.h"
@ -28,11 +29,11 @@ namespace dom {
 namespace cache {
 namespace db {

-const int32_t kMaxWipeSchemaVersion = 8;
+const int32_t kMaxWipeSchemaVersion = 9;

 namespace {

-const int32_t kLatestSchemaVersion = 8;
+const int32_t kLatestSchemaVersion = 9;
 const int32_t kMaxEntriesPerStatement = 255;

 } // anonymous namespace
@ -151,6 +152,14 @@ namespace {

 typedef int32_t EntryId;

+struct IdCount
+{
+  IdCount() : mId(-1), mCount(0) { }
+  explicit IdCount(int32_t aId) : mId(aId), mCount(1) { }
+  int32_t mId;
+  int32_t mCount;
+};
+
 static nsresult QueryAll(mozIStorageConnection* aConn, CacheId aCacheId,
                         nsTArray<EntryId>& aEntryIdListOut);
 static nsresult QueryCache(mozIStorageConnection* aConn, CacheId aCacheId,
@ -164,7 +173,14 @@ static nsresult MatchByVaryHeader(mozIStorageConnection* aConn,
 static nsresult DeleteEntries(mozIStorageConnection* aConn,
                              const nsTArray<EntryId>& aEntryIdList,
                              nsTArray<nsID>& aDeletedBodyIdListOut,
+                              nsTArray<IdCount>& aDeletedSecurityIdListOut,
                              uint32_t aPos=0, int32_t aLen=-1);
+static nsresult InsertSecurity(mozIStorageConnection* aConn,
+                               const nsACString& aData, int32_t *aIdOut);
+static nsresult DeleteSecurityInfo(mozIStorageConnection* aConn, int32_t aId,
+                                   int32_t aCount);
+static nsresult DeleteSecurityInfoList(mozIStorageConnection* aConn,
+                                       const nsTArray<IdCount>& aDeletedStorageIdList);
 static nsresult InsertEntry(mozIStorageConnection* aConn, CacheId aCacheId,
                            const CacheRequest& aRequest,
                            const nsID* aRequestBodyId,
@ -241,6 +257,24 @@ CreateSchema(mozIStorageConnection* aConn)
    ));
    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

+    // Security blobs are quite large and duplicated for every Response from
+    // the same https origin.  This table is used to de-duplicate this data.
+    rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+      "CREATE TABLE security_info ("
+        "id INTEGER NOT NULL PRIMARY KEY, "
+        "hash BLOB NOT NULL, "  // first 8-bytes of the sha1 hash of data column
+        "data BLOB NOT NULL, "  // full security info data, usually a few KB
+        "refcount INTEGER NOT NULL"
+      ");"
+    ));
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+    // Index the smaller hash value instead of the large security data blob.
+    rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+      "CREATE INDEX security_info_hash_index ON security_info (hash);"
+    ));
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
    rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
      "CREATE TABLE entries ("
        "id INTEGER NOT NULL PRIMARY KEY, "
@ -261,7 +295,7 @@ CreateSchema(mozIStorageConnection* aConn)
        "response_status_text TEXT NOT NULL, "
        "response_headers_guard INTEGER NOT NULL, "
        "response_body_id TEXT NULL, "
-        "response_security_info BLOB NULL, "
+        "response_security_info_id INTEGER NULL REFERENCES security_info(id), "
        "cache_id INTEGER NOT NULL REFERENCES caches(id) ON DELETE CASCADE"
      ");"
    ));
@ -406,7 +440,12 @@ DeleteCacheId(mozIStorageConnection* aConn, CacheId aCacheId,
  nsresult rv = QueryAll(aConn, aCacheId, matches);
  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

-  rv = DeleteEntries(aConn, matches, aDeletedBodyIdListOut);
+  nsAutoTArray<IdCount, 16> deletedSecurityIdList;
+  rv = DeleteEntries(aConn, matches, aDeletedBodyIdListOut,
+                     deletedSecurityIdList);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = DeleteSecurityInfoList(aConn, deletedSecurityIdList);
  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  // Delete the remainder of the cache using cascade semantics.
@ -538,13 +577,20 @@ CachePut(mozIStorageConnection* aConn, CacheId aCacheId,
  nsresult rv = QueryCache(aConn, aCacheId, aRequest, params, matches);
  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

-  rv = DeleteEntries(aConn, matches, aDeletedBodyIdListOut);
+  nsAutoTArray<IdCount, 16> deletedSecurityIdList;
+  rv = DeleteEntries(aConn, matches, aDeletedBodyIdListOut,
+                     deletedSecurityIdList);
  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  rv = InsertEntry(aConn, aCacheId, aRequest, aRequestBodyId, aResponse,
                   aResponseBodyId);
  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

+  // Delete the security values after doing the insert to avoid churning
+  // the security table when its not necessary.
+  rv = DeleteSecurityInfoList(aConn, deletedSecurityIdList);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
  return rv;
 }

@ -568,7 +614,12 @@ CacheDelete(mozIStorageConnection* aConn, CacheId aCacheId,
    return rv;
  }

-  rv = DeleteEntries(aConn, matches, aDeletedBodyIdListOut);
+  nsAutoTArray<IdCount, 16> deletedSecurityIdList;
+  rv = DeleteEntries(aConn, matches, aDeletedBodyIdListOut,
+                     deletedSecurityIdList);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = DeleteSecurityInfoList(aConn, deletedSecurityIdList);
  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  *aSuccessOut = true;
@ -1024,6 +1075,7 @@ nsresult
 DeleteEntries(mozIStorageConnection* aConn,
              const nsTArray<EntryId>& aEntryIdList,
              nsTArray<nsID>& aDeletedBodyIdListOut,
+              nsTArray<IdCount>& aDeletedSecurityIdListOut,
              uint32_t aPos, int32_t aLen)
 {
  MOZ_ASSERT(!NS_IsMainThread());
@ -1048,7 +1100,7 @@ DeleteEntries(mozIStorageConnection* aConn,
      int32_t max = kMaxEntriesPerStatement;
      int32_t curLen = std::min(max, remaining);
      nsresult rv = DeleteEntries(aConn, aEntryIdList, aDeletedBodyIdListOut,
-                                  curPos, curLen);
+                                  aDeletedSecurityIdListOut, curPos, curLen);
      if (NS_FAILED(rv)) { return rv; }

      curPos += curLen;
@ -1059,7 +1111,8 @@ DeleteEntries(mozIStorageConnection* aConn,

  nsCOMPtr<mozIStorageStatement> state;
  nsAutoCString query(
-    "SELECT request_body_id, response_body_id FROM entries WHERE id IN ("
+    "SELECT request_body_id, response_body_id, response_security_info_id "
+    "FROM entries WHERE id IN ("
  );
  AppendListParamsToQuery(query, aEntryIdList, aPos, aLen);
  query.AppendLiteral(")");
@ -1086,6 +1139,33 @@ DeleteEntries(mozIStorageConnection* aConn,
        aDeletedBodyIdListOut.AppendElement(id);
      }
    }
+
+    // and then a possible third entry for the security id
+    bool isNull = false;
+    rv = state->GetIsNull(2, &isNull);
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+    if (!isNull) {
+      int32_t securityId = -1;
+      rv = state->GetInt32(2, &securityId);
+      if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+      // First try to increment the count for this ID if we're already
+      // seen it
+      bool found = false;
+      for (uint32_t i = 0; i < aDeletedSecurityIdListOut.Length(); ++i) {
+        if (aDeletedSecurityIdListOut[i].mId == securityId) {
+          found = true;
+          aDeletedSecurityIdListOut[i].mCount += 1;
+          break;
+        }
+      }
+
+      // Otherwise add a new entry for this ID with a count of 1
+      if (!found) {
+        aDeletedSecurityIdListOut.AppendElement(IdCount(securityId));
+      }
+    }
  }

  // Dependent records removed via ON DELETE CASCADE
@ -1108,6 +1188,194 @@ DeleteEntries(mozIStorageConnection* aConn,
  return rv;
 }

+nsresult
+InsertSecurity(mozIStorageConnection* aConn, const nsACString& aData,
+               int32_t *aIdOut)
+{
+  MOZ_ASSERT(aConn);
+  MOZ_ASSERT(aIdOut);
+  MOZ_ASSERT(!aData.IsEmpty());
+
+  // We want to use an index to find existing security blobs, but indexing
+  // the full blob would be quite expensive.  Instead, we index a small
+  // hash value.  Calculate this hash as the first 8 bytes of the SHA1 of
+  // the full data.
+  nsresult rv;
+  nsCOMPtr<nsICryptoHash> crypto =
+    do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = crypto->Init(nsICryptoHash::SHA1);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = crypto->Update(reinterpret_cast<const uint8_t*>(aData.BeginReading()),
+                      aData.Length());
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  nsAutoCString fullHash;
+  rv = crypto->Finish(false /* based64 result */, fullHash);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  nsDependentCSubstring hash(fullHash, 0, 8);
+
+  // Next, search for an existing entry for this blob by comparing the hash
+  // value first and then the full data.  SQLite is smart enough to use
+  // the index on the hash to search the table before doing the expensive
+  // comparison of the large data column.  (This was verified with EXPLAIN.)
+  nsCOMPtr<mozIStorageStatement> state;
+  rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
+    // Note that hash and data are blobs, but we can use = here since the
+    // columns are NOT NULL.
+    "SELECT id, refcount FROM security_info WHERE hash=:hash AND data=:data;"
+  ), getter_AddRefs(state));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = state->BindUTF8StringAsBlobByName(NS_LITERAL_CSTRING("hash"), hash);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = state->BindUTF8StringAsBlobByName(NS_LITERAL_CSTRING("data"), aData);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  bool hasMoreData = false;
+  rv = state->ExecuteStep(&hasMoreData);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  // This security info blob is already in the database
+  if (hasMoreData) {
+    // get the existing security blob id to return
+    rv = state->GetInt32(0, aIdOut);
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+    int32_t refcount = -1;
+    rv = state->GetInt32(1, &refcount);
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+    // But first, update the refcount in the database.
+    refcount += 1;
+
+    rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
+      "UPDATE security_info SET refcount=:refcount WHERE id=:id;"
+    ), getter_AddRefs(state));
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+    rv = state->BindInt32ByName(NS_LITERAL_CSTRING("refcount"), refcount);
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+    rv = state->BindInt32ByName(NS_LITERAL_CSTRING("id"), *aIdOut);
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+    rv = state->Execute();
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+    return NS_OK;
+  }
+
+  // This is a new security info blob.  Create a new row in the security table
+  // with an initial refcount of 1.
+  rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
+    "INSERT INTO security_info (hash, data, refcount) VALUES (:hash, :data, 1);"
+  ), getter_AddRefs(state));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = state->BindUTF8StringAsBlobByName(NS_LITERAL_CSTRING("hash"), hash);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = state->BindUTF8StringAsBlobByName(NS_LITERAL_CSTRING("data"), aData);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = state->Execute();
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
+    "SELECT last_insert_rowid()"
+  ), getter_AddRefs(state));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  hasMoreData = false;
+  rv = state->ExecuteStep(&hasMoreData);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = state->GetInt32(0, aIdOut);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  return NS_OK;
+}
+
+nsresult
+DeleteSecurityInfo(mozIStorageConnection* aConn, int32_t aId, int32_t aCount)
+{
+  // First, we need to determine the current refcount for this security blob.
+  nsCOMPtr<mozIStorageStatement> state;
+  nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
+    "SELECT refcount FROM security_info WHERE id=:id;"
+  ), getter_AddRefs(state));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = state->BindInt32ByName(NS_LITERAL_CSTRING("id"), aId);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  bool hasMoreData = false;
+  rv = state->ExecuteStep(&hasMoreData);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  int32_t refcount = -1;
+  rv = state->GetInt32(0, &refcount);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  MOZ_ASSERT(refcount >= aCount);
+
+  // Next, calculate the new refcount
+  int32_t newCount = refcount - aCount;
+
+  // If the last reference to this security blob was removed we can
+  // just remove the entire row.
+  if (newCount == 0) {
+    rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
+      "DELETE FROM security_info WHERE id=:id;"
+    ), getter_AddRefs(state));
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+    rv = state->BindInt32ByName(NS_LITERAL_CSTRING("id"), aId);
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+    rv = state->Execute();
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+    return NS_OK;
+  }
+
+  // Otherwise update the refcount in the table to reflect the reduced
+  // number of references to the security blob.
+  rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
+    "UPDATE security_info SET refcount=:refcount WHERE id=:id;"
+  ), getter_AddRefs(state));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = state->BindInt32ByName(NS_LITERAL_CSTRING("refcount"), newCount);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = state->BindInt32ByName(NS_LITERAL_CSTRING("id"), aId);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = state->Execute();
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  return NS_OK;
+}
+
+nsresult
+DeleteSecurityInfoList(mozIStorageConnection* aConn,
+                        const nsTArray<IdCount>& aDeletedStorageIdList)
+{
+  for (uint32_t i = 0; i < aDeletedStorageIdList.Length(); ++i) {
+    nsresult rv = DeleteSecurityInfo(aConn, aDeletedStorageIdList[i].mId,
+                                     aDeletedStorageIdList[i].mCount);
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+  }
+
+  return NS_OK;
+}
+
 nsresult
 InsertEntry(mozIStorageConnection* aConn, CacheId aCacheId,
            const CacheRequest& aRequest,
@ -1118,8 +1386,16 @@ InsertEntry(mozIStorageConnection* aConn, CacheId aCacheId,
  MOZ_ASSERT(!NS_IsMainThread());
  MOZ_ASSERT(aConn);

+  nsresult rv = NS_OK;
+  int32_t securityId = -1;
+
+  if (!aResponse.securityInfo().IsEmpty()) {
+    rv = InsertSecurity(aConn, aResponse.securityInfo(), &securityId);
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+  }
+
  nsCOMPtr<mozIStorageStatement> state;
-  nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
+  rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
    "INSERT INTO entries ("
      "request_method, "
      "request_url, "
@ -1138,7 +1414,7 @@ InsertEntry(mozIStorageConnection* aConn, CacheId aCacheId,
      "response_status_text, "
      "response_headers_guard, "
      "response_body_id, "
-      "response_security_info, "
+      "response_security_info_id, "
      "cache_id "
    ") VALUES ("
      ":request_method, "
@ -1158,7 +1434,7 @@ InsertEntry(mozIStorageConnection* aConn, CacheId aCacheId,
      ":response_status_text, "
      ":response_headers_guard, "
      ":response_body_id, "
-      ":response_security_info, "
+      ":response_security_info_id, "
      ":cache_id "
    ");"
  ), getter_AddRefs(state));
@ -1230,8 +1506,12 @@ InsertEntry(mozIStorageConnection* aConn, CacheId aCacheId,
  rv = BindId(state, NS_LITERAL_CSTRING("response_body_id"), aResponseBodyId);
  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

-  rv = state->BindUTF8StringAsBlobByName(NS_LITERAL_CSTRING("response_security_info"),
-                                         aResponse.securityInfo());
+  if (aResponse.securityInfo().IsEmpty()) {
+    rv = state->BindNullByName(NS_LITERAL_CSTRING("response_security_info_id"));
+  } else {
+    rv = state->BindInt32ByName(NS_LITERAL_CSTRING("response_security_info_id"),
+                                securityId);
+  }
  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

  rv = state->BindInt64ByName(NS_LITERAL_CSTRING("cache_id"), aCacheId);
@ -1319,15 +1599,17 @@ ReadResponse(mozIStorageConnection* aConn, EntryId aEntryId,
  nsCOMPtr<mozIStorageStatement> state;
  nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
    "SELECT "
-      "response_type, "
-      "response_url, "
-      "response_status, "
-      "response_status_text, "
-      "response_headers_guard, "
-      "response_body_id, "
-      "response_security_info "
+      "entries.response_type, "
+      "entries.response_url, "
+      "entries.response_status, "
+      "entries.response_status_text, "
+      "entries.response_headers_guard, "
+      "entries.response_body_id, "
+      "security_info.data "
    "FROM entries "
-    "WHERE id=:id;"
+    "LEFT OUTER JOIN security_info "
+    "ON entries.response_security_info_id=security_info.id "
+    "WHERE entries.id=:id;"
  ), getter_AddRefs(state));
  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }

--- a/dom/cache/test/mochitest/mochitest.ini
+++ b/dom/cache/test/mochitest/mochitest.ini
@ -20,6 +20,7 @@ support-files =
  test_cache_requestCache.js
  test_cache_delete.js
  test_cache_put_reorder.js
+  test_cache_https.js

 [test_cache.html]
 [test_cache_add.html]
@ -33,4 +34,6 @@ support-files =
 [test_cache_requestCache.html]
 [test_cache_delete.html]
 [test_cache_put_reorder.html]
+[test_cache_https.html]
+  skip-if = buildapp == 'b2g' # bug 1162353
 [test_cache_restart.html]
--- a/dom/cache/test/mochitest/test_cache_https.html
+++ b/dom/cache/test/mochitest/test_cache_https.html
@ -0,0 +1,20 @@
+<!-- Any copyright is dedicated to the Public Domain.
+   - http://creativecommons.org/publicdomain/zero/1.0/ -->
+<!DOCTYPE HTML>
+<html>
+<head>
+  <title>Validate Interfaces Exposed to Workers</title>
+  <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" />
+  <script type="text/javascript" src="driver.js"></script>
+</head>
+<body>
+<iframe id="frame"></iframe>
+<script class="testbody" type="text/javascript">
+  runTests("test_cache_https.js")
+    .then(function() {
+      SimpleTest.finish();
+    });
+</script>
+</body>
+</html>
--- a/dom/cache/test/mochitest/test_cache_https.js
+++ b/dom/cache/test/mochitest/test_cache_https.js
@ -0,0 +1,23 @@
+var cache = null;
+var name = 'https_' + context;
+var urlBase = 'https://example.com/tests/dom/cache/test/mochitest';
+var url1 = urlBase + '/test_cache.js';
+var url2 = urlBase + '/test_cache_add.js';
+
+caches.open(name).then(function(c) {
+  cache = c;
+  return cache.addAll([new Request(url1, { mode: 'no-cors' }),
+                       new Request(url2, { mode: 'no-cors' })]);
+}).then(function() {
+  return cache.delete(url1);
+}).then(function(result) {
+  ok(result, 'Cache entry should be deleted');
+  return cache.delete(url2);
+}).then(function(result) {
+  ok(result, 'Cache entry should be deleted');
+  cache = null;
+  return caches.delete(name);
+}).then(function(result) {
+  ok(result, 'Cache should be deleted');
+  testDone();
+});