Bug 1166038 De-duplicate security info stored in the Cache API. r=ehsan

dom/cache/DBSchema.cpp

@@ -18,6 +18,7 @@
 #include "nsTArray.h"
 #include "nsCRT.h"
 #include "nsHttp.h"
+#include "nsICryptoHash.h"
 #include "mozilla/dom/HeadersBinding.h"
 #include "mozilla/dom/RequestBinding.h"
 #include "mozilla/dom/ResponseBinding.h"
@@ -28,11 +29,11 @@ namespace dom {
 namespace cache {
 namespace db {
 
-const int32_t kMaxWipeSchemaVersion = 8;
+const int32_t kMaxWipeSchemaVersion = 9;
 
 namespace {
 
-const int32_t kLatestSchemaVersion = 8;
+const int32_t kLatestSchemaVersion = 9;
 const int32_t kMaxEntriesPerStatement = 255;
 
 } // anonymous namespace
@@ -151,6 +152,14 @@ namespace {
 
 typedef int32_t EntryId;
 
+struct IdCount
+{
+  IdCount() : mId(-1), mCount(0) { }
+  explicit IdCount(int32_t aId) : mId(aId), mCount(1) { }
+  int32_t mId;
+  int32_t mCount;
+};
+
 static nsresult QueryAll(mozIStorageConnection* aConn, CacheId aCacheId,
                          nsTArray<EntryId>& aEntryIdListOut);
 static nsresult QueryCache(mozIStorageConnection* aConn, CacheId aCacheId,
@@ -164,7 +173,14 @@ static nsresult MatchByVaryHeader(mozIStorageConnection* aConn,
 static nsresult DeleteEntries(mozIStorageConnection* aConn,
                               const nsTArray<EntryId>& aEntryIdList,
                               nsTArray<nsID>& aDeletedBodyIdListOut,
+                              nsTArray<IdCount>& aDeletedSecurityIdListOut,
                               uint32_t aPos=0, int32_t aLen=-1);
+static nsresult InsertSecurity(mozIStorageConnection* aConn,
+                               const nsACString& aData, int32_t *aIdOut);
+static nsresult DeleteSecurityInfo(mozIStorageConnection* aConn, int32_t aId,
+                                   int32_t aCount);
+static nsresult DeleteSecurityInfoList(mozIStorageConnection* aConn,
+                                       const nsTArray<IdCount>& aDeletedStorageIdList);
 static nsresult InsertEntry(mozIStorageConnection* aConn, CacheId aCacheId,
                             const CacheRequest& aRequest,
                             const nsID* aRequestBodyId,
@@ -241,6 +257,24 @@ CreateSchema(mozIStorageConnection* aConn)
     ));
     if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
+    // Security blobs are quite large and duplicated for every Response from
+    // the same https origin.  This table is used to de-duplicate this data.
+    rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+      "CREATE TABLE security_info ("
+        "id INTEGER NOT NULL PRIMARY KEY, "
+        "hash BLOB NOT NULL, "  // first 8-bytes of the sha1 hash of data column
+        "data BLOB NOT NULL, "  // full security info data, usually a few KB
+        "refcount INTEGER NOT NULL"
+      ");"
+    ));
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+    // Index the smaller hash value instead of the large security data blob.
+    rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+      "CREATE INDEX security_info_hash_index ON security_info (hash);"
+    ));
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
     rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
       "CREATE TABLE entries ("
         "id INTEGER NOT NULL PRIMARY KEY, "
@@ -261,7 +295,7 @@ CreateSchema(mozIStorageConnection* aConn)
         "response_status_text TEXT NOT NULL, "
         "response_headers_guard INTEGER NOT NULL, "
         "response_body_id TEXT NULL, "
-        "response_security_info BLOB NULL, "
+        "response_security_info_id INTEGER NULL REFERENCES security_info(id), "
         "cache_id INTEGER NOT NULL REFERENCES caches(id) ON DELETE CASCADE"
       ");"
     ));
@@ -406,7 +440,12 @@ DeleteCacheId(mozIStorageConnection* aConn, CacheId aCacheId,
   nsresult rv = QueryAll(aConn, aCacheId, matches);
   if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
-  rv = DeleteEntries(aConn, matches, aDeletedBodyIdListOut);
+  nsAutoTArray<IdCount, 16> deletedSecurityIdList;
+  rv = DeleteEntries(aConn, matches, aDeletedBodyIdListOut,
+                     deletedSecurityIdList);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = DeleteSecurityInfoList(aConn, deletedSecurityIdList);
   if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
   // Delete the remainder of the cache using cascade semantics.
@@ -538,13 +577,20 @@ CachePut(mozIStorageConnection* aConn, CacheId aCacheId,
   nsresult rv = QueryCache(aConn, aCacheId, aRequest, params, matches);
   if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
-  rv = DeleteEntries(aConn, matches, aDeletedBodyIdListOut);
+  nsAutoTArray<IdCount, 16> deletedSecurityIdList;
+  rv = DeleteEntries(aConn, matches, aDeletedBodyIdListOut,
+                     deletedSecurityIdList);
   if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
   rv = InsertEntry(aConn, aCacheId, aRequest, aRequestBodyId, aResponse,
                    aResponseBodyId);
   if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
+  // Delete the security values after doing the insert to avoid churning
+  // the security table when its not necessary.
+  rv = DeleteSecurityInfoList(aConn, deletedSecurityIdList);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
   return rv;
 }
 
@@ -568,7 +614,12 @@ CacheDelete(mozIStorageConnection* aConn, CacheId aCacheId,
     return rv;
   }
 
-  rv = DeleteEntries(aConn, matches, aDeletedBodyIdListOut);
+  nsAutoTArray<IdCount, 16> deletedSecurityIdList;
+  rv = DeleteEntries(aConn, matches, aDeletedBodyIdListOut,
+                     deletedSecurityIdList);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = DeleteSecurityInfoList(aConn, deletedSecurityIdList);
   if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
   *aSuccessOut = true;
@@ -1024,6 +1075,7 @@ nsresult
 DeleteEntries(mozIStorageConnection* aConn,
               const nsTArray<EntryId>& aEntryIdList,
               nsTArray<nsID>& aDeletedBodyIdListOut,
+              nsTArray<IdCount>& aDeletedSecurityIdListOut,
               uint32_t aPos, int32_t aLen)
 {
   MOZ_ASSERT(!NS_IsMainThread());
@@ -1048,7 +1100,7 @@ DeleteEntries(mozIStorageConnection* aConn,
       int32_t max = kMaxEntriesPerStatement;
       int32_t curLen = std::min(max, remaining);
       nsresult rv = DeleteEntries(aConn, aEntryIdList, aDeletedBodyIdListOut,
-                                  curPos, curLen);
+                                  aDeletedSecurityIdListOut, curPos, curLen);
       if (NS_FAILED(rv)) { return rv; }
 
       curPos += curLen;
@@ -1059,7 +1111,8 @@ DeleteEntries(mozIStorageConnection* aConn,
 
   nsCOMPtr<mozIStorageStatement> state;
   nsAutoCString query(
-    "SELECT request_body_id, response_body_id FROM entries WHERE id IN ("
+    "SELECT request_body_id, response_body_id, response_security_info_id "
+    "FROM entries WHERE id IN ("
   );
   AppendListParamsToQuery(query, aEntryIdList, aPos, aLen);
   query.AppendLiteral(")");
@@ -1086,6 +1139,33 @@ DeleteEntries(mozIStorageConnection* aConn,
         aDeletedBodyIdListOut.AppendElement(id);
       }
     }
+
+    // and then a possible third entry for the security id
+    bool isNull = false;
+    rv = state->GetIsNull(2, &isNull);
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+    if (!isNull) {
+      int32_t securityId = -1;
+      rv = state->GetInt32(2, &securityId);
+      if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+      // First try to increment the count for this ID if we're already
+      // seen it
+      bool found = false;
+      for (uint32_t i = 0; i < aDeletedSecurityIdListOut.Length(); ++i) {
+        if (aDeletedSecurityIdListOut[i].mId == securityId) {
+          found = true;
+          aDeletedSecurityIdListOut[i].mCount += 1;
+          break;
+        }
+      }
+
+      // Otherwise add a new entry for this ID with a count of 1
+      if (!found) {
+        aDeletedSecurityIdListOut.AppendElement(IdCount(securityId));
+      }
+    }
   }
 
   // Dependent records removed via ON DELETE CASCADE
@@ -1108,6 +1188,194 @@ DeleteEntries(mozIStorageConnection* aConn,
   return rv;
 }
 
+nsresult
+InsertSecurity(mozIStorageConnection* aConn, const nsACString& aData,
+               int32_t *aIdOut)
+{
+  MOZ_ASSERT(aConn);
+  MOZ_ASSERT(aIdOut);
+  MOZ_ASSERT(!aData.IsEmpty());
+
+  // We want to use an index to find existing security blobs, but indexing
+  // the full blob would be quite expensive.  Instead, we index a small
+  // hash value.  Calculate this hash as the first 8 bytes of the SHA1 of
+  // the full data.
+  nsresult rv;
+  nsCOMPtr<nsICryptoHash> crypto =
+    do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = crypto->Init(nsICryptoHash::SHA1);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = crypto->Update(reinterpret_cast<const uint8_t*>(aData.BeginReading()),
+                      aData.Length());
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  nsAutoCString fullHash;
+  rv = crypto->Finish(false /* based64 result */, fullHash);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  nsDependentCSubstring hash(fullHash, 0, 8);
+
+  // Next, search for an existing entry for this blob by comparing the hash
+  // value first and then the full data.  SQLite is smart enough to use
+  // the index on the hash to search the table before doing the expensive
+  // comparison of the large data column.  (This was verified with EXPLAIN.)
+  nsCOMPtr<mozIStorageStatement> state;
+  rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
+    // Note that hash and data are blobs, but we can use = here since the
+    // columns are NOT NULL.
+    "SELECT id, refcount FROM security_info WHERE hash=:hash AND data=:data;"
+  ), getter_AddRefs(state));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = state->BindUTF8StringAsBlobByName(NS_LITERAL_CSTRING("hash"), hash);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = state->BindUTF8StringAsBlobByName(NS_LITERAL_CSTRING("data"), aData);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  bool hasMoreData = false;
+  rv = state->ExecuteStep(&hasMoreData);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  // This security info blob is already in the database
+  if (hasMoreData) {
+    // get the existing security blob id to return
+    rv = state->GetInt32(0, aIdOut);
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+    int32_t refcount = -1;
+    rv = state->GetInt32(1, &refcount);
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+    // But first, update the refcount in the database.
+    refcount += 1;
+
+    rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
+      "UPDATE security_info SET refcount=:refcount WHERE id=:id;"
+    ), getter_AddRefs(state));
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+    rv = state->BindInt32ByName(NS_LITERAL_CSTRING("refcount"), refcount);
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+    rv = state->BindInt32ByName(NS_LITERAL_CSTRING("id"), *aIdOut);
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+    rv = state->Execute();
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+    return NS_OK;
+  }
+
+  // This is a new security info blob.  Create a new row in the security table
+  // with an initial refcount of 1.
+  rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
+    "INSERT INTO security_info (hash, data, refcount) VALUES (:hash, :data, 1);"
+  ), getter_AddRefs(state));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = state->BindUTF8StringAsBlobByName(NS_LITERAL_CSTRING("hash"), hash);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = state->BindUTF8StringAsBlobByName(NS_LITERAL_CSTRING("data"), aData);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = state->Execute();
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
+    "SELECT last_insert_rowid()"
+  ), getter_AddRefs(state));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  hasMoreData = false;
+  rv = state->ExecuteStep(&hasMoreData);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = state->GetInt32(0, aIdOut);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  return NS_OK;
+}
+
+nsresult
+DeleteSecurityInfo(mozIStorageConnection* aConn, int32_t aId, int32_t aCount)
+{
+  // First, we need to determine the current refcount for this security blob.
+  nsCOMPtr<mozIStorageStatement> state;
+  nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
+    "SELECT refcount FROM security_info WHERE id=:id;"
+  ), getter_AddRefs(state));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = state->BindInt32ByName(NS_LITERAL_CSTRING("id"), aId);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  bool hasMoreData = false;
+  rv = state->ExecuteStep(&hasMoreData);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  int32_t refcount = -1;
+  rv = state->GetInt32(0, &refcount);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  MOZ_ASSERT(refcount >= aCount);
+
+  // Next, calculate the new refcount
+  int32_t newCount = refcount - aCount;
+
+  // If the last reference to this security blob was removed we can
+  // just remove the entire row.
+  if (newCount == 0) {
+    rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
+      "DELETE FROM security_info WHERE id=:id;"
+    ), getter_AddRefs(state));
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+    rv = state->BindInt32ByName(NS_LITERAL_CSTRING("id"), aId);
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+    rv = state->Execute();
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+    return NS_OK;
+  }
+
+  // Otherwise update the refcount in the table to reflect the reduced
+  // number of references to the security blob.
+  rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
+    "UPDATE security_info SET refcount=:refcount WHERE id=:id;"
+  ), getter_AddRefs(state));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = state->BindInt32ByName(NS_LITERAL_CSTRING("refcount"), newCount);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = state->BindInt32ByName(NS_LITERAL_CSTRING("id"), aId);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  rv = state->Execute();
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  return NS_OK;
+}
+
+nsresult
+DeleteSecurityInfoList(mozIStorageConnection* aConn,
+                        const nsTArray<IdCount>& aDeletedStorageIdList)
+{
+  for (uint32_t i = 0; i < aDeletedStorageIdList.Length(); ++i) {
+    nsresult rv = DeleteSecurityInfo(aConn, aDeletedStorageIdList[i].mId,
+                                     aDeletedStorageIdList[i].mCount);
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+  }
+
+  return NS_OK;
+}
+
 nsresult
 InsertEntry(mozIStorageConnection* aConn, CacheId aCacheId,
             const CacheRequest& aRequest,
@@ -1118,8 +1386,16 @@ InsertEntry(mozIStorageConnection* aConn, CacheId aCacheId,
   MOZ_ASSERT(!NS_IsMainThread());
   MOZ_ASSERT(aConn);
 
+  nsresult rv = NS_OK;
+  int32_t securityId = -1;
+
+  if (!aResponse.securityInfo().IsEmpty()) {
+    rv = InsertSecurity(aConn, aResponse.securityInfo(), &securityId);
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+  }
+
   nsCOMPtr<mozIStorageStatement> state;
-  nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
+  rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
     "INSERT INTO entries ("
       "request_method, "
       "request_url, "
@@ -1138,7 +1414,7 @@ InsertEntry(mozIStorageConnection* aConn, CacheId aCacheId,
       "response_status_text, "
       "response_headers_guard, "
       "response_body_id, "
-      "response_security_info, "
+      "response_security_info_id, "
       "cache_id "
     ") VALUES ("
       ":request_method, "
@@ -1158,7 +1434,7 @@ InsertEntry(mozIStorageConnection* aConn, CacheId aCacheId,
       ":response_status_text, "
       ":response_headers_guard, "
       ":response_body_id, "
-      ":response_security_info, "
+      ":response_security_info_id, "
       ":cache_id "
     ");"
   ), getter_AddRefs(state));
@@ -1230,8 +1506,12 @@ InsertEntry(mozIStorageConnection* aConn, CacheId aCacheId,
   rv = BindId(state, NS_LITERAL_CSTRING("response_body_id"), aResponseBodyId);
   if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
-  rv = state->BindUTF8StringAsBlobByName(NS_LITERAL_CSTRING("response_security_info"),
+  if (aResponse.securityInfo().IsEmpty()) {
-                                         aResponse.securityInfo());
+    rv = state->BindNullByName(NS_LITERAL_CSTRING("response_security_info_id"));
+  } else {
+    rv = state->BindInt32ByName(NS_LITERAL_CSTRING("response_security_info_id"),
+                                securityId);
+  }
   if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
   rv = state->BindInt64ByName(NS_LITERAL_CSTRING("cache_id"), aCacheId);
@@ -1319,15 +1599,17 @@ ReadResponse(mozIStorageConnection* aConn, EntryId aEntryId,
   nsCOMPtr<mozIStorageStatement> state;
   nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
     "SELECT "
-      "response_type, "
+      "entries.response_type, "
-      "response_url, "
+      "entries.response_url, "
-      "response_status, "
+      "entries.response_status, "
-      "response_status_text, "
+      "entries.response_status_text, "
-      "response_headers_guard, "
+      "entries.response_headers_guard, "
-      "response_body_id, "
+      "entries.response_body_id, "
-      "response_security_info "
+      "security_info.data "
     "FROM entries "
-    "WHERE id=:id;"
+    "LEFT OUTER JOIN security_info "
+    "ON entries.response_security_info_id=security_info.id "
+    "WHERE entries.id=:id;"
   ), getter_AddRefs(state));
   if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 

dom/cache/test/mochitest/mochitest.ini

@@ -20,6 +20,7 @@ support-files =
   test_cache_requestCache.js
   test_cache_delete.js
   test_cache_put_reorder.js
+  test_cache_https.js
 
 [test_cache.html]
 [test_cache_add.html]
@@ -33,4 +34,6 @@ support-files =
 [test_cache_requestCache.html]
 [test_cache_delete.html]
 [test_cache_put_reorder.html]
+[test_cache_https.html]
+  skip-if = buildapp == 'b2g' # bug 1162353
 [test_cache_restart.html]

dom/cache/test/mochitest/test_cache_https.html

@@ -0,0 +1,20 @@
+<!-- Any copyright is dedicated to the Public Domain.
+   - http://creativecommons.org/publicdomain/zero/1.0/ -->
+<!DOCTYPE HTML>
+<html>
+<head>
+  <title>Validate Interfaces Exposed to Workers</title>
+  <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" />
+  <script type="text/javascript" src="driver.js"></script>
+</head>
+<body>
+<iframe id="frame"></iframe>
+<script class="testbody" type="text/javascript">
+  runTests("test_cache_https.js")
+    .then(function() {
+      SimpleTest.finish();
+    });
+</script>
+</body>
+</html>

dom/cache/test/mochitest/test_cache_https.js

@@ -0,0 +1,23 @@
+var cache = null;
+var name = 'https_' + context;
+var urlBase = 'https://example.com/tests/dom/cache/test/mochitest';
+var url1 = urlBase + '/test_cache.js';
+var url2 = urlBase + '/test_cache_add.js';
+
+caches.open(name).then(function(c) {
+  cache = c;
+  return cache.addAll([new Request(url1, { mode: 'no-cors' }),
+                       new Request(url2, { mode: 'no-cors' })]);
+}).then(function() {
+  return cache.delete(url1);
+}).then(function(result) {
+  ok(result, 'Cache entry should be deleted');
+  return cache.delete(url2);
+}).then(function(result) {
+  ok(result, 'Cache entry should be deleted');
+  cache = null;
+  return caches.delete(name);
+}).then(function(result) {
+  ok(result, 'Cache should be deleted');
+  testDone();
+});