From 4a673bf1efbeeaddd631619f863123de1094eb76 Mon Sep 17 00:00:00 2001 From: Ben Kelly Date: Thu, 21 May 2015 18:16:36 -0700 Subject: [PATCH] Bug 1166038 De-duplicate security info stored in the Cache API. r=ehsan --- dom/cache/DBSchema.cpp | 324 ++++++++++++++++-- dom/cache/test/mochitest/mochitest.ini | 3 + .../test/mochitest/test_cache_https.html | 20 ++ dom/cache/test/mochitest/test_cache_https.js | 23 ++ 4 files changed, 349 insertions(+), 21 deletions(-) create mode 100644 dom/cache/test/mochitest/test_cache_https.html create mode 100644 dom/cache/test/mochitest/test_cache_https.js diff --git a/dom/cache/DBSchema.cpp b/dom/cache/DBSchema.cpp index 1ab63632d5f8..aabf0586f6ba 100644 --- a/dom/cache/DBSchema.cpp +++ b/dom/cache/DBSchema.cpp @@ -18,6 +18,7 @@ #include "nsTArray.h" #include "nsCRT.h" #include "nsHttp.h" +#include "nsICryptoHash.h" #include "mozilla/dom/HeadersBinding.h" #include "mozilla/dom/RequestBinding.h" #include "mozilla/dom/ResponseBinding.h" @@ -28,11 +29,11 @@ namespace dom { namespace cache { namespace db { -const int32_t kMaxWipeSchemaVersion = 8; +const int32_t kMaxWipeSchemaVersion = 9; namespace { -const int32_t kLatestSchemaVersion = 8; +const int32_t kLatestSchemaVersion = 9; const int32_t kMaxEntriesPerStatement = 255; } // anonymous namespace @@ -151,6 +152,14 @@ namespace { typedef int32_t EntryId; +struct IdCount +{ + IdCount() : mId(-1), mCount(0) { } + explicit IdCount(int32_t aId) : mId(aId), mCount(1) { } + int32_t mId; + int32_t mCount; +}; + static nsresult QueryAll(mozIStorageConnection* aConn, CacheId aCacheId, nsTArray& aEntryIdListOut); static nsresult QueryCache(mozIStorageConnection* aConn, CacheId aCacheId, @@ -164,7 +173,14 @@ static nsresult MatchByVaryHeader(mozIStorageConnection* aConn, static nsresult DeleteEntries(mozIStorageConnection* aConn, const nsTArray& aEntryIdList, nsTArray& aDeletedBodyIdListOut, + nsTArray& aDeletedSecurityIdListOut, uint32_t aPos=0, int32_t aLen=-1); +static nsresult InsertSecurity(mozIStorageConnection* aConn, + const nsACString& aData, int32_t *aIdOut); +static nsresult DeleteSecurityInfo(mozIStorageConnection* aConn, int32_t aId, + int32_t aCount); +static nsresult DeleteSecurityInfoList(mozIStorageConnection* aConn, + const nsTArray& aDeletedStorageIdList); static nsresult InsertEntry(mozIStorageConnection* aConn, CacheId aCacheId, const CacheRequest& aRequest, const nsID* aRequestBodyId, @@ -241,6 +257,24 @@ CreateSchema(mozIStorageConnection* aConn) )); if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + // Security blobs are quite large and duplicated for every Response from + // the same https origin. This table is used to de-duplicate this data. + rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "CREATE TABLE security_info (" + "id INTEGER NOT NULL PRIMARY KEY, " + "hash BLOB NOT NULL, " // first 8-bytes of the sha1 hash of data column + "data BLOB NOT NULL, " // full security info data, usually a few KB + "refcount INTEGER NOT NULL" + ");" + )); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Index the smaller hash value instead of the large security data blob. + rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "CREATE INDEX security_info_hash_index ON security_info (hash);" + )); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( "CREATE TABLE entries (" "id INTEGER NOT NULL PRIMARY KEY, " @@ -261,7 +295,7 @@ CreateSchema(mozIStorageConnection* aConn) "response_status_text TEXT NOT NULL, " "response_headers_guard INTEGER NOT NULL, " "response_body_id TEXT NULL, " - "response_security_info BLOB NULL, " + "response_security_info_id INTEGER NULL REFERENCES security_info(id), " "cache_id INTEGER NOT NULL REFERENCES caches(id) ON DELETE CASCADE" ");" )); @@ -406,7 +440,12 @@ DeleteCacheId(mozIStorageConnection* aConn, CacheId aCacheId, nsresult rv = QueryAll(aConn, aCacheId, matches); if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } - rv = DeleteEntries(aConn, matches, aDeletedBodyIdListOut); + nsAutoTArray deletedSecurityIdList; + rv = DeleteEntries(aConn, matches, aDeletedBodyIdListOut, + deletedSecurityIdList); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = DeleteSecurityInfoList(aConn, deletedSecurityIdList); if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } // Delete the remainder of the cache using cascade semantics. @@ -538,13 +577,20 @@ CachePut(mozIStorageConnection* aConn, CacheId aCacheId, nsresult rv = QueryCache(aConn, aCacheId, aRequest, params, matches); if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } - rv = DeleteEntries(aConn, matches, aDeletedBodyIdListOut); + nsAutoTArray deletedSecurityIdList; + rv = DeleteEntries(aConn, matches, aDeletedBodyIdListOut, + deletedSecurityIdList); if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } rv = InsertEntry(aConn, aCacheId, aRequest, aRequestBodyId, aResponse, aResponseBodyId); if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + // Delete the security values after doing the insert to avoid churning + // the security table when its not necessary. + rv = DeleteSecurityInfoList(aConn, deletedSecurityIdList); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + return rv; } @@ -568,7 +614,12 @@ CacheDelete(mozIStorageConnection* aConn, CacheId aCacheId, return rv; } - rv = DeleteEntries(aConn, matches, aDeletedBodyIdListOut); + nsAutoTArray deletedSecurityIdList; + rv = DeleteEntries(aConn, matches, aDeletedBodyIdListOut, + deletedSecurityIdList); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = DeleteSecurityInfoList(aConn, deletedSecurityIdList); if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } *aSuccessOut = true; @@ -1024,6 +1075,7 @@ nsresult DeleteEntries(mozIStorageConnection* aConn, const nsTArray& aEntryIdList, nsTArray& aDeletedBodyIdListOut, + nsTArray& aDeletedSecurityIdListOut, uint32_t aPos, int32_t aLen) { MOZ_ASSERT(!NS_IsMainThread()); @@ -1048,7 +1100,7 @@ DeleteEntries(mozIStorageConnection* aConn, int32_t max = kMaxEntriesPerStatement; int32_t curLen = std::min(max, remaining); nsresult rv = DeleteEntries(aConn, aEntryIdList, aDeletedBodyIdListOut, - curPos, curLen); + aDeletedSecurityIdListOut, curPos, curLen); if (NS_FAILED(rv)) { return rv; } curPos += curLen; @@ -1059,7 +1111,8 @@ DeleteEntries(mozIStorageConnection* aConn, nsCOMPtr state; nsAutoCString query( - "SELECT request_body_id, response_body_id FROM entries WHERE id IN (" + "SELECT request_body_id, response_body_id, response_security_info_id " + "FROM entries WHERE id IN (" ); AppendListParamsToQuery(query, aEntryIdList, aPos, aLen); query.AppendLiteral(")"); @@ -1086,6 +1139,33 @@ DeleteEntries(mozIStorageConnection* aConn, aDeletedBodyIdListOut.AppendElement(id); } } + + // and then a possible third entry for the security id + bool isNull = false; + rv = state->GetIsNull(2, &isNull); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + if (!isNull) { + int32_t securityId = -1; + rv = state->GetInt32(2, &securityId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // First try to increment the count for this ID if we're already + // seen it + bool found = false; + for (uint32_t i = 0; i < aDeletedSecurityIdListOut.Length(); ++i) { + if (aDeletedSecurityIdListOut[i].mId == securityId) { + found = true; + aDeletedSecurityIdListOut[i].mCount += 1; + break; + } + } + + // Otherwise add a new entry for this ID with a count of 1 + if (!found) { + aDeletedSecurityIdListOut.AppendElement(IdCount(securityId)); + } + } } // Dependent records removed via ON DELETE CASCADE @@ -1108,6 +1188,194 @@ DeleteEntries(mozIStorageConnection* aConn, return rv; } +nsresult +InsertSecurity(mozIStorageConnection* aConn, const nsACString& aData, + int32_t *aIdOut) +{ + MOZ_ASSERT(aConn); + MOZ_ASSERT(aIdOut); + MOZ_ASSERT(!aData.IsEmpty()); + + // We want to use an index to find existing security blobs, but indexing + // the full blob would be quite expensive. Instead, we index a small + // hash value. Calculate this hash as the first 8 bytes of the SHA1 of + // the full data. + nsresult rv; + nsCOMPtr crypto = + do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = crypto->Init(nsICryptoHash::SHA1); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = crypto->Update(reinterpret_cast(aData.BeginReading()), + aData.Length()); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsAutoCString fullHash; + rv = crypto->Finish(false /* based64 result */, fullHash); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsDependentCSubstring hash(fullHash, 0, 8); + + // Next, search for an existing entry for this blob by comparing the hash + // value first and then the full data. SQLite is smart enough to use + // the index on the hash to search the table before doing the expensive + // comparison of the large data column. (This was verified with EXPLAIN.) + nsCOMPtr state; + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + // Note that hash and data are blobs, but we can use = here since the + // columns are NOT NULL. + "SELECT id, refcount FROM security_info WHERE hash=:hash AND data=:data;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindUTF8StringAsBlobByName(NS_LITERAL_CSTRING("hash"), hash); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindUTF8StringAsBlobByName(NS_LITERAL_CSTRING("data"), aData); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + bool hasMoreData = false; + rv = state->ExecuteStep(&hasMoreData); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // This security info blob is already in the database + if (hasMoreData) { + // get the existing security blob id to return + rv = state->GetInt32(0, aIdOut); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + int32_t refcount = -1; + rv = state->GetInt32(1, &refcount); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // But first, update the refcount in the database. + refcount += 1; + + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "UPDATE security_info SET refcount=:refcount WHERE id=:id;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("refcount"), refcount); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("id"), *aIdOut); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->Execute(); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return NS_OK; + } + + // This is a new security info blob. Create a new row in the security table + // with an initial refcount of 1. + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "INSERT INTO security_info (hash, data, refcount) VALUES (:hash, :data, 1);" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindUTF8StringAsBlobByName(NS_LITERAL_CSTRING("hash"), hash); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindUTF8StringAsBlobByName(NS_LITERAL_CSTRING("data"), aData); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->Execute(); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "SELECT last_insert_rowid()" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + hasMoreData = false; + rv = state->ExecuteStep(&hasMoreData); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->GetInt32(0, aIdOut); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return NS_OK; +} + +nsresult +DeleteSecurityInfo(mozIStorageConnection* aConn, int32_t aId, int32_t aCount) +{ + // First, we need to determine the current refcount for this security blob. + nsCOMPtr state; + nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "SELECT refcount FROM security_info WHERE id=:id;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("id"), aId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + bool hasMoreData = false; + rv = state->ExecuteStep(&hasMoreData); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + int32_t refcount = -1; + rv = state->GetInt32(0, &refcount); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + MOZ_ASSERT(refcount >= aCount); + + // Next, calculate the new refcount + int32_t newCount = refcount - aCount; + + // If the last reference to this security blob was removed we can + // just remove the entire row. + if (newCount == 0) { + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "DELETE FROM security_info WHERE id=:id;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("id"), aId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->Execute(); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return NS_OK; + } + + // Otherwise update the refcount in the table to reflect the reduced + // number of references to the security blob. + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "UPDATE security_info SET refcount=:refcount WHERE id=:id;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("refcount"), newCount); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("id"), aId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->Execute(); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return NS_OK; +} + +nsresult +DeleteSecurityInfoList(mozIStorageConnection* aConn, + const nsTArray& aDeletedStorageIdList) +{ + for (uint32_t i = 0; i < aDeletedStorageIdList.Length(); ++i) { + nsresult rv = DeleteSecurityInfo(aConn, aDeletedStorageIdList[i].mId, + aDeletedStorageIdList[i].mCount); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + } + + return NS_OK; +} + nsresult InsertEntry(mozIStorageConnection* aConn, CacheId aCacheId, const CacheRequest& aRequest, @@ -1118,8 +1386,16 @@ InsertEntry(mozIStorageConnection* aConn, CacheId aCacheId, MOZ_ASSERT(!NS_IsMainThread()); MOZ_ASSERT(aConn); + nsresult rv = NS_OK; + int32_t securityId = -1; + + if (!aResponse.securityInfo().IsEmpty()) { + rv = InsertSecurity(aConn, aResponse.securityInfo(), &securityId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + } + nsCOMPtr state; - nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( "INSERT INTO entries (" "request_method, " "request_url, " @@ -1138,7 +1414,7 @@ InsertEntry(mozIStorageConnection* aConn, CacheId aCacheId, "response_status_text, " "response_headers_guard, " "response_body_id, " - "response_security_info, " + "response_security_info_id, " "cache_id " ") VALUES (" ":request_method, " @@ -1158,7 +1434,7 @@ InsertEntry(mozIStorageConnection* aConn, CacheId aCacheId, ":response_status_text, " ":response_headers_guard, " ":response_body_id, " - ":response_security_info, " + ":response_security_info_id, " ":cache_id " ");" ), getter_AddRefs(state)); @@ -1230,8 +1506,12 @@ InsertEntry(mozIStorageConnection* aConn, CacheId aCacheId, rv = BindId(state, NS_LITERAL_CSTRING("response_body_id"), aResponseBodyId); if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } - rv = state->BindUTF8StringAsBlobByName(NS_LITERAL_CSTRING("response_security_info"), - aResponse.securityInfo()); + if (aResponse.securityInfo().IsEmpty()) { + rv = state->BindNullByName(NS_LITERAL_CSTRING("response_security_info_id")); + } else { + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("response_security_info_id"), + securityId); + } if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } rv = state->BindInt64ByName(NS_LITERAL_CSTRING("cache_id"), aCacheId); @@ -1319,15 +1599,17 @@ ReadResponse(mozIStorageConnection* aConn, EntryId aEntryId, nsCOMPtr state; nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING( "SELECT " - "response_type, " - "response_url, " - "response_status, " - "response_status_text, " - "response_headers_guard, " - "response_body_id, " - "response_security_info " + "entries.response_type, " + "entries.response_url, " + "entries.response_status, " + "entries.response_status_text, " + "entries.response_headers_guard, " + "entries.response_body_id, " + "security_info.data " "FROM entries " - "WHERE id=:id;" + "LEFT OUTER JOIN security_info " + "ON entries.response_security_info_id=security_info.id " + "WHERE entries.id=:id;" ), getter_AddRefs(state)); if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } diff --git a/dom/cache/test/mochitest/mochitest.ini b/dom/cache/test/mochitest/mochitest.ini index 4ebc3d844d4b..573a515938c2 100644 --- a/dom/cache/test/mochitest/mochitest.ini +++ b/dom/cache/test/mochitest/mochitest.ini @@ -20,6 +20,7 @@ support-files = test_cache_requestCache.js test_cache_delete.js test_cache_put_reorder.js + test_cache_https.js [test_cache.html] [test_cache_add.html] @@ -33,4 +34,6 @@ support-files = [test_cache_requestCache.html] [test_cache_delete.html] [test_cache_put_reorder.html] +[test_cache_https.html] + skip-if = buildapp == 'b2g' # bug 1162353 [test_cache_restart.html] diff --git a/dom/cache/test/mochitest/test_cache_https.html b/dom/cache/test/mochitest/test_cache_https.html new file mode 100644 index 000000000000..8ec509f0e812 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_https.html @@ -0,0 +1,20 @@ + + + + + Validate Interfaces Exposed to Workers + + + + + + + + + diff --git a/dom/cache/test/mochitest/test_cache_https.js b/dom/cache/test/mochitest/test_cache_https.js new file mode 100644 index 000000000000..1c1cfffebd2b --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_https.js @@ -0,0 +1,23 @@ +var cache = null; +var name = 'https_' + context; +var urlBase = 'https://example.com/tests/dom/cache/test/mochitest'; +var url1 = urlBase + '/test_cache.js'; +var url2 = urlBase + '/test_cache_add.js'; + +caches.open(name).then(function(c) { + cache = c; + return cache.addAll([new Request(url1, { mode: 'no-cors' }), + new Request(url2, { mode: 'no-cors' })]); +}).then(function() { + return cache.delete(url1); +}).then(function(result) { + ok(result, 'Cache entry should be deleted'); + return cache.delete(url2); +}).then(function(result) { + ok(result, 'Cache entry should be deleted'); + cache = null; + return caches.delete(name); +}).then(function(result) { + ok(result, 'Cache should be deleted'); + testDone(); +});