diff --git a/netwerk/cache/nsCacheService.cpp b/netwerk/cache/nsCacheService.cpp index 56e2827e7e30..c1e756970b2f 100644 --- a/netwerk/cache/nsCacheService.cpp +++ b/netwerk/cache/nsCacheService.cpp @@ -2852,6 +2852,12 @@ nsCacheService::ClearPendingRequests(nsCacheEntry * entry) } } +bool +nsCacheService::IsDoomListEmpty() +{ + nsCacheEntry * entry = (nsCacheEntry *)PR_LIST_HEAD(&mDoomedEntries); + return &mDoomedEntries == entry; +} void nsCacheService::ClearDoomList() diff --git a/netwerk/cache/nsCacheService.h b/netwerk/cache/nsCacheService.h index b73b6e8a3468..ed222f477b86 100644 --- a/netwerk/cache/nsCacheService.h +++ b/netwerk/cache/nsCacheService.h @@ -178,6 +178,7 @@ public: { gService->mLock.AssertCurrentThreadOwns(); } static void LeavePrivateBrowsing(); + bool IsDoomListEmpty(); typedef bool (*DoomCheckFn)(nsCacheEntry* entry); @@ -190,6 +191,7 @@ private: friend class nsSetDiskSmartSizeCallback; friend class nsDoomEvent; friend class nsDisableOldMaxSmartSizePrefEvent; + friend class nsDiskCacheMap; /** * Internal Methods diff --git a/netwerk/cache/nsDiskCache.h b/netwerk/cache/nsDiskCache.h index 319b6b51ee3a..e27f15fd4d39 100644 --- a/netwerk/cache/nsDiskCache.h +++ b/netwerk/cache/nsDiskCache.h @@ -49,7 +49,10 @@ public: kBlockFileSizeLessThanBitMap = 19, kBlockFileBitMapReadError = 20, kBlockFileEstimatedSizeError = 21, - kFlushHeaderError = 22 + kFlushHeaderError = 22, + kCacheCleanFilePathError = 23, + kCacheCleanOpenFileError = 24, + kCacheCleanTimerError = 25 }; // Parameter initval initializes internal state of hash function. Hash values are different diff --git a/netwerk/cache/nsDiskCacheMap.cpp b/netwerk/cache/nsDiskCacheMap.cpp index 5410258b5b7b..a427c89e3d50 100644 --- a/netwerk/cache/nsDiskCacheMap.cpp +++ b/netwerk/cache/nsDiskCacheMap.cpp @@ -7,6 +7,7 @@ #include "nsDiskCacheMap.h" #include "nsDiskCacheBinding.h" #include "nsDiskCacheEntry.h" +#include "nsCacheService.h" #include "nsCache.h" @@ -18,6 +19,8 @@ #include "mozilla/Telemetry.h" +using namespace mozilla; + /****************************************************************************** * nsDiskCacheMap *****************************************************************************/ @@ -56,9 +59,14 @@ nsDiskCacheMap::Open(nsIFile * cacheDirectory, bool cacheFilesExist = CacheFilesExist(); rv = NS_ERROR_FILE_CORRUPTED; // presume the worst + PRUint32 mapSize = PR_Available(mMapFD); + + if (NS_FAILED(InitCacheClean(cacheDirectory, corruptInfo))) { + // corruptInfo is set in the call to InitCacheClean + goto error_exit; + } // check size of map file - PRUint32 mapSize = PR_Available(mMapFD); if (mapSize == 0) { // creating a new _CACHE_MAP_ // block files shouldn't exist if we're creating the _CACHE_MAP_ @@ -173,8 +181,7 @@ nsDiskCacheMap::Open(nsIFile * cacheDirectory, // extra scope so the compiler doesn't barf on the above gotos jumping // past this declaration down here PRUint32 overhead = moz_malloc_size_of(mRecordArray); - mozilla::Telemetry::Accumulate(mozilla::Telemetry::HTTP_DISK_CACHE_OVERHEAD, - overhead); + Telemetry::Accumulate(Telemetry::HTTP_DISK_CACHE_OVERHEAD, overhead); } *corruptInfo = nsDiskCache::kNotCorrupt; @@ -192,6 +199,12 @@ nsDiskCacheMap::Close(bool flush) { nsresult rv = NS_OK; + // Cancel any pending cache validation event, the FlushRecords call below + // will validate the cache. + if (mCleanCacheTimer) { + mCleanCacheTimer->Cancel(); + } + // If cache map file and its block files are still open, close them if (mMapFD) { // close block files @@ -210,6 +223,12 @@ nsDiskCacheMap::Close(bool flush) mMapFD = nullptr; } + + if (mCleanFD) { + PR_Close(mCleanFD); + mCleanFD = nullptr; + } + PR_FREEIF(mRecordArray); PR_FREEIF(mBuffer); mBufferSize = 0; @@ -235,6 +254,7 @@ nsDiskCacheMap::Trim() nsresult nsDiskCacheMap::FlushHeader() { + RevalidateCache(); if (!mMapFD) return NS_ERROR_NOT_AVAILABLE; // seek to beginning of cache map @@ -347,6 +367,9 @@ nsDiskCacheMap::GrowRecords() // Set as the new record array mRecordArray = newArray; mHeader.mRecordCount = newCount; + + InvalidateCache(); + return NS_OK; } @@ -393,6 +416,9 @@ nsDiskCacheMap::ShrinkRecords() // Set as the new record array mRecordArray = newArray; mHeader.mRecordCount = newCount; + + InvalidateCache(); + return NS_OK; } @@ -421,6 +447,7 @@ nsDiskCacheMap::AddRecord( nsDiskCacheRecord * mapRecord, mHeader.mBucketUsage[bucketIndex]++; if (mHeader.mEvictionRank[bucketIndex] < mapRecord->EvictionRank()) mHeader.mEvictionRank[bucketIndex] = mapRecord->EvictionRank(); + InvalidateCache(); } else { // Find the record with the highest eviction rank nsDiskCacheRecord * mostEvictable = &records[0]; @@ -436,6 +463,7 @@ nsDiskCacheMap::AddRecord( nsDiskCacheRecord * mapRecord, mHeader.mEvictionRank[bucketIndex] = mapRecord->EvictionRank(); if (oldRecord->EvictionRank() >= mHeader.mEvictionRank[bucketIndex]) mHeader.mEvictionRank[bucketIndex] = GetBucketRank(bucketIndex, 0); + InvalidateCache(); } NS_ASSERTION(mHeader.mEvictionRank[bucketIndex] == GetBucketRank(bucketIndex, 0), @@ -466,6 +494,8 @@ nsDiskCacheMap::UpdateRecord( nsDiskCacheRecord * mapRecord) else if (mHeader.mEvictionRank[bucketIndex] == oldRank) mHeader.mEvictionRank[bucketIndex] = GetBucketRank(bucketIndex, 0); + InvalidateCache(); + NS_ASSERTION(mHeader.mEvictionRank[bucketIndex] == GetBucketRank(bucketIndex, 0), "eviction rank out of sync"); return NS_OK; @@ -521,6 +551,8 @@ nsDiskCacheMap::DeleteRecord( nsDiskCacheRecord * mapRecord) mHeader.mEvictionRank[bucketIndex] = GetBucketRank(bucketIndex, 0); } + InvalidateCache(); + NS_ASSERTION(mHeader.mEvictionRank[bucketIndex] == GetBucketRank(bucketIndex, 0), "eviction rank out of sync"); return NS_OK; @@ -551,6 +583,7 @@ nsDiskCacheMap::VisitEachRecord(PRUint32 bucketIndex, --count; records[i] = records[count]; records[count].SetHashNumber(0); + InvalidateCache(); } } @@ -1174,3 +1207,186 @@ nsDiskCacheMap::NotifyCapacityChange(PRUint32 capacity) mMaxRecordCount = maxRecordCount; } } + +nsresult +nsDiskCacheMap::InitCacheClean(nsIFile * cacheDirectory, + nsDiskCache::CorruptCacheInfo * corruptInfo) +{ + // The _CACHE_CLEAN_ file will be used in the future to determine + // if the cache is clean or not. + bool cacheCleanFileExists = false; + nsCOMPtr cacheCleanFile; + nsresult rv = cacheDirectory->Clone(getter_AddRefs(cacheCleanFile)); + if (NS_SUCCEEDED(rv)) { + rv = cacheCleanFile->AppendNative( + NS_LITERAL_CSTRING("_CACHE_CLEAN_")); + if (NS_SUCCEEDED(rv)) { + // Check if the file already exists, if it does, we will later read the + // value and report it to telemetry. + cacheCleanFile->Exists(&cacheCleanFileExists); + } + } + if (NS_FAILED(rv)) { + NS_WARNING("Could not build cache clean file path"); + *corruptInfo = nsDiskCache::kCacheCleanFilePathError; + return rv; + } + + // Make sure the _CACHE_CLEAN_ file exists + rv = cacheCleanFile->OpenNSPRFileDesc(PR_RDWR | PR_CREATE_FILE, + 00600, &mCleanFD); + if (NS_FAILED(rv)) { + NS_WARNING("Could not open cache clean file"); + *corruptInfo = nsDiskCache::kCacheCleanOpenFileError; + return rv; + } + + if (cacheCleanFileExists) { + char clean = '0'; + PRInt32 bytesRead = PR_Read(mCleanFD, &clean, 1); + if (bytesRead != 1) { + NS_WARNING("Could not read _CACHE_CLEAN_ file contents"); + } else { + Telemetry::Accumulate(Telemetry::DISK_CACHE_REDUCTION_TRIAL, + clean == '1' ? 1 : 0); + } + } + + // Create a timer that will be used to validate the cache + // as long as an activity threshold was met + mCleanCacheTimer = do_CreateInstance("@mozilla.org/timer;1", &rv); + if (NS_SUCCEEDED(rv)) { + mCleanCacheTimer->SetTarget(nsCacheService::GlobalInstance()->mCacheIOThread); + rv = ResetCacheTimer(); + } + + if (NS_FAILED(rv)) { + NS_WARNING("Could not create cache clean timer"); + mCleanCacheTimer = nullptr; + *corruptInfo = nsDiskCache::kCacheCleanTimerError; + return rv; + } + + return NS_OK; +} + +nsresult +nsDiskCacheMap::WriteCacheClean(bool clean) +{ + nsCacheService::AssertOwnsLock(); + CACHE_LOG_DEBUG(("CACHE: WriteCacheClean: %d\n", clean? 1 : 0)); + // I'm using a simple '1' or '0' to denote cache clean + // since it can be edited easily by any text editor for testing. + char data = clean? '1' : '0'; + PRInt32 filePos = PR_Seek(mCleanFD, 0, PR_SEEK_SET); + if (filePos != 0) { + NS_WARNING("Could not seek in cache map file!"); + return NS_ERROR_FAILURE; + } + PRInt32 bytesWritten = PR_Write(mCleanFD, &data, 1); + if (bytesWritten != 1) { + NS_WARNING("Could not write cache map file!"); + return NS_ERROR_FAILURE; + } + PRStatus err = PR_Sync(mCleanFD); + if (err != PR_SUCCESS) { + NS_WARNING("Could not flush mCleanFD!"); + } + + return NS_OK; +} + +nsresult +nsDiskCacheMap::InvalidateCache() +{ + nsCacheService::AssertOwnsLock(); + CACHE_LOG_DEBUG(("CACHE: InvalidateCache\n")); + nsresult rv; + + if (!mIsDirtyCacheFlushed) { + rv = WriteCacheClean(false); + NS_ENSURE_SUCCESS(rv, rv); + mIsDirtyCacheFlushed = true; + } + + rv = ResetCacheTimer(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult +nsDiskCacheMap::ResetCacheTimer(PRInt32 timeout) +{ + mCleanCacheTimer->Cancel(); + nsresult rv = + mCleanCacheTimer->InitWithFuncCallback(RevalidateTimerCallback, + this, timeout, + nsITimer::TYPE_ONE_SHOT); + NS_ENSURE_SUCCESS(rv, rv); + mLastInvalidateTime = PR_IntervalNow(); + + return rv; +} + +void +nsDiskCacheMap::RevalidateTimerCallback(nsITimer *aTimer, void *arg) +{ + nsDiskCacheMap *diskCacheMap = reinterpret_cast(arg); + nsresult rv; + + // Intentional braces to scope mutex to only what is needed + { + nsCacheServiceAutoLock lock(LOCK_TELEM(NSDISKCACHEMAP_REVALIDATION)); + // If we have less than kLastInvalidateTime since the last timer was + // issued then another thread called InvalidateCache. This won't catch + // all cases where we wanted to cancel the timer, but under the lock it + // is always OK to revalidate as long as IsCacheInSafeState() returns + // true. We just want to avoid revalidating when we can to reduce IO + // and this check will do that. + PRUint32 delta = + PR_IntervalToMilliseconds(PR_IntervalNow() - + diskCacheMap->mLastInvalidateTime) + + kRevalidateCacheTimeoutTolerance; + if (delta < kRevalidateCacheTimeout) { + diskCacheMap->ResetCacheTimer(); + return; + } + rv = diskCacheMap->RevalidateCache(); + } + + if (NS_FAILED(rv)) { + diskCacheMap->ResetCacheTimer(kRevalidateCacheErrorTimeout); + } +} + +bool +nsDiskCacheMap::IsCacheInSafeState() +{ + return nsCacheService::GlobalInstance()->IsDoomListEmpty(); +} + +nsresult +nsDiskCacheMap::RevalidateCache() +{ + CACHE_LOG_DEBUG(("CACHE: RevalidateCache\n")); + nsresult rv; + + if (!IsCacheInSafeState()) { + CACHE_LOG_DEBUG(("CACHE: Revalidation not performed because " + "cache not in a safe state\n")); + return NS_ERROR_FAILURE; + } + + // We want this after the lock to prove that flushing a file isn't that expensive + Telemetry::AutoTimer totalTimer; + + // If telemetry data shows it is worth it, we'll be flushing headers and + // records before flushing the clean cache file. + + // Write out the _CACHE_CLEAN_ file with '1' + rv = WriteCacheClean(true); + mIsDirtyCacheFlushed = false; + + return NS_OK; +} diff --git a/netwerk/cache/nsDiskCacheMap.h b/netwerk/cache/nsDiskCacheMap.h index 6da0638af54b..1c530aef2c0e 100644 --- a/netwerk/cache/nsDiskCacheMap.h +++ b/netwerk/cache/nsDiskCacheMap.h @@ -14,6 +14,7 @@ #include "nsDebug.h" #include "nsError.h" #include "nsIFile.h" +#include "nsITimer.h" #include "nsDiskCache.h" #include "nsDiskCacheBlockFile.h" @@ -78,6 +79,12 @@ struct nsDiskCacheEntry; // preallocate up to 1MB of separate cache file #define kPreallocateLimit 1 * 1024 * 1024 +// The minimum amount of milliseconds to wait before re-attempting to +// revalidate the cache. +#define kRevalidateCacheTimeout 5000 +#define kRevalidateCacheTimeoutTolerance 10 +#define kRevalidateCacheErrorTimeout 1000 + class nsDiskCacheRecord { private: @@ -378,13 +385,17 @@ public: nsDiskCacheMap() : mCacheDirectory(nullptr), mMapFD(nullptr), + mCleanFD(nullptr), mRecordArray(nullptr), mBufferSize(0), mBuffer(nullptr), - mMaxRecordCount(16384) // this default value won't matter + mMaxRecordCount(16384), // this default value won't matter + mIsDirtyCacheFlushed(false), + mLastInvalidateTime(0) { } - ~nsDiskCacheMap() { + ~nsDiskCacheMap() + { (void) Close(true); } @@ -526,18 +537,39 @@ private: nsDiskCacheEntry * CreateDiskCacheEntry(nsDiskCacheBinding * binding, PRUint32 * size); -/** + // Initializes the _CACHE_CLEAN_ related functionality + nsresult InitCacheClean(nsIFile * cacheDirectory, + nsDiskCache::CorruptCacheInfo * corruptInfo); + // Writes out a value of '0' or '1' in the _CACHE_CLEAN_ file + nsresult WriteCacheClean(bool clean); + // Resets the timout for revalidating the cache + nsresult ResetCacheTimer(PRInt32 timeout = kRevalidateCacheTimeout); + // Invalidates the cache, calls WriteCacheClean and ResetCacheTimer + nsresult InvalidateCache(); + // Determines if the cache is in a safe state + bool IsCacheInSafeState(); + // Revalidates the cache by writting out the header, records, and finally + // by calling WriteCacheClean(true). + nsresult RevalidateCache(); + // Timer which revalidates the cache + static void RevalidateTimerCallback(nsITimer *aTimer, void *arg); + +/** * data members */ private: + nsCOMPtr mCleanCacheTimer; nsCOMPtr mCacheDirectory; PRFileDesc * mMapFD; + PRFileDesc * mCleanFD; nsDiskCacheRecord * mRecordArray; nsDiskCacheBlockFile mBlockFile[kNumBlockFiles]; PRUint32 mBufferSize; char * mBuffer; nsDiskCacheHeader mHeader; PRInt32 mMaxRecordCount; + bool mIsDirtyCacheFlushed; + PRIntervalTime mLastInvalidateTime; }; #endif // _nsDiskCacheMap_h_ diff --git a/toolkit/components/telemetry/TelemetryHistograms.h b/toolkit/components/telemetry/TelemetryHistograms.h index 168f280a2262..1faf85693ef5 100644 --- a/toolkit/components/telemetry/TelemetryHistograms.h +++ b/toolkit/components/telemetry/TelemetryHistograms.h @@ -201,6 +201,7 @@ HISTOGRAM(SPDY_SETTINGS_IW, 1, 1000, 50, EXPONENTIAL, "SPDY: Settings IW (round #undef HTTP_HISTOGRAMS HISTOGRAM(DISK_CACHE_CORRUPT_DETAILS, 1, 50, 51, LINEAR, "Why the HTTP disk cache was corrupted at startup") +HISTOGRAM_BOOLEAN(DISK_CACHE_REDUCTION_TRIAL, "Stores 1 if the cache would be corrupted with the disk cache corruption plan of Bug 105843") HISTOGRAM_ENUMERATED_VALUES(HTTP_CACHE_DISPOSITION_2, 5, "HTTP Cache Hit, Reval, Failed-Reval, Miss") HISTOGRAM_ENUMERATED_VALUES(HTTP_DISK_CACHE_DISPOSITION_2, 5, "HTTP Disk Cache Hit, Reval, Failed-Reval, Miss") HISTOGRAM_ENUMERATED_VALUES(HTTP_MEMORY_CACHE_DISPOSITION_2, 5, "HTTP Memory Cache Hit, Reval, Failed-Reval, Miss") @@ -279,6 +280,7 @@ CACHE_LOCK_HISTOGRAM(NSCACHEENTRYDESCRIPTOR_GETPREDICTEDDATASIZE) CACHE_LOCK_HISTOGRAM(NSCACHEENTRYDESCRIPTOR_GETLASTFETCHED) CACHE_LOCK_HISTOGRAM(NSCACHEENTRYDESCRIPTOR_GETCLIENTID) CACHE_LOCK_HISTOGRAM(NSBLOCKONCACHETHREADEVENT_RUN) +CACHE_LOCK_HISTOGRAM(NSDISKCACHEMAP_REVALIDATION) #undef CACHE_LOCK_HISTOGRAM @@ -332,6 +334,7 @@ HISTOGRAM(NETWORK_DISK_CACHE_DELETEDIR, 1, 10000, 10, EXPONENTIAL, "Time spent d HISTOGRAM(NETWORK_DISK_CACHE_DELETEDIR_SHUTDOWN, 1, 10000, 10, EXPONENTIAL, "Time spent during showdown stopping thread deleting old disk cache (ms)") HISTOGRAM(NETWORK_DISK_CACHE_SHUTDOWN, 1, 10000, 10, EXPONENTIAL, "Total Time spent (ms) during disk cache showdown") HISTOGRAM(NETWORK_DISK_CACHE_SHUTDOWN_CLEAR_PRIVATE, 1, 10000, 10, EXPONENTIAL, "Time spent (ms) during showdown deleting disk cache for 'clear private data' option") +HISTOGRAM(NETWORK_DISK_CACHE_REVALIDATION, 1, 10000, 10, EXPONENTIAL, "Total Time spent (ms) during disk cache revalidation") HISTOGRAM(NETWORK_DISK_CACHE_OUTPUT_STREAM_CLOSE, 1, 10000, 10, EXPONENTIAL, "Time spent in nsDiskCacheOutputStream::Close() on non-main thread (ms)") HISTOGRAM(NETWORK_DISK_CACHE_OUTPUT_STREAM_CLOSE_MAIN_THREAD, 1, 10000, 10, EXPONENTIAL, "Time spent in nsDiskCacheOutputStream::Close() on the main thread (ms)") HISTOGRAM(NETWORK_DISK_CACHE_OUTPUT_STREAM_CLOSE_INTERNAL, 1, 10000, 10, EXPONENTIAL, "Time spent in nsDiskCacheOutputStream::CloseInternal() on non-main thread (ms)")