From 8eb68da410685a53487e55f691a7919365690a76 Mon Sep 17 00:00:00 2001 From: Tom Tung Date: Tue, 18 Jul 2017 18:57:54 +0800 Subject: [PATCH] Bug 1290481 - P8: Implement few utility functions to access direcotry padding file. r=bkelly MozReview-Commit-ID: KlVsaGhpABk --HG-- extra : rebase_source : 23145f66eb62f2f858311513f6bec4ead01d18cd --- dom/cache/FileUtils.cpp | 266 ++++++++++++++++++++++++++++++++++++++ dom/cache/FileUtils.h | 49 +++++++ dom/cache/QuotaClient.cpp | 115 +++++++++++++++- dom/cache/QuotaClient.h | 41 ++++++ 4 files changed, 469 insertions(+), 2 deletions(-) diff --git a/dom/cache/FileUtils.cpp b/dom/cache/FileUtils.cpp index dac9281a890f..25c6f70a3393 100644 --- a/dom/cache/FileUtils.cpp +++ b/dom/cache/FileUtils.cpp @@ -11,9 +11,12 @@ #include "mozilla/dom/quota/QuotaManager.h" #include "mozilla/SnappyCompressOutputStream.h" #include "mozilla/Unused.h" +#include "nsIBinaryInputStream.h" +#include "nsIBinaryOutputStream.h" #include "nsIFile.h" #include "nsIUUIDGenerator.h" #include "nsNetCID.h" +#include "nsNetUtil.h" #include "nsISimpleEnumerator.h" #include "nsServiceManagerUtils.h" #include "nsString.h" @@ -23,6 +26,9 @@ namespace mozilla { namespace dom { namespace cache { +#define PADDING_FILE_NAME ".padding" +#define PADDING_TMP_FILE_NAME ".padding-tmp" + using mozilla::dom::quota::FileInputStream; using mozilla::dom::quota::FileOutputStream; using mozilla::dom::quota::PERSISTENCE_TYPE_DEFAULT; @@ -57,6 +63,10 @@ RoundUp(const int64_t aX, const int64_t aY); int64_t BodyGeneratePadding(const int64_t aBodyFileSize, const uint32_t aPaddingInfo); +nsresult +LockedDirectoryPaddingWrite(nsIFile* aBaseDir, DirPaddingFile aPaddingFileType, + int64_t aPaddingSize); + } // namespace // static @@ -388,6 +398,41 @@ BodyGeneratePadding(const int64_t aBodyFileSize, const uint32_t aPaddingInfo) return RoundUp(randomSize, kRoundUpNumber) - aBodyFileSize; } +nsresult +LockedDirectoryPaddingWrite(nsIFile* aBaseDir, DirPaddingFile aPaddingFileType, + int64_t aPaddingSize) +{ + MOZ_DIAGNOSTIC_ASSERT(aBaseDir); + MOZ_DIAGNOSTIC_ASSERT(aPaddingSize >= 0); + + nsCOMPtr file; + nsresult rv = aBaseDir->Clone(getter_AddRefs(file)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + if (aPaddingFileType == DirPaddingFile::TMP_FILE) { + rv = file->Append(NS_LITERAL_STRING(PADDING_TMP_FILE_NAME)); + } else { + rv = file->Append(NS_LITERAL_STRING(PADDING_FILE_NAME)); + } + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsCOMPtr outputStream; + rv = NS_NewLocalFileOutputStream(getter_AddRefs(outputStream), file); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsCOMPtr binaryStream = + do_CreateInstance("@mozilla.org/binaryoutputstream;1"); + if (NS_WARN_IF(!binaryStream)) { return NS_ERROR_FAILURE; } + + rv = binaryStream->SetOutputStream(outputStream); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = binaryStream->Write64(aPaddingSize); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return rv; +} + } // namespace nsresult @@ -648,6 +693,227 @@ DecreaseUsageForQuotaInfo(const QuotaInfo& aQuotaInfo, aUpdatingSize); } +// static +bool +DirectoryPaddingFileExists(nsIFile* aBaseDir, DirPaddingFile aPaddingFileType) +{ + MOZ_DIAGNOSTIC_ASSERT(aBaseDir); + + nsCOMPtr file; + nsresult rv = aBaseDir->Clone(getter_AddRefs(file)); + if (NS_WARN_IF(NS_FAILED(rv))) { return false; } + + nsString fileName; + if (aPaddingFileType == DirPaddingFile::TMP_FILE) { + fileName = NS_LITERAL_STRING(PADDING_TMP_FILE_NAME); + } else { + fileName = NS_LITERAL_STRING(PADDING_FILE_NAME); + } + + rv = file->Append(fileName); + if (NS_WARN_IF(NS_FAILED(rv))) { return false; } + + bool exists = false; + rv = file->Exists(&exists); + if (NS_WARN_IF(NS_FAILED(rv))) { return false; } + + return exists; +} + +// static +nsresult +LockedDirectoryPaddingGet(nsIFile* aBaseDir, int64_t* aPaddingSizeOut) +{ + MOZ_DIAGNOSTIC_ASSERT(aBaseDir); + MOZ_DIAGNOSTIC_ASSERT(aPaddingSizeOut); + MOZ_DIAGNOSTIC_ASSERT(!DirectoryPaddingFileExists(aBaseDir, + DirPaddingFile::TMP_FILE)); + + nsCOMPtr file; + nsresult rv = aBaseDir->Clone(getter_AddRefs(file)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = file->Append(NS_LITERAL_STRING(PADDING_FILE_NAME)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsCOMPtr stream; + rv = NS_NewLocalFileInputStream(getter_AddRefs(stream), file); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsCOMPtr bufferedStream; + rv = NS_NewBufferedInputStream(getter_AddRefs(bufferedStream), stream, 512); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsCOMPtr binaryStream = + do_CreateInstance("@mozilla.org/binaryinputstream;1"); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = binaryStream->SetInputStream(bufferedStream); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + uint64_t paddingSize = 0; + rv = binaryStream->Read64(&paddingSize); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + *aPaddingSizeOut = paddingSize; + + return rv; +} + +// static +nsresult +LockedDirectoryPaddingInit(nsIFile* aBaseDir) +{ + MOZ_DIAGNOSTIC_ASSERT(aBaseDir); + + nsresult rv = LockedDirectoryPaddingWrite(aBaseDir, DirPaddingFile::FILE, 0); + Unused << NS_WARN_IF(NS_FAILED(rv)); + + return rv; +} + +// static +nsresult +LockedMaybeUpdateDirectoryPaddingFile(nsIFile* aBaseDir, + mozIStorageConnection* aConn, + const int64_t aIncreaseSize, + const int64_t aDecreaseSize, + bool* aUpdatedOut) +{ + MOZ_DIAGNOSTIC_ASSERT(aBaseDir); + MOZ_DIAGNOSTIC_ASSERT(aConn); + MOZ_DIAGNOSTIC_ASSERT(aIncreaseSize >= 0); + MOZ_DIAGNOSTIC_ASSERT(aDecreaseSize >= 0); + MOZ_DIAGNOSTIC_ASSERT(aUpdatedOut); + + // Temporary should be removed at the end of each action. If not, it means the + // failure happened. + bool temporaryFileExisted = + DirectoryPaddingFileExists(aBaseDir, DirPaddingFile::TMP_FILE); + + nsresult rv = NS_OK; + + if (aIncreaseSize == aDecreaseSize && !temporaryFileExisted) { + return rv; + } + + int64_t currentPaddingSize = 0; + rv = LockedDirectoryPaddingGet(aBaseDir, ¤tPaddingSize); + if (NS_WARN_IF(NS_FAILED(rv)) || temporaryFileExisted) { + // Fail to read padding size from the dir padding file, so try to restore. + if (rv != NS_ERROR_FILE_NOT_FOUND && + rv != NS_ERROR_FILE_TARGET_DOES_NOT_EXIST) { + // Not delete the temporary padding file here, because we're going to + // overwrite it below anyway. + rv = LockedDirectoryPaddingDeleteFile(aBaseDir, DirPaddingFile::FILE); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + } + + // XXXtt: will have a function to retore from db. + } else { + if (aIncreaseSize > 0) { + MOZ_DIAGNOSTIC_ASSERT(INT64_MAX - currentPaddingSize >= aIncreaseSize); + currentPaddingSize += aIncreaseSize; + } + + if (aDecreaseSize > 0) { + MOZ_DIAGNOSTIC_ASSERT(currentPaddingSize >= aDecreaseSize); + currentPaddingSize -= aDecreaseSize; + } + } + + MOZ_DIAGNOSTIC_ASSERT(currentPaddingSize >= 0); + + rv = LockedDirectoryPaddingTemporaryWrite(aBaseDir, currentPaddingSize); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + *aUpdatedOut = true; + + return rv; +} + +// static +nsresult +LockedDirectoryPaddingTemporaryWrite(nsIFile* aBaseDir, int64_t aPaddingSize) +{ + MOZ_DIAGNOSTIC_ASSERT(aBaseDir); + MOZ_DIAGNOSTIC_ASSERT(aPaddingSize >= 0); + + nsresult rv = LockedDirectoryPaddingWrite(aBaseDir, DirPaddingFile::TMP_FILE, + aPaddingSize); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return rv; +} + +// static +nsresult +LockedDirectoryPaddingFinalizeWrite(nsIFile* aBaseDir) +{ + MOZ_DIAGNOSTIC_ASSERT(aBaseDir); + MOZ_DIAGNOSTIC_ASSERT(DirectoryPaddingFileExists(aBaseDir, + DirPaddingFile::TMP_FILE)); + + nsCOMPtr file; + nsresult rv = aBaseDir->Clone(getter_AddRefs(file)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = file->Append(NS_LITERAL_STRING(PADDING_TMP_FILE_NAME)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = file->RenameTo(nullptr, NS_LITERAL_STRING(PADDING_FILE_NAME)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return rv; +} + +// static +nsresult +LockedDirectoryPaddingRestore(nsIFile* aBaseDir, mozIStorageConnection* aConn) +{ + MOZ_DIAGNOSTIC_ASSERT(aBaseDir); + MOZ_DIAGNOSTIC_ASSERT(aConn); + + // The content of padding file is untrusted, so remove it here. + nsresult rv = LockedDirectoryPaddingDeleteFile(aBaseDir, + DirPaddingFile::TMP_FILE); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = LockedDirectoryPaddingDeleteFile(aBaseDir, DirPaddingFile::FILE); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // XXXtt: will have a function to retore from db. + + return rv; +} + +// static +nsresult +LockedDirectoryPaddingDeleteFile(nsIFile* aBaseDir, + DirPaddingFile aPaddingFileType) +{ + MOZ_DIAGNOSTIC_ASSERT(aBaseDir); + + nsCOMPtr file; + nsresult rv = aBaseDir->Clone(getter_AddRefs(file)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + if (aPaddingFileType == DirPaddingFile::TMP_FILE) { + rv = file->Append(NS_LITERAL_STRING(PADDING_TMP_FILE_NAME)); + } else { + rv = file->Append(NS_LITERAL_STRING(PADDING_FILE_NAME)); + } + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = file->Remove( /* recursive */ false); + if (rv == NS_ERROR_FILE_NOT_FOUND || + rv == NS_ERROR_FILE_TARGET_DOES_NOT_EXIST) { + return NS_OK; + } + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return rv; +} } // namespace cache } // namespace dom } // namespace mozilla diff --git a/dom/cache/FileUtils.h b/dom/cache/FileUtils.h index 7035988239d7..4fb2fd5eb8a5 100644 --- a/dom/cache/FileUtils.h +++ b/dom/cache/FileUtils.h @@ -19,6 +19,12 @@ namespace mozilla { namespace dom { namespace cache { +enum DirPaddingFile +{ + FILE, + TMP_FILE +}; + nsresult BodyCreateDir(nsIFile* aBaseDir); @@ -77,6 +83,49 @@ RemoveNsIFile(const QuotaInfo& aQuotaInfo, nsIFile* aFile); void DecreaseUsageForQuotaInfo(const QuotaInfo& aQuotaInfo, const int64_t& aUpdatingSize); + +/** + * This function is used to check if the directory padding file is existed. + */ + +bool +DirectoryPaddingFileExists(nsIFile* aBaseDir, DirPaddingFile aPaddingFileType); + +/** + * + * The functions below are used to read/write/delete the directory padding file + * after acquiring the mutex lock. The mutex lock is held by + * CacheQuotaClient to prevent multi-thread accessing issue. To avoid deadlock, + * these functions should only access by static functions in + * dom/cache/QuotaClient.cpp. + * + */ + +nsresult +LockedDirectoryPaddingGet(nsIFile* aBaseDir, int64_t* aPaddingSizeOut); + +nsresult +LockedDirectoryPaddingInit(nsIFile* aBaseDir); + +nsresult +LockedMaybeUpdateDirectoryPaddingFile(nsIFile* aBaseDir, + mozIStorageConnection* aConn, + const int64_t aIncreaseSize, + const int64_t aDecreaseSize, + bool* aUpdatedOut); + +nsresult +LockedDirectoryPaddingTemporaryWrite(nsIFile* aBaseDir, int64_t aPaddingSize); + +nsresult +LockedDirectoryPaddingFinalizeWrite(nsIFile* aBaseDir); + +nsresult +LockedDirectoryPaddingRestore(nsIFile* aBaseDir, mozIStorageConnection* aConn); + +nsresult +LockedDirectoryPaddingDeleteFile(nsIFile* aBaseDir, + DirPaddingFile aPaddingFileType); } // namespace cache } // namespace dom } // namespace mozilla diff --git a/dom/cache/QuotaClient.cpp b/dom/cache/QuotaClient.cpp index 1dccc42e1964..301b56a4af54 100644 --- a/dom/cache/QuotaClient.cpp +++ b/dom/cache/QuotaClient.cpp @@ -226,9 +226,10 @@ public: AssertIsOnIOThread(); MOZ_DIAGNOSTIC_ASSERT(aDirectory); - // XXXtt: Will have a patch to write padding size to the file + nsresult rv = mozilla::dom::cache::InitPaddingFile(aDirectory); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } - return NS_OK; + return rv; } private: @@ -250,6 +251,7 @@ namespace mozilla { namespace dom { namespace cache { +// static already_AddRefed CreateQuotaClient() { AssertIsOnBackgroundThread(); @@ -258,6 +260,115 @@ already_AddRefed CreateQuotaClient() return ref.forget(); } + +// static +nsresult +InitPaddingFile(nsIFile* aBaseDir) +{ + MOZ_DIAGNOSTIC_ASSERT(aBaseDir); + + // XXXtt: Acquire lock here + + nsresult rv = LockedDirectoryPaddingInit(aBaseDir); + Unused << NS_WARN_IF(NS_FAILED(rv)); + + return rv; +} + +// static +template +nsresult +MaybeUpdatePaddingFile(nsIFile* aBaseDir, + mozIStorageConnection* aConn, + const int64_t aIncreaseSize, + const int64_t aDecreaseSize, + Callable aCommitHook) +{ + MOZ_DIAGNOSTIC_ASSERT(aBaseDir); + MOZ_DIAGNOSTIC_ASSERT(aConn); + MOZ_DIAGNOSTIC_ASSERT(aIncreaseSize >= 0); + MOZ_DIAGNOSTIC_ASSERT(aDecreaseSize >= 0); + + // XXXtt: Acquire lock here + + bool updated = false; + nsresult rv = + LockedMaybeUpdateDirectoryPaddingFile(aBaseDir, aConn, aIncreaseSize, + aDecreaseSize, &updated); + if (NS_WARN_IF(NS_FAILED(rv))) { + LockedDirectoryPaddingDeleteFile(aBaseDir, DirPaddingFile::TMP_FILE); + return rv; + } + + rv = aCommitHook(); + if (NS_WARN_IF(NS_FAILED(rv))) { + LockedDirectoryPaddingDeleteFile(aBaseDir, DirPaddingFile::TMP_FILE); + return rv; + } + + if (updated) { + rv = LockedDirectoryPaddingFinalizeWrite(aBaseDir); + if (NS_WARN_IF(NS_FAILED(rv))) { + // Force restore file next time. + LockedDirectoryPaddingDeleteFile(aBaseDir, DirPaddingFile::FILE); + return rv; + } + } + + return rv; +} + +// static +nsresult +RestorePaddingFile(nsIFile* aBaseDir, mozIStorageConnection* aConn) +{ + MOZ_DIAGNOSTIC_ASSERT(aBaseDir); + MOZ_DIAGNOSTIC_ASSERT(aConn); + + // XXXtt: Acquire lock here + + nsresult rv = LockedDirectoryPaddingRestore(aBaseDir, aConn); + Unused << NS_WARN_IF(NS_FAILED(rv)); + + return rv; +} + +// static +nsresult +WipePaddingFile(const QuotaInfo& aQuotaInfo, nsIFile* aBaseDir) +{ + MOZ_DIAGNOSTIC_ASSERT(aBaseDir); + + // XXXtt: Acquire lock here + + // Remove temporary file if we have one. + nsresult rv = LockedDirectoryPaddingDeleteFile(aBaseDir, + DirPaddingFile::TMP_FILE); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + MOZ_DIAGNOSTIC_ASSERT(DirectoryPaddingFileExists(aBaseDir, + DirPaddingFile::FILE)); + + int64_t paddingSize = 0; + rv = LockedDirectoryPaddingGet(aBaseDir, &paddingSize); + if (NS_WARN_IF(NS_FAILED(rv))) { + // If read file fail, there is nothing we can do to recover the file. + NS_WARNING("Cannnot read padding size from file!"); + paddingSize = 0; + } + + if (paddingSize > 0) { + DecreaseUsageForQuotaInfo(aQuotaInfo, paddingSize); + } + + rv = LockedDirectoryPaddingDeleteFile(aBaseDir, DirPaddingFile::FILE); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = LockedDirectoryPaddingInit(aBaseDir); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return rv; +} } // namespace cache } // namespace dom } // namespace mozilla diff --git a/dom/cache/QuotaClient.h b/dom/cache/QuotaClient.h index 25b9c1c2dba4..4024d79af24b 100644 --- a/dom/cache/QuotaClient.h +++ b/dom/cache/QuotaClient.h @@ -8,6 +8,7 @@ #define mozilla_dom_cache_QuotaClient_h #include "mozilla/Attributes.h" +#include "mozilla/dom/cache/Types.h" #include "mozilla/dom/quota/Client.h" namespace mozilla { @@ -17,6 +18,46 @@ namespace cache { already_AddRefed CreateQuotaClient(); +/** + * The following functions are used to access the directory padding file. The + * directory padding file lives in DOM Cache base directory + * (e.g. foo.com/cache/.padding). It is used to keep the current overall padding + * size for an origin, so that the QuotaManager doesn't need to access the + * database when getting quota clients' usage. + * + * For the directory padding file, it's only accessed on Quota IO thread + * (for getting current usage) and Cache IO threads (for tracking padding size + * change). Besides, the padding file is protected by a mutex lock held by + * CacheQuotaClient. + * + * Each padding file should only take 8 bytes (int64_t) to record the overall + * padding size. Besides, we use the temporary padding file to indicate if the + * previous action is completed successfully. If the temporary file exists, it + * represents that the previous action is failed and the content of padding file + * cannot be trusted, and we need to restore the padding file from the database. + */ + +nsresult +InitPaddingFile(nsIFile* aBaseDir); + +/** + * Note: The aCommitHook argument will be invoked while a lock is held. Callers + * should be careful not to pass a hook that might lock on something else and + * trigger a deadlock. + */ +template +nsresult +MaybeUpdatePaddingFile(nsIFile* aBaseDir, + mozIStorageConnection* aConn, + const int64_t aIncreaseSize, + const int64_t aDecreaseSize, + Callable aCommitHook); + +nsresult +RestorePaddingFile(nsIFile* aBaseDir, mozIStorageConnection* aConn); + +nsresult +WipePaddingFile(const QuotaInfo& aQuotaInfo, nsIFile* aBaseDir); } // namespace cache } // namespace dom } // namespace mozilla