Bug 1546310 - LSNG: Parent process should store values in UTF8; r=asuth

Differential Revision: https://phabricator.services.mozilla.com/D29139
This commit is contained in:
Jan Varga 2019-04-29 06:09:42 +02:00
Родитель d3494ff018
Коммит 6360443a3b
14 изменённых файлов: 263 добавлений и 64 удалений

Просмотреть файл

@ -140,8 +140,8 @@ mozilla::ipc::IPCResult LSObserverChild::RecvObserve(
return IPC_FAIL_NO_REASON(this);
}
Storage::NotifyChange(/* aStorage */ nullptr, principal, aKey, aOldValue,
aNewValue,
Storage::NotifyChange(/* aStorage */ nullptr, principal, aKey,
aOldValue.AsString(), aNewValue.AsString(),
/* aStorageType */ kLocalStorageType, aDocumentURI,
/* aIsPrivate */ !!aPrivateBrowsingId,
/* aImmediateDispatch */ true);

Просмотреть файл

@ -93,7 +93,7 @@ typedef nsClassHashtable<nsCStringHashKey, ArchivedOriginInfo>
******************************************************************************/
// Major schema version. Bump for almost everything.
const uint32_t kMajorSchemaVersion = 2;
const uint32_t kMajorSchemaVersion = 3;
// Minor schema version. Should almost always be 0 (maybe bump on release
// branches if we have to).
@ -321,6 +321,7 @@ nsresult CreateTables(mozIStorageConnection* aConnection) {
NS_LITERAL_CSTRING("CREATE TABLE data"
"( key TEXT PRIMARY KEY"
", value TEXT NOT NULL"
", utf16Length INTEGER NOT NULL DEFAULT 0"
", compressed INTEGER NOT NULL DEFAULT 0"
", lastAccessTime INTEGER NOT NULL DEFAULT 0"
");"));
@ -362,6 +363,30 @@ nsresult UpgradeSchemaFrom1_0To2_0(mozIStorageConnection* aConnection) {
return NS_OK;
}
nsresult UpgradeSchemaFrom2_0To3_0(mozIStorageConnection* aConnection) {
AssertIsOnIOThread();
MOZ_ASSERT(aConnection);
nsresult rv = aConnection->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
"ALTER TABLE data ADD COLUMN utf16Length INTEGER NOT NULL DEFAULT 0;"));
if (NS_WARN_IF(NS_FAILED(rv))) {
return rv;
}
rv = aConnection->ExecuteSimpleSQL(
NS_LITERAL_CSTRING("UPDATE data SET utf16Length = utf16Length(value);"));
if (NS_WARN_IF(NS_FAILED(rv))) {
return rv;
}
rv = aConnection->SetSchemaVersion(MakeSchemaVersion(3, 0));
if (NS_WARN_IF(NS_FAILED(rv))) {
return rv;
}
return NS_OK;
}
nsresult SetDefaultPragmas(mozIStorageConnection* aConnection) {
MOZ_ASSERT(!NS_IsMainThread());
MOZ_ASSERT(aConnection);
@ -511,12 +536,14 @@ nsresult CreateStorageConnection(nsIFile* aDBFile, nsIFile* aUsageFile,
}
} else {
// This logic needs to change next time we change the schema!
static_assert(kSQLiteSchemaVersion == int32_t((2 << 4) + 0),
static_assert(kSQLiteSchemaVersion == int32_t((3 << 4) + 0),
"Upgrade function needed due to schema version increase.");
while (schemaVersion != kSQLiteSchemaVersion) {
if (schemaVersion == MakeSchemaVersion(1, 0)) {
rv = UpgradeSchemaFrom1_0To2_0(connection);
} else if (schemaVersion == MakeSchemaVersion(2, 0)) {
rv = UpgradeSchemaFrom2_0To3_0(connection);
} else {
LS_WARNING(
"Unable to open LocalStorage database, no upgrade path is "
@ -3763,8 +3790,9 @@ nsresult WriteOptimizer::AddItemInfo::Perform(Connection* aConnection,
Connection::CachedStatement stmt;
nsresult rv = aConnection->GetCachedStatement(
NS_LITERAL_CSTRING("INSERT OR REPLACE INTO data (key, value) "
"VALUES(:key, :value)"),
NS_LITERAL_CSTRING(
"INSERT OR REPLACE INTO data (key, value, utf16Length) "
"VALUES(:key, :value, :utf16Length)"),
&stmt);
if (NS_WARN_IF(NS_FAILED(rv))) {
return rv;
@ -3775,7 +3803,13 @@ nsresult WriteOptimizer::AddItemInfo::Perform(Connection* aConnection,
return rv;
}
rv = stmt->BindStringByName(NS_LITERAL_CSTRING("value"), mValue);
rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("value"), mValue);
if (NS_WARN_IF(NS_FAILED(rv))) {
return rv;
}
rv = stmt->BindInt32ByName(NS_LITERAL_CSTRING("utf16Length"),
mValue.UTF16Length());
if (NS_WARN_IF(NS_FAILED(rv))) {
return rv;
}
@ -3820,7 +3854,7 @@ nsresult WriteOptimizer::AddItemInfo::Perform(Connection* aConnection,
return rv;
}
rv = stmt->BindStringByName(NS_LITERAL_CSTRING("value"), mValue);
rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("value"), mValue);
if (NS_WARN_IF(NS_FAILED(rv))) {
return rv;
}
@ -4894,34 +4928,37 @@ void Datastore::SetItem(Database* aDatabase, const nsString& aDocumentURI,
mValues.Put(aKey, aValue);
int64_t sizeOfItem;
int64_t delta;
if (isNewItem) {
mWriteOptimizer.AddItem(aKey, aValue);
int64_t sizeOfKey = static_cast<int64_t>(aKey.Length());
sizeOfItem = sizeOfKey + static_cast<int64_t>(aValue.Length());
mUpdateBatchUsage += sizeOfItem;
delta = sizeOfKey + static_cast<int64_t>(aValue.UTF16Length());
mUpdateBatchUsage += delta;
mSizeOfKeys += sizeOfKey;
mSizeOfItems += sizeOfItem;
mSizeOfItems += sizeOfKey + static_cast<int64_t>(aValue.Length());
;
} else {
mWriteOptimizer.UpdateItem(aKey, aValue);
sizeOfItem = static_cast<int64_t>(aValue.Length()) -
static_cast<int64_t>(oldValue.Length());
delta = static_cast<int64_t>(aValue.UTF16Length()) -
static_cast<int64_t>(oldValue.UTF16Length());
mUpdateBatchUsage += sizeOfItem;
mUpdateBatchUsage += delta;
mSizeOfItems += sizeOfItem;
mSizeOfItems += static_cast<int64_t>(aValue.Length()) -
static_cast<int64_t>(oldValue.Length());
}
if (IsPersistent()) {
if (oldValue.IsVoid()) {
mConnection->AddItem(aKey, aValue, sizeOfItem);
mConnection->AddItem(aKey, aValue, delta);
} else {
mConnection->UpdateItem(aKey, aValue, sizeOfItem);
mConnection->UpdateItem(aKey, aValue, delta);
}
}
}
@ -4947,15 +4984,16 @@ void Datastore::RemoveItem(Database* aDatabase, const nsString& aDocumentURI,
mWriteOptimizer.RemoveItem(aKey);
int64_t sizeOfKey = static_cast<int64_t>(aKey.Length());
int64_t sizeOfItem = sizeOfKey + static_cast<int64_t>(oldValue.Length());
mUpdateBatchUsage -= sizeOfItem;
int64_t delta = -sizeOfKey - static_cast<int64_t>(oldValue.UTF16Length());
mUpdateBatchUsage += delta;
mSizeOfKeys -= sizeOfKey;
mSizeOfItems -= sizeOfItem;
mSizeOfItems -= sizeOfKey + static_cast<int64_t>(oldValue.Length());
if (IsPersistent()) {
mConnection->RemoveItem(aKey, -sizeOfItem);
mConnection->RemoveItem(aKey, delta);
}
}
@ -4969,13 +5007,13 @@ void Datastore::Clear(Database* aDatabase, const nsString& aDocumentURI) {
MOZ_ASSERT(mInUpdateBatch);
if (mValues.Count()) {
int64_t sizeOfItems = 0;
int64_t delta = 0;
for (auto iter = mValues.ConstIter(); !iter.Done(); iter.Next()) {
const nsAString& key = iter.Key();
const LSValue& value = iter.Data();
sizeOfItems += (static_cast<int64_t>(key.Length()) +
static_cast<int64_t>(value.Length()));
delta += -static_cast<int64_t>(key.Length()) -
static_cast<int64_t>(value.UTF16Length());
NotifySnapshots(aDatabase, key, value, /* aAffectsOrder */ true);
}
@ -4984,13 +5022,13 @@ void Datastore::Clear(Database* aDatabase, const nsString& aDocumentURI) {
mWriteOptimizer.Clear();
mUpdateBatchUsage -= sizeOfItems;
mUpdateBatchUsage += delta;
mSizeOfKeys = 0;
mSizeOfItems = 0;
if (IsPersistent()) {
mConnection->Clear(-sizeOfItems);
mConnection->Clear(delta);
}
}
@ -6662,8 +6700,8 @@ nsresult PrepareDatastoreOp::DatabaseWork() {
nsCOMPtr<mozIStorageStatement> stmt;
rv = connection->CreateStatement(
NS_LITERAL_CSTRING("INSERT INTO data (key, value) "
"SELECT key, value "
NS_LITERAL_CSTRING("INSERT INTO data (key, value, utf16Length) "
"SELECT key, value, utf16Length(value) "
"FROM webappsstore2 "
"WHERE originKey = :originKey "
"AND originAttributes = :originAttributes;"
@ -7284,10 +7322,10 @@ nsresult PrepareDatastoreOp::LoadDataOp::DoDatastoreWork() {
}
Connection::CachedStatement stmt;
nsresult rv =
mConnection->GetCachedStatement(NS_LITERAL_CSTRING("SELECT key, value "
"FROM data;"),
&stmt);
nsresult rv = mConnection->GetCachedStatement(
NS_LITERAL_CSTRING("SELECT key, value, utf16Length "
"FROM data;"),
&stmt);
if (NS_WARN_IF(NS_FAILED(rv))) {
return rv;
}
@ -7300,13 +7338,19 @@ nsresult PrepareDatastoreOp::LoadDataOp::DoDatastoreWork() {
return rv;
}
nsString buffer;
rv = stmt->GetString(1, buffer);
nsCString buffer;
rv = stmt->GetUTF8String(1, buffer);
if (NS_WARN_IF(NS_FAILED(rv))) {
return rv;
}
LSValue value(buffer);
int32_t utf16Length;
rv = stmt->GetInt32(2, &utf16Length);
if (NS_WARN_IF(NS_FAILED(rv))) {
return rv;
}
LSValue value(buffer, utf16Length);
mPrepareDatastoreOp->mValues.Put(key, value);
auto item = mPrepareDatastoreOp->mOrderedItems.AppendElement();
@ -7315,7 +7359,7 @@ nsresult PrepareDatastoreOp::LoadDataOp::DoDatastoreWork() {
mPrepareDatastoreOp->mSizeOfKeys += key.Length();
mPrepareDatastoreOp->mSizeOfItems += key.Length() + value.Length();
#ifdef DEBUG
mPrepareDatastoreOp->mDEBUGUsage += key.Length() + value.Length();
mPrepareDatastoreOp->mDEBUGUsage += key.Length() + value.UTF16Length();
#endif
}
if (NS_WARN_IF(NS_FAILED(rv))) {

Просмотреть файл

@ -82,7 +82,7 @@ nsresult LSSnapshot::Init(const nsAString& aKey,
mLoadedItems.PutEntry(itemInfo.key());
}
mValues.Put(itemInfo.key(), value);
mValues.Put(itemInfo.key(), value.AsString());
}
if (loadState == LoadState::Partial) {
@ -443,7 +443,7 @@ nsresult LSSnapshot::GetItemInternal(const nsAString& aKey,
return NS_ERROR_FAILURE;
}
result = value;
result = value.AsString();
if (result.IsVoid()) {
mUnknownItems.PutEntry(aKey);
@ -458,7 +458,7 @@ nsresult LSSnapshot::GetItemInternal(const nsAString& aKey,
const LSItemInfo& itemInfo = itemInfos[i];
mLoadedItems.PutEntry(itemInfo.key());
mValues.Put(itemInfo.key(), itemInfo.value());
mValues.Put(itemInfo.key(), itemInfo.value().AsString());
}
if (mLoadedItems.Count() == mInitLength) {
@ -491,7 +491,7 @@ nsresult LSSnapshot::GetItemInternal(const nsAString& aKey,
return NS_ERROR_FAILURE;
}
result = value;
result = value.AsString();
MOZ_ASSERT(!result.IsVoid());
@ -504,7 +504,7 @@ nsresult LSSnapshot::GetItemInternal(const nsAString& aKey,
const LSItemInfo& itemInfo = itemInfos[i];
mLoadedItems.PutEntry(itemInfo.key());
mValues.Put(itemInfo.key(), itemInfo.value());
mValues.Put(itemInfo.key(), itemInfo.value().AsString());
}
if (mLoadedItems.Count() == mInitLength) {

Просмотреть файл

@ -10,7 +10,7 @@ namespace mozilla {
namespace dom {
const LSValue& VoidLSValue() {
static const LSValue sVoidLSValue(VoidString());
static const LSValue sVoidLSValue(VoidCString(), 0);
return sVoidLSValue;
}

Просмотреть файл

@ -14,35 +14,86 @@ namespace dom {
* Represents a LocalStorage value. From content's perspective, values (if
* present) are always DOMStrings. This is also true from a quota-tracking
* perspective. However, for memory and disk efficiency it's preferable to store
* the value in alternate representations. The LSValue type exists to support
* these alternate representations in future.
* the value in alternate utf-8 encoding representations. The LSValue type
* exists to support these alternate representations, dynamically re-encoding to
* utf-16 while still tracking value size on a utf-16 basis for quota purposes.
*/
class LSValue final {
friend struct IPC::ParamTraits<LSValue>;
nsString mBuffer;
nsCString mBuffer;
uint32_t mUTF16Length;
public:
LSValue() {}
LSValue() : mUTF16Length(0) {}
explicit LSValue(const nsAString& aBuffer) : mBuffer(aBuffer) {}
explicit LSValue(const nsACString& aBuffer, uint32_t aUTF16Length)
: mBuffer(aBuffer), mUTF16Length(aUTF16Length) {}
explicit LSValue(const nsAString& aBuffer) : mUTF16Length(aBuffer.Length()) {
if (aBuffer.IsVoid()) {
mBuffer.SetIsVoid(true);
} else {
CopyUTF16toUTF8(aBuffer, mBuffer);
}
}
bool IsVoid() const { return mBuffer.IsVoid(); }
void SetIsVoid(bool aVal) { mBuffer.SetIsVoid(aVal); }
/**
* This represents the "physical" length that the parent process uses for
* the size of value/item computation. This can also be used to see how much
* memory the value is using at rest or what the cost is for sending the value
* over IPC.
*/
uint32_t Length() const { return mBuffer.Length(); }
/*
* This represents the "logical" length that content sees and that is also
* used for quota management purposes.
*/
uint32_t UTF16Length() const { return mUTF16Length; }
bool Equals(const LSValue& aOther) const {
return mBuffer == aOther.mBuffer &&
mBuffer.IsVoid() == aOther.mBuffer.IsVoid();
mBuffer.IsVoid() == aOther.mBuffer.IsVoid() &&
mUTF16Length == aOther.mUTF16Length;
}
bool operator==(const LSValue& aOther) const { return Equals(aOther); }
bool operator!=(const LSValue& aOther) const { return !Equals(aOther); }
operator const nsString&() const { return mBuffer; }
operator const nsCString&() const { return mBuffer; }
operator Span<const char>() const { return mBuffer; }
class Converter {
nsString mBuffer;
public:
explicit Converter(const LSValue& aValue) {
if (aValue.mBuffer.IsVoid()) {
mBuffer.SetIsVoid(true);
} else {
CopyUTF8toUTF16(aValue.mBuffer, mBuffer);
}
}
Converter(Converter&& aOther) : mBuffer(aOther.mBuffer) {}
~Converter() {}
operator const nsString&() const { return mBuffer; }
private:
Converter() = delete;
Converter(const Converter&) = delete;
Converter& operator=(const Converter&) = delete;
Converter& operator=(const Converter&&) = delete;
};
Converter AsString() const { return Converter(const_cast<LSValue&>(*this)); }
};
const LSValue& VoidLSValue();

Просмотреть файл

@ -27,15 +27,18 @@ struct ParamTraits<mozilla::dom::LSValue> {
static void Write(Message* aMsg, const paramType& aParam) {
WriteParam(aMsg, aParam.mBuffer);
WriteParam(aMsg, aParam.mUTF16Length);
}
static bool Read(const Message* aMsg, PickleIterator* aIter,
paramType* aResult) {
return ReadParam(aMsg, aIter, &aResult->mBuffer);
return ReadParam(aMsg, aIter, &aResult->mBuffer) &&
ReadParam(aMsg, aIter, &aResult->mUTF16Length);
}
static void Log(const paramType& aParam, std::wstring* aLog) {
LogParam(aParam.mBuffer, aLog);
LogParam(aParam.mUTF16Length, aLog);
}
};

Двоичные данные
dom/localstorage/test/unit/schema3upgrade_profile.zip Normal file

Двоичный файл не отображается.

Просмотреть файл

@ -0,0 +1,39 @@
/**
* Any copyright is dedicated to the Public Domain.
* http://creativecommons.org/publicdomain/zero/1.0/
*/
async function testSteps() {
const url = "http://example.com";
info("Setting pref");
Services.prefs.setBoolPref("dom.storage.next_gen", true);
info("Clearing");
let request = clear();
await requestFinished(request);
info("Installing package");
// The profile contains one initialized origin directory with local
// storage data, local storage archive, a script for origin initialization,
// the storage database and the web apps store database:
// - storage/default/http+++example.com
// - storage/ls-archive.sqlite
// - create_db.js
// - storage.sqlite
// - webappsstore.sqlite
// The file create_db.js in the package was run locally (with a build with
// local storage archive version 1 and database schema version 2),
// specifically it was temporarily added to xpcshell.ini and then executed:
// mach xpcshell-test --interactive dom/localstorage/test/unit/create_db.js
// Note: to make it become the profile in the test, additional manual steps
// are needed.
// 1. Remove the folder "storage/temporary".
installPackage("schema3upgrade_profile");
let storage = getLocalStorage(getPrincipal(url));
storage.open();
}

Просмотреть файл

@ -9,6 +9,7 @@ support-files =
corruptedDatabase_profile.zip
groupMismatch_profile.zip
migration_profile.zip
schema3upgrade_profile.zip
stringLength2_profile.zip
stringLength_profile.zip
@ -35,6 +36,7 @@ run-sequentially = this test depends on a file produced by test_databaseShadowin
[test_groupMismatch.js]
[test_migration.js]
[test_originInit.js]
[test_schema3upgrade.js]
[test_snapshotting.js]
requesttimeoutfactor = 4
[test_stringLength.js]

Просмотреть файл

@ -218,7 +218,7 @@ const char kResourceOriginPrefix[] = "resource://";
#define LS_ARCHIVE_FILE_NAME "ls-archive.sqlite"
#define LS_ARCHIVE_TMP_FILE_NAME "ls-archive-tmp.sqlite"
const uint32_t kLocalStorageArchiveVersion = 1;
const uint32_t kLocalStorageArchiveVersion = 2;
const char kProfileDoChangeTopic[] = "profile-do-change";
@ -446,7 +446,6 @@ nsresult LoadLocalStorageArchiveVersion(mozIStorageConnection* aConnection,
return NS_OK;
}
/*
nsresult SaveLocalStorageArchiveVersion(mozIStorageConnection* aConnection,
uint32_t aVersion) {
AssertIsOnIOThread();
@ -472,7 +471,6 @@ nsresult SaveLocalStorageArchiveVersion(mozIStorageConnection* aConnection,
return NS_OK;
}
*/
/******************************************************************************
* Quota manager class declarations
@ -5261,7 +5259,6 @@ nsresult QuotaManager::UpgradeLocalStorageArchiveFrom0To1(
return NS_OK;
}
/*
nsresult QuotaManager::UpgradeLocalStorageArchiveFrom1To2(
nsCOMPtr<mozIStorageConnection>& aConnection) {
nsresult rv = SaveLocalStorageArchiveVersion(aConnection, 2);
@ -5271,7 +5268,6 @@ nsresult QuotaManager::UpgradeLocalStorageArchiveFrom1To2(
return NS_OK;
}
*/
#ifdef DEBUG
@ -5495,17 +5491,16 @@ nsresult QuotaManager::EnsureStorageIsInitialized() {
return rv;
}
} else {
static_assert(kLocalStorageArchiveVersion == 1,
static_assert(kLocalStorageArchiveVersion == 2,
"Upgrade function needed due to LocalStorage archive "
"version increase.");
while (version != kLocalStorageArchiveVersion) {
if (version == 0) {
rv = UpgradeLocalStorageArchiveFrom0To1(connection);
} /* else if (version == 1) {
} else if (version == 1) {
rv = UpgradeLocalStorageArchiveFrom1To2(connection);
} */
else {
} else {
QM_WARNING(
"Unable to initialize LocalStorage archive, no upgrade path is "
"available!");

Просмотреть файл

@ -440,10 +440,8 @@ class QuotaManager final : public BackgroundThreadObject {
nsresult UpgradeLocalStorageArchiveFrom0To1(
nsCOMPtr<mozIStorageConnection>& aConnection);
/*
nsresult UpgradeLocalStorageArchiveFrom1To2(
nsCOMPtr<mozIStorageConnection>& aConnection);
*/
nsresult UpgradeLocalStorageArchiveFrom1To2(
nsCOMPtr<mozIStorageConnection>& aConnection);
nsresult InitializeRepository(PersistenceType aPersistenceType);

Двоичные данные
dom/quota/test/unit/localStorageArchive2upgrade_profile.zip Normal file

Двоичный файл не отображается.

Просмотреть файл

@ -0,0 +1,65 @@
/**
* Any copyright is dedicated to the Public Domain.
* http://creativecommons.org/publicdomain/zero/1.0/
*/
/**
* This test is mainly to verify that local storage directories are not removed
* during local storage archive upgrade from version 1 to version 2.
* See bug 1546310.
*/
async function testSteps() {
const lsDirs = [
"storage/default/http+++example.com/ls",
"storage/default/http+++localhost/ls",
"storage/default/http+++www.mozilla.org/ls",
];
info("Clearing");
let request = clear();
await requestFinished(request);
info("Installing package");
// The profile contains three initialized origin directories with local
// storage data, local storage archive, a script for origin initialization,
// the storage database and the web apps store database:
// - storage/default/https+++example.com
// - storage/default/https+++localhost
// - storage/default/https+++www.mozilla.org
// - storage/ls-archive.sqlite
// - create_db.js
// - storage.sqlite
// - webappsstore.sqlite
// The file create_db.js in the package was run locally (with a build with
// local storage archive version 1), specifically it was temporarily added to
// xpcshell.ini and then executed:
// mach xpcshell-test --interactive dom/localstorage/test/unit/create_db.js
// Note: to make it become the profile in the test, additional manual steps
// are needed.
// 1. Remove the folder "storage/temporary".
installPackage("localStorageArchive2upgrade_profile");
info("Checking ls dirs");
for (let lsDir of lsDirs) {
let dir = getRelativeFile(lsDir);
exists = dir.exists();
ok(exists, "ls directory does exist");
}
request = init();
request = await requestFinished(request);
info("Checking ls dirs");
for (let lsDir of lsDirs) {
let dir = getRelativeFile(lsDir);
exists = dir.exists();
ok(exists, "ls directory does exist");
}
}

Просмотреть файл

@ -14,6 +14,7 @@ support-files =
idbSubdirUpgrade1_profile.zip
idbSubdirUpgrade2_profile.zip
localStorageArchive1upgrade_profile.zip
localStorageArchive2upgrade_profile.zip
localStorageArchiveDowngrade_profile.zip
morgueCleanup_profile.zip
obsoleteOriginAttributes_profile.zip
@ -36,6 +37,7 @@ support-files =
[test_initTemporaryStorage.js]
[test_listInitializedOrigins.js]
[test_localStorageArchive1upgrade.js]
[test_localStorageArchive2upgrade.js]
[test_localStorageArchiveDowngrade.js]
[test_morgueCleanup.js]
[test_obsoleteOriginAttributesUpgrade.js]