Bug 668378: Telemetry sqlite IO r=asuth

This commit is contained in:
Taras Glek 2011-07-21 11:12:31 -07:00
Родитель 4dea7e2ccc
Коммит f2972192d0
7 изменённых файлов: 523 добавлений и 3 удалений

Просмотреть файл

@ -81,6 +81,7 @@ CPPSRCS = \
StorageBaseStatementInternal.cpp \
SQLCollations.cpp \
VacuumManager.cpp \
TelemetryVFS.cpp \
$(NULL)
# For nsDependentJSString

Просмотреть файл

@ -0,0 +1,452 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ :
* ***** BEGIN LICENSE BLOCK *****
* Version: MPL 1.1/GPL 2.0/LGPL 2.1
*
* The contents of this file are subject to the Mozilla Public License Version
* 1.1 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* The Original Code is Oracle Corporation code.
*
* The Initial Developer of the Original Code is
* Oracle Corporation
* Portions created by the Initial Developer are Copyright (C) 2011
* the Initial Developer. All Rights Reserved.
*
* Contributor(s):
* Taras Glek <tglek@mozilla.com>
*
* Alternatively, the contents of this file may be used under the terms of
* either the GNU General Public License Version 2 or later (the "GPL"), or
* the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
* in which case the provisions of the GPL or the LGPL are applicable instead
* of those above. If you wish to allow use of your version of this file only
* under the terms of either the GPL or the LGPL, and not to allow others to
* use your version of this file under the terms of the MPL, indicate your
* decision by deleting the provisions above and replace them with the notice
* and other provisions required by the GPL or the LGPL. If you do not delete
* the provisions above, a recipient may use your version of this file under
* the terms of any one of the MPL, the GPL or the LGPL.
*
* ***** END LICENSE BLOCK ***** */
#include "mozilla/Telemetry.h"
#include "sqlite3.h"
#include <string.h>
namespace {
using namespace mozilla;
struct Histograms {
const char *name;
Telemetry::ID readB;
Telemetry::ID writeB;
Telemetry::ID syncMs;
};
Histograms gHistograms[] = {
{"places.sqlite",
Telemetry::MOZ_SQLITE_PLACES_READ_B,
Telemetry::MOZ_SQLITE_PLACES_WRITE_B,
Telemetry::MOZ_SQLITE_PLACES_SYNC },
{"urlclassifier3.sqlite",
Telemetry::MOZ_SQLITE_URLCLASSIFIER_READ_B,
Telemetry::MOZ_SQLITE_URLCLASSIFIER_WRITE_B,
Telemetry::MOZ_SQLITE_URLCLASSIFIER_SYNC },
{"cookies.sqlite",
Telemetry::MOZ_SQLITE_COOKIES_READ_B,
Telemetry::MOZ_SQLITE_COOKIES_WRITE_B,
Telemetry::MOZ_SQLITE_COOKIES_SYNC },
{NULL,
Telemetry::MOZ_SQLITE_OTHER_READ_B,
Telemetry::MOZ_SQLITE_OTHER_WRITE_B,
Telemetry::MOZ_SQLITE_OTHER_SYNC }
};
struct telemetry_file {
sqlite3_file base; // Base class. Must be first
Histograms *histograms; // histograms pertaining to this file
sqlite3_file pReal[1]; // This contains the vfs that actually does work
};
/*
** Close a telemetry_file.
*/
int
xClose(sqlite3_file *pFile)
{
telemetry_file *p = (telemetry_file *)pFile;
int rc;
rc = p->pReal->pMethods->xClose(p->pReal);
if( rc==SQLITE_OK ){
delete p->base.pMethods;
p->base.pMethods = NULL;
}
return rc;
}
/*
** Read data from a telemetry_file.
*/
int
xRead(sqlite3_file *pFile, void *zBuf, int iAmt, sqlite_int64 iOfst)
{
telemetry_file *p = (telemetry_file *)pFile;
Telemetry::AutoTimer<Telemetry::MOZ_SQLITE_READ_MS> timer;
int rc;
rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst);
// sqlite likes to read from empty files, this is normal, ignore it.
if (rc != SQLITE_IOERR_SHORT_READ)
Telemetry::Accumulate(p->histograms->readB, rc == SQLITE_OK ? iAmt : 0);
return rc;
}
/*
** Write data to a telemetry_file.
*/
int
xWrite(sqlite3_file *pFile, const void *zBuf, int iAmt, sqlite_int64 iOfst)
{
telemetry_file *p = (telemetry_file *)pFile;
Telemetry::AutoTimer<Telemetry::MOZ_SQLITE_WRITE_MS> timer;
int rc;
rc = p->pReal->pMethods->xWrite(p->pReal, zBuf, iAmt, iOfst);
Telemetry::Accumulate(p->histograms->writeB, rc == SQLITE_OK ? iAmt : 0);
return rc;
}
/*
** Truncate a telemetry_file.
*/
int
xTruncate(sqlite3_file *pFile, sqlite_int64 size)
{
telemetry_file *p = (telemetry_file *)pFile;
int rc;
Telemetry::AutoTimer<Telemetry::MOZ_SQLITE_TRUNCATE> timer;
rc = p->pReal->pMethods->xTruncate(p->pReal, size);
return rc;
}
/*
** Sync a telemetry_file.
*/
int
xSync(sqlite3_file *pFile, int flags)
{
telemetry_file *p = (telemetry_file *)pFile;
const TimeStamp start = TimeStamp::Now();
int rc = p->pReal->pMethods->xSync(p->pReal, flags);
Telemetry::Accumulate(p->histograms->syncMs, static_cast<PRUint32>((TimeStamp::Now() - start).ToMilliseconds()));
return rc;
}
/*
** Return the current file-size of a telemetry_file.
*/
int
xFileSize(sqlite3_file *pFile, sqlite_int64 *pSize)
{
telemetry_file *p = (telemetry_file *)pFile;
int rc;
rc = p->pReal->pMethods->xFileSize(p->pReal, pSize);
return rc;
}
/*
** Lock a telemetry_file.
*/
int
xLock(sqlite3_file *pFile, int eLock)
{
telemetry_file *p = (telemetry_file *)pFile;
int rc;
rc = p->pReal->pMethods->xLock(p->pReal, eLock);
return rc;
}
/*
** Unlock a telemetry_file.
*/
int
xUnlock(sqlite3_file *pFile, int eLock)
{
telemetry_file *p = (telemetry_file *)pFile;
int rc;
rc = p->pReal->pMethods->xUnlock(p->pReal, eLock);
return rc;
}
/*
** Check if another file-handle holds a RESERVED lock on a telemetry_file.
*/
int
xCheckReservedLock(sqlite3_file *pFile, int *pResOut)
{
telemetry_file *p = (telemetry_file *)pFile;
int rc = p->pReal->pMethods->xCheckReservedLock(p->pReal, pResOut);
return rc;
}
/*
** File control method. For custom operations on a telemetry_file.
*/
int
xFileControl(sqlite3_file *pFile, int op, void *pArg)
{
telemetry_file *p = (telemetry_file *)pFile;
int rc = p->pReal->pMethods->xFileControl(p->pReal, op, pArg);
return rc;
}
/*
** Return the sector-size in bytes for a telemetry_file.
*/
int
xSectorSize(sqlite3_file *pFile)
{
telemetry_file *p = (telemetry_file *)pFile;
int rc;
rc = p->pReal->pMethods->xSectorSize(p->pReal);
return rc;
}
/*
** Return the device characteristic flags supported by a telemetry_file.
*/
int
xDeviceCharacteristics(sqlite3_file *pFile)
{
telemetry_file *p = (telemetry_file *)pFile;
int rc;
rc = p->pReal->pMethods->xDeviceCharacteristics(p->pReal);
return rc;
}
/*
** Shared-memory operations.
*/
int
xShmLock(sqlite3_file *pFile, int ofst, int n, int flags)
{
telemetry_file *p = (telemetry_file *)pFile;
return p->pReal->pMethods->xShmLock(p->pReal, ofst, n, flags);
}
int
xShmMap(sqlite3_file *pFile, int iRegion, int szRegion, int isWrite, void volatile **pp)
{
telemetry_file *p = (telemetry_file *)pFile;
int rc;
rc = p->pReal->pMethods->xShmMap(p->pReal, iRegion, szRegion, isWrite, pp);
return rc;
}
void
xShmBarrier(sqlite3_file *pFile){
telemetry_file *p = (telemetry_file *)pFile;
p->pReal->pMethods->xShmBarrier(p->pReal);
}
int
xShmUnmap(sqlite3_file *pFile, int delFlag){
telemetry_file *p = (telemetry_file *)pFile;
int rc;
rc = p->pReal->pMethods->xShmUnmap(p->pReal, delFlag);
return rc;
}
int
xOpen(sqlite3_vfs* vfs, const char *zName, sqlite3_file* pFile,
int flags, int *pOutFlags)
{
Telemetry::AutoTimer<Telemetry::MOZ_SQLITE_OPEN> timer;
sqlite3_vfs *orig_vfs = static_cast<sqlite3_vfs*>(vfs->pAppData);
int rc;
telemetry_file *p = (telemetry_file *)pFile;
Histograms *h = NULL;
// check if the filename is one we are probing for
for(size_t i = 0;i < sizeof(gHistograms)/sizeof(gHistograms[0]);i++) {
h = &gHistograms[i];
// last probe is the fallback probe
if (!h->name)
break;
if (!zName)
continue;
const char *match = strstr(zName, h->name);
if (!match)
continue;
char c = match[strlen(h->name)];
// include -wal/-journal too
if (!c || c == '-')
break;
}
p->histograms = h;
rc = orig_vfs->xOpen(orig_vfs, zName, p->pReal, flags, pOutFlags);
if( p->pReal->pMethods ){
sqlite3_io_methods *pNew = new sqlite3_io_methods;
const sqlite3_io_methods *pSub = p->pReal->pMethods;
memset(pNew, 0, sizeof(*pNew));
pNew->iVersion = pSub->iVersion;
pNew->xClose = xClose;
pNew->xRead = xRead;
pNew->xWrite = xWrite;
pNew->xTruncate = xTruncate;
pNew->xSync = xSync;
pNew->xFileSize = xFileSize;
pNew->xLock = xLock;
pNew->xUnlock = xUnlock;
pNew->xCheckReservedLock = xCheckReservedLock;
pNew->xFileControl = xFileControl;
pNew->xSectorSize = xSectorSize;
pNew->xDeviceCharacteristics = xDeviceCharacteristics;
if( pNew->iVersion>=2 ){
pNew->xShmMap = pSub->xShmMap ? xShmMap : 0;
pNew->xShmLock = pSub->xShmLock ? xShmLock : 0;
pNew->xShmBarrier = pSub->xShmBarrier ? xShmBarrier : 0;
pNew->xShmUnmap = pSub->xShmUnmap ? xShmUnmap : 0;
}
pFile->pMethods = pNew;
}
return rc;
}
int
xDelete(sqlite3_vfs* vfs, const char *zName, int syncDir)
{
sqlite3_vfs *orig_vfs = static_cast<sqlite3_vfs*>(vfs->pAppData);
return orig_vfs->xDelete(orig_vfs, zName, syncDir);
}
int
xAccess(sqlite3_vfs *vfs, const char *zName, int flags, int *pResOut)
{
sqlite3_vfs *orig_vfs = static_cast<sqlite3_vfs*>(vfs->pAppData);
return orig_vfs->xAccess(orig_vfs, zName, flags, pResOut);
}
int
xFullPathname(sqlite3_vfs *vfs, const char *zName, int nOut, char *zOut)
{
sqlite3_vfs *orig_vfs = static_cast<sqlite3_vfs*>(vfs->pAppData);
return orig_vfs->xFullPathname(orig_vfs, zName, nOut, zOut);
}
void*
xDlOpen(sqlite3_vfs *vfs, const char *zFilename)
{
sqlite3_vfs *orig_vfs = static_cast<sqlite3_vfs*>(vfs->pAppData);
return orig_vfs->xDlOpen(orig_vfs, zFilename);
}
void
xDlError(sqlite3_vfs *vfs, int nByte, char *zErrMsg)
{
sqlite3_vfs *orig_vfs = static_cast<sqlite3_vfs*>(vfs->pAppData);
orig_vfs->xDlError(orig_vfs, nByte, zErrMsg);
}
void
(*xDlSym(sqlite3_vfs *vfs, void *pHdle, const char *zSym))(void){
sqlite3_vfs *orig_vfs = static_cast<sqlite3_vfs*>(vfs->pAppData);
return orig_vfs->xDlSym(orig_vfs, pHdle, zSym);
}
void
xDlClose(sqlite3_vfs *vfs, void *pHandle)
{
sqlite3_vfs *orig_vfs = static_cast<sqlite3_vfs*>(vfs->pAppData);
orig_vfs->xDlClose(orig_vfs, pHandle);
}
int
xRandomness(sqlite3_vfs *vfs, int nByte, char *zOut)
{
sqlite3_vfs *orig_vfs = static_cast<sqlite3_vfs*>(vfs->pAppData);
return orig_vfs->xRandomness(orig_vfs, nByte, zOut);
}
int
xSleep(sqlite3_vfs *vfs, int microseconds)
{
sqlite3_vfs *orig_vfs = static_cast<sqlite3_vfs*>(vfs->pAppData);
return orig_vfs->xSleep(orig_vfs, microseconds);
}
int
xCurrentTime(sqlite3_vfs *vfs, double *prNow)
{
sqlite3_vfs *orig_vfs = static_cast<sqlite3_vfs*>(vfs->pAppData);
return orig_vfs->xCurrentTime(orig_vfs, prNow);
}
int
xGetLastError(sqlite3_vfs *vfs, int nBuf, char *zBuf)
{
sqlite3_vfs *orig_vfs = static_cast<sqlite3_vfs*>(vfs->pAppData);
return orig_vfs->xGetLastError(orig_vfs, nBuf, zBuf);
}
int
xCurrentTimeInt64(sqlite3_vfs *vfs, sqlite3_int64 *piNow)
{
sqlite3_vfs *orig_vfs = static_cast<sqlite3_vfs*>(vfs->pAppData);
return orig_vfs->xCurrentTimeInt64(orig_vfs, piNow);
}
}
namespace mozilla {
namespace storage {
sqlite3_vfs* ConstructTelemetryVFS()
{
#if defined(XP_WIN)
#define EXPECTED_VFS "win32"
#else
#define EXPECTED_VFS "unix"
#endif
sqlite3_vfs *vfs = sqlite3_vfs_find(NULL);
const bool expected_vfs = vfs->zName && !strcmp(vfs->zName, EXPECTED_VFS);
if (!expected_vfs) {
return NULL;
}
sqlite3_vfs *tvfs = new ::sqlite3_vfs;
memset(tvfs, 0, sizeof(::sqlite3_vfs));
tvfs->iVersion = 2;
NS_ASSERTION(vfs->iVersion == tvfs->iVersion, "Telemetry wrapper needs to be updated");
tvfs->szOsFile = sizeof(telemetry_file) - sizeof(sqlite3_file) + vfs->szOsFile;
tvfs->mxPathname = vfs->mxPathname;
tvfs->zName = "telemetry-vfs";
tvfs->pAppData = vfs;
tvfs->xOpen = xOpen;
tvfs->xDelete = xDelete;
tvfs->xAccess = xAccess;
tvfs->xFullPathname = xFullPathname;
tvfs->xDlOpen = xDlOpen;
tvfs->xDlError = xDlError;
tvfs->xDlSym = xDlSym;
tvfs->xDlClose = xDlClose;
tvfs->xRandomness = xRandomness;
tvfs->xSleep = xSleep;
tvfs->xCurrentTime = xCurrentTime;
tvfs->xGetLastError = xGetLastError;
tvfs->xCurrentTimeInt64 = xCurrentTimeInt64;
return tvfs;
}
}
}

Просмотреть файл

@ -278,15 +278,20 @@ Service::getSynchronousPref()
}
Service::Service()
: mMutex("Service::mMutex")
: mMutex("Service::mMutex"),
mSqliteVFS(nsnull)
{
}
Service::~Service()
{
int rc = sqlite3_vfs_unregister(mSqliteVFS);
if (rc != SQLITE_OK)
NS_WARNING("Failed to unregister sqlite vfs wrapper.");
// Shutdown the sqlite3 API. Warn if shutdown did not turn out okay, but
// there is nothing actionable we can do in that case.
int rc = ::sqlite3_quota_shutdown();
rc = ::sqlite3_quota_shutdown();
if (rc != SQLITE_OK)
NS_WARNING("sqlite3 did not shutdown cleanly.");
@ -298,6 +303,8 @@ Service::~Service()
NS_ASSERTION(shutdownObserved, "Shutdown was not observed!");
gService = nsnull;
delete mSqliteVFS;
mSqliteVFS = nsnull;
}
void
@ -306,6 +313,8 @@ Service::shutdown()
NS_IF_RELEASE(sXPConnect);
}
sqlite3_vfs* ConstructTelemetryVFS();
nsresult
Service::initialize()
{
@ -320,7 +329,15 @@ Service::initialize()
if (rc != SQLITE_OK)
return convertResultCode(rc);
rc = ::sqlite3_quota_initialize(NULL, 0);
mSqliteVFS = ConstructTelemetryVFS();
if (mSqliteVFS) {
rc = sqlite3_vfs_register(mSqliteVFS, 1);
if (rc != SQLITE_OK)
return convertResultCode(rc);
} else {
NS_WARNING("Failed to register telemetry VFS");
}
rc = ::sqlite3_quota_initialize("telemetry-vfs", 0);
if (rc != SQLITE_OK)
return convertResultCode(rc);

Просмотреть файл

@ -53,6 +53,7 @@
#include "mozIStorageServiceQuotaManagement.h"
class nsIXPConnect;
struct sqlite3_vfs;
namespace mozilla {
namespace storage {
@ -112,6 +113,8 @@ private:
* synchronizing access to mLocaleCollation.
*/
Mutex mMutex;
sqlite3_vfs *mSqliteVFS;
/**
* Shuts down the storage service, freeing all of the acquired resources.

Просмотреть файл

@ -0,0 +1,30 @@
/* Any copyright is dedicated to the Public Domain.
* http://creativecommons.org/publicdomain/zero/1.0/
*/
// Make sure that there are telemetry entries created by sqlite io
function run_sql(d, sql) {
var stmt = d.createStatement(sql)
stmt.execute()
stmt.finalize();
}
function new_file(name)
{
var file = dirSvc.get("ProfD", Ci.nsIFile);
file.append(name);
return file;
}
function run_test()
{
const Telemetry = Cc["@mozilla.org/base/telemetry;1"].getService(Ci.nsITelemetry);
let read_hgram = Telemetry.getHistogramById("MOZ_SQLITE_OTHER_READ_B");
let old_sum = read_hgram.snapshot().sum;
const file = new_file("telemetry.sqlite");
var d = getDatabase(file);
run_sql(d, "CREATE TABLE bloat(data varchar)");
run_sql(d, "DROP TABLE bloat")
do_check_true(read_hgram.snapshot().sum > old_sum)
}

Просмотреть файл

@ -29,3 +29,4 @@ tail =
[test_storage_value_array.js]
[test_unicode.js]
[test_vacuum.js]
[test_telemetry_vfs.js]

Просмотреть файл

@ -103,5 +103,21 @@ HTTP_HISTOGRAMS(SUB, "subitem: ")
#undef HTTP_HISTOGRAMS
HISTOGRAM(FIND_PLUGINS, 1, 3000, 10, EXPONENTIAL, "Time spent scanning filesystem for plugins (ms)")
HISTOGRAM(CHECK_JAVA_ENABLED, 1, 3000, 10, EXPONENTIAL, "Time spent checking if Java is enabled (ms)")
HISTOGRAM(MOZ_SQLITE_OPEN, 1, 3000, 10, EXPONENTIAL, "Time spent on sqlite open() (ms)")
HISTOGRAM(MOZ_SQLITE_READ_MS, 1, 3000, 10, EXPONENTIAL, "Time spent on sqlite read() (ms)")
HISTOGRAM(MOZ_SQLITE_OTHER_READ_B, 1, 32768, 3, LINEAR, "Sqlite read() (bytes)")
HISTOGRAM(MOZ_SQLITE_PLACES_READ_B, 1, 32768, 3, LINEAR, "Sqlite read() (bytes)")
HISTOGRAM(MOZ_SQLITE_COOKIES_READ_B, 1, 32768, 3, LINEAR, "Sqlite read() (bytes)")
HISTOGRAM(MOZ_SQLITE_URLCLASSIFIER_READ_B, 1, 32768, 3, LINEAR, "Sqlite read() (bytes)")
HISTOGRAM(MOZ_SQLITE_WRITE_MS, 1, 3000, 10, EXPONENTIAL, "Time spent on sqlite write() (ms)")
HISTOGRAM(MOZ_SQLITE_PLACES_WRITE_B, 1, 32768, 3, LINEAR, "Sqlite write (bytes)")
HISTOGRAM(MOZ_SQLITE_COOKIES_WRITE_B, 1, 32768, 3, LINEAR, "Sqlite write (bytes)")
HISTOGRAM(MOZ_SQLITE_URLCLASSIFIER_WRITE_B, 1, 32768, 3, LINEAR, "Sqlite write (bytes)")
HISTOGRAM(MOZ_SQLITE_OTHER_WRITE_B, 1, 32768, 3, LINEAR, "Sqlite write (bytes)")
HISTOGRAM(MOZ_SQLITE_TRUNCATE, 1, 3000, 10, EXPONENTIAL, "Time spent on sqlite truncate() (ms)")
HISTOGRAM(MOZ_SQLITE_PLACES_SYNC, 1, 10000, 10, EXPONENTIAL, "Time spent on sqlite fsync() (ms)")
HISTOGRAM(MOZ_SQLITE_COOKIES_SYNC, 1, 10000, 10, EXPONENTIAL, "Time spent on sqlite fsync() (ms)")
HISTOGRAM(MOZ_SQLITE_URLCLASSIFIER_SYNC, 1, 10000, 10, EXPONENTIAL, "Time spent on sqlite fsync() (ms)")
HISTOGRAM(MOZ_SQLITE_OTHER_SYNC, 1, 10000, 10, EXPONENTIAL, "Time spent on sqlite fsync() (ms)")
HISTOGRAM(STARTUP_MEASUREMENT_ERRORS, 1, 3, 4, LINEAR, "Flags errors in startup calculation()")
HISTOGRAM(NETWORK_DISK_CACHE_OPEN, 1, 10000, 10, EXPONENTIAL, "Time spent opening disk cache (ms)")