зеркало из https://github.com/mozilla/gecko-dev.git
Bug 385066 - "Remove preloading from mozStorage" [p=comrade693+bmo@gmail.com (Shawn Wilsher [sdwilsh]) r=gavin a=blocking1.9+]
This commit is contained in:
reed@reedloden.com
Родитель
69f13796a0
Коммит
d342defa35
|
|
@ -18,13 +18,6 @@ Simply copy the sqlite3.h and sqlite3.c files from the amalgamation of sqlite.
|
|||
Then you need to update sqlite3file.h, which pulls out random bits of the
|
||||
internal files that we need to export. If any of these internal structures
|
||||
change, they need to be changed in sqlite3file.h as well. This may involve
|
||||
downloading the whole souce (not the amalgamation) to check.
|
||||
downloading the whole source (not the amalgamation) to check.
|
||||
|
||||
-- Shawn Wilsher <me@shawnwilsher.com> 06/2007
|
||||
|
||||
XXX: This is temporarily obsolete (See Bug 385066)
|
||||
You also need to apply preload-cache.patch. This patch provides the preload
|
||||
functionality used by mozIStorageConnection.preload to fill the cache with
|
||||
pages from the disk.
|
||||
|
||||
-- Brett Wilson <brettw@gmail.com> 04/2006
|
||||
|
|
|
|||
|
|
@ -1,307 +0,0 @@
|
|||
Index: build.c
|
||||
===================================================================
|
||||
RCS file: /cvsroot/mozilla/db/sqlite3/src/build.c,v
|
||||
retrieving revision 1.4.8.3
|
||||
diff -u -u -8 -p -r1.4.8.3 build.c
|
||||
--- build.c 22 Feb 2006 20:56:28 -0000 1.4.8.3
|
||||
+++ build.c 11 Apr 2006 17:28:51 -0000
|
||||
@@ -22,16 +22,19 @@
|
||||
** COMMIT
|
||||
** ROLLBACK
|
||||
**
|
||||
** $Id: preload-cache.patch,v 1.2 2006/04/11 17:35:25 brettw%gmail.com Exp $
|
||||
*/
|
||||
#include "sqliteInt.h"
|
||||
#include <ctype.h>
|
||||
|
||||
+#include "pager.h"
|
||||
+#include "btree.h"
|
||||
+
|
||||
/*
|
||||
** This routine is called when a new SQL statement is beginning to
|
||||
** be parsed. Initialize the pParse structure as needed.
|
||||
*/
|
||||
void sqlite3BeginParse(Parse *pParse, int explainFlag){
|
||||
pParse->explain = explainFlag;
|
||||
pParse->nVar = 0;
|
||||
}
|
||||
@@ -3214,8 +3217,34 @@ KeyInfo *sqlite3IndexKeyinfo(Parse *pPar
|
||||
}
|
||||
|
||||
if( pParse->nErr ){
|
||||
sqliteFree(pKey);
|
||||
pKey = 0;
|
||||
}
|
||||
return pKey;
|
||||
}
|
||||
+
|
||||
+
|
||||
+/* See declaration in sqlite3.h for information */
|
||||
+int sqlite3Preload(sqlite3* db)
|
||||
+{
|
||||
+ Pager* pPager;
|
||||
+ Btree* pBt;
|
||||
+ int rc;
|
||||
+ int i;
|
||||
+ int dbsLoaded = 0;
|
||||
+
|
||||
+ for (i = 0; i < db->nDb; i ++) {
|
||||
+ pBt = db->aDb[i].pBt;
|
||||
+ if (! pBt)
|
||||
+ continue;
|
||||
+ pPager = sqlite3BtreePager(pBt);
|
||||
+ if (pPager) {
|
||||
+ rc = sqlite3pager_loadall(pPager);
|
||||
+ if (rc == SQLITE_OK)
|
||||
+ dbsLoaded ++;
|
||||
+ }
|
||||
+ }
|
||||
+ if (dbsLoaded == 0)
|
||||
+ return SQLITE_ERROR;
|
||||
+ return SQLITE_OK;
|
||||
+}
|
||||
Index: pager.c
|
||||
===================================================================
|
||||
RCS file: /cvsroot/mozilla/db/sqlite3/src/pager.c,v
|
||||
retrieving revision 1.3.8.3
|
||||
diff -u -u -8 -p -r1.3.8.3 pager.c
|
||||
--- pager.c 22 Feb 2006 20:56:28 -0000 1.3.8.3
|
||||
+++ pager.c 11 Apr 2006 17:28:51 -0000
|
||||
@@ -2521,16 +2521,30 @@ int sqlite3pager_release_memory(int nReq
|
||||
** to find a page in the in-memory cache first. If the page is not already
|
||||
** in memory, this routine goes to disk to read it in whereas _lookup()
|
||||
** just returns 0. This routine acquires a read-lock the first time it
|
||||
** has to go to disk, and could also playback an old journal if necessary.
|
||||
** Since _lookup() never goes to disk, it never has to deal with locks
|
||||
** or journal files.
|
||||
*/
|
||||
int sqlite3pager_get(Pager *pPager, Pgno pgno, void **ppPage){
|
||||
+ /* This just passes through to our modified version with a NULL data pointer */
|
||||
+ return sqlite3pager_get2(pPager, pgno, ppPage, 0);
|
||||
+}
|
||||
+
|
||||
+
|
||||
+/*
|
||||
+** This is an internal version of pager_get that takes an extra parameter of
|
||||
+** data to use to fill the page with. This allows more efficient filling for
|
||||
+** preloaded data. If this extra parameter is NULL, we'll go to the file.
|
||||
+**
|
||||
+** See sqlite3pager_loadall which uses this function.
|
||||
+*/
|
||||
+int sqlite3pager_get2(Pager *pPager, Pgno pgno, void **ppPage,
|
||||
+ unsigned char* pDataToFill) {
|
||||
PgHdr *pPg;
|
||||
int rc;
|
||||
|
||||
/* The maximum page number is 2^31. Return SQLITE_CORRUPT if a page
|
||||
** number greater than this, or zero, is requested.
|
||||
*/
|
||||
if( pgno>PAGER_MAX_PGNO || pgno==0 || pgno==PAGER_MJ_PGNO(pPager) ){
|
||||
return SQLITE_CORRUPT_BKPT;
|
||||
@@ -2676,39 +2690,46 @@ int sqlite3pager_get(Pager *pPager, Pgno
|
||||
}
|
||||
|
||||
/* Populate the page with data, either by reading from the database
|
||||
** file, or by setting the entire page to zero.
|
||||
*/
|
||||
if( sqlite3pager_pagecount(pPager)<(int)pgno || MEMDB ){
|
||||
memset(PGHDR_TO_DATA(pPg), 0, pPager->pageSize);
|
||||
}else{
|
||||
- assert( MEMDB==0 );
|
||||
- rc = sqlite3OsSeek(pPager->fd, (pgno-1)*(i64)pPager->pageSize);
|
||||
- if( rc==SQLITE_OK ){
|
||||
- rc = sqlite3OsRead(pPager->fd, PGHDR_TO_DATA(pPg),
|
||||
- pPager->pageSize);
|
||||
- }
|
||||
- TRACE3("FETCH %d page %d\n", PAGERID(pPager), pPg->pgno);
|
||||
- CODEC(pPager, PGHDR_TO_DATA(pPg), pPg->pgno, 3);
|
||||
- if( rc!=SQLITE_OK ){
|
||||
- i64 fileSize;
|
||||
- int rc2 = sqlite3OsFileSize(pPager->fd, &fileSize);
|
||||
- if( rc2!=SQLITE_OK || fileSize>=pgno*pPager->pageSize ){
|
||||
- /* An IO error occured in one of the the sqlite3OsSeek() or
|
||||
- ** sqlite3OsRead() calls above. */
|
||||
- pPg->pgno = 0;
|
||||
- sqlite3pager_unref(PGHDR_TO_DATA(pPg));
|
||||
- return rc;
|
||||
+ if (pDataToFill) {
|
||||
+ /* Just copy from the given memory */
|
||||
+ memcpy(PGHDR_TO_DATA(pPg), pDataToFill, pPager->pageSize);
|
||||
+ CODEC(pPager, PGHDR_TO_DATA(pPg), pPg->pgno, 3);
|
||||
+ } else {
|
||||
+ /* Load from disk */
|
||||
+ assert( MEMDB==0 );
|
||||
+ rc = sqlite3OsSeek(pPager->fd, (pgno-1)*(i64)pPager->pageSize);
|
||||
+ if( rc==SQLITE_OK ){
|
||||
+ rc = sqlite3OsRead(pPager->fd, PGHDR_TO_DATA(pPg),
|
||||
+ pPager->pageSize);
|
||||
+ }
|
||||
+ TRACE3("FETCH %d page %d\n", PAGERID(pPager), pPg->pgno);
|
||||
+ CODEC(pPager, PGHDR_TO_DATA(pPg), pPg->pgno, 3);
|
||||
+ if( rc!=SQLITE_OK ){
|
||||
+ i64 fileSize;
|
||||
+ int rc2 = sqlite3OsFileSize(pPager->fd, &fileSize);
|
||||
+ if( rc2!=SQLITE_OK || fileSize>=pgno*pPager->pageSize ){
|
||||
+ /* An IO error occured in one of the the sqlite3OsSeek() or
|
||||
+ ** sqlite3OsRead() calls above. */
|
||||
+ pPg->pgno = 0;
|
||||
+ sqlite3pager_unref(PGHDR_TO_DATA(pPg));
|
||||
+ return rc;
|
||||
+ }else{
|
||||
+ clear_simulated_io_error();
|
||||
+ memset(PGHDR_TO_DATA(pPg), 0, pPager->pageSize);
|
||||
+ }
|
||||
}else{
|
||||
- clear_simulated_io_error();
|
||||
- memset(PGHDR_TO_DATA(pPg), 0, pPager->pageSize);
|
||||
+ TEST_INCR(pPager->nRead);
|
||||
}
|
||||
- }else{
|
||||
- TEST_INCR(pPager->nRead);
|
||||
}
|
||||
}
|
||||
|
||||
/* Link the page into the page hash table */
|
||||
h = pager_hash(pgno);
|
||||
pPg->pNextHash = pPager->aHash[h];
|
||||
pPager->aHash[h] = pPg;
|
||||
if( pPg->pNextHash ){
|
||||
@@ -3773,16 +3794,80 @@ int sqlite3pager_movepage(Pager *pPager,
|
||||
DATA_TO_PGHDR(pNeedSync)->dirty = 1;
|
||||
sqlite3pager_unref(pNeedSync);
|
||||
}
|
||||
|
||||
return SQLITE_OK;
|
||||
}
|
||||
#endif
|
||||
|
||||
+
|
||||
+/**
|
||||
+ * Addition for Mozilla: This will attempt to populate the database cache with
|
||||
+ * the first N bytes of the file, where N is the total size of the cache.
|
||||
+ * Because we can load this as one chunk from the disk, this is much faster
|
||||
+ * than loading a subset of the pages one at a time in random order.
|
||||
+ *
|
||||
+ * The pager must be initialized before this function is called. This means a
|
||||
+ * statement must be open that has initialized the pager and is keeping the
|
||||
+ * cache in memory.
|
||||
+ */
|
||||
+int sqlite3pager_loadall(Pager* pPager)
|
||||
+{
|
||||
+ int i;
|
||||
+ int rc;
|
||||
+ int loadSize;
|
||||
+ int loadPages;
|
||||
+ unsigned char* fileData;
|
||||
+
|
||||
+ if (pPager->dbSize < 0 || pPager->pageSize < 0) {
|
||||
+ /* pager not initialized, this means a statement is not open */
|
||||
+ return SQLITE_MISUSE;
|
||||
+ }
|
||||
+
|
||||
+ /* compute sizes */
|
||||
+ if (pPager->mxPage < pPager->dbSize)
|
||||
+ loadPages = pPager->mxPage;
|
||||
+ else
|
||||
+ loadPages = pPager->dbSize;
|
||||
+ loadSize = loadPages * pPager->pageSize;
|
||||
+
|
||||
+ rc = sqlite3OsSeek(pPager->fd, 0);
|
||||
+ if (rc != SQLITE_OK)
|
||||
+ return rc;
|
||||
+
|
||||
+ /* load the file as one chunk */
|
||||
+ fileData = sqliteMallocRaw(loadSize);
|
||||
+ if (! fileData)
|
||||
+ return SQLITE_NOMEM;
|
||||
+ rc = sqlite3OsRead(pPager->fd, fileData, loadSize);
|
||||
+ if (rc != SQLITE_OK) {
|
||||
+ sqliteFree(fileData);
|
||||
+ return rc;
|
||||
+ }
|
||||
+
|
||||
+ /* Copy the data to each page. Note that the page numbers we pass to _get
|
||||
+ * are one-based, 0 is a marker for no page. We also need to check that we
|
||||
+ * haven't loaded more pages than the cache can hold total. There may have
|
||||
+ * already been a few pages loaded before, so we may fill the cache before
|
||||
+ * loading all of the pages we want to.
|
||||
+ */
|
||||
+ for (i = 1; i <= loadPages && pPager->nPage < pPager->mxPage; i ++) {
|
||||
+ void *pPage;
|
||||
+ rc = sqlite3pager_get2(pPager, 1, &pPage,
|
||||
+ &fileData[(i-1)*(i64)pPager->pageSize]);
|
||||
+ if (rc != SQLITE_OK)
|
||||
+ break;
|
||||
+ sqlite3pager_unref(pPage);
|
||||
+ }
|
||||
+ sqliteFree(fileData);
|
||||
+ return SQLITE_OK;
|
||||
+}
|
||||
+
|
||||
+
|
||||
#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
|
||||
/*
|
||||
** Return the current state of the file lock for the given pager.
|
||||
** The return value is one of NO_LOCK, SHARED_LOCK, RESERVED_LOCK,
|
||||
** PENDING_LOCK, or EXCLUSIVE_LOCK.
|
||||
*/
|
||||
int sqlite3pager_lockstate(Pager *pPager){
|
||||
return sqlite3OsLockState(pPager->fd);
|
||||
Index: pager.h
|
||||
===================================================================
|
||||
RCS file: /cvsroot/mozilla/db/sqlite3/src/pager.h,v
|
||||
retrieving revision 1.2.8.3
|
||||
diff -u -u -8 -p -r1.2.8.3 pager.h
|
||||
--- pager.h 22 Feb 2006 20:56:28 -0000 1.2.8.3
|
||||
+++ pager.h 11 Apr 2006 17:28:51 -0000
|
||||
@@ -104,16 +104,17 @@ const char *sqlite3pager_filename(Pager*
|
||||
const char *sqlite3pager_dirname(Pager*);
|
||||
const char *sqlite3pager_journalname(Pager*);
|
||||
int sqlite3pager_nosync(Pager*);
|
||||
int sqlite3pager_rename(Pager*, const char *zNewName);
|
||||
void sqlite3pager_set_codec(Pager*,void(*)(void*,void*,Pgno,int),void*);
|
||||
int sqlite3pager_movepage(Pager*,void*,Pgno);
|
||||
int sqlite3pager_reset(Pager*);
|
||||
int sqlite3pager_release_memory(int);
|
||||
+int sqlite3pager_loadall(Pager*);
|
||||
|
||||
#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
|
||||
int sqlite3pager_lockstate(Pager*);
|
||||
#endif
|
||||
|
||||
#ifdef SQLITE_TEST
|
||||
void sqlite3pager_refdump(Pager*);
|
||||
int pager3_refinfo_enable;
|
||||
Index: sqlite3.h
|
||||
===================================================================
|
||||
RCS file: /cvsroot/mozilla/db/sqlite3/src/sqlite3.h,v
|
||||
retrieving revision 1.2.2.4
|
||||
diff -u -u -8 -p -r1.2.2.4 sqlite3.h
|
||||
--- sqlite3.h 22 Feb 2006 20:56:28 -0000 1.2.2.4
|
||||
+++ sqlite3.h 11 Apr 2006 17:28:51 -0000
|
||||
@@ -1482,12 +1482,25 @@ int sqlite3_table_column_metadata(
|
||||
*/
|
||||
int sqlite3_bind_parameter_indexes(
|
||||
sqlite3_stmt *pStmt,
|
||||
const char *zName,
|
||||
int **pIndexes
|
||||
);
|
||||
void sqlite3_free_parameter_indexes(int *pIndexes);
|
||||
|
||||
+/*
|
||||
+** Preload the databases into the pager cache, up to the maximum size of the
|
||||
+** pager cache.
|
||||
+**
|
||||
+** For a database to be loaded successfully, the pager must be active. That is,
|
||||
+** there must be an open statement on that database. See sqlite3pager_loadall
|
||||
+**
|
||||
+** There might be many databases attached to the given connection. We iterate
|
||||
+** them all and try to load them. If none are loadable successfully, we return
|
||||
+** an error. Otherwise, we return OK.
|
||||
+*/
|
||||
+int sqlite3Preload(sqlite3* db);
|
||||
+
|
||||
#ifdef __cplusplus
|
||||
} /* End of the 'extern "C"' block */
|
||||
#endif
|
||||
#endif
|
||||
|
|
@ -54,7 +54,7 @@ interface nsIFile;
|
|||
* creating prepared statements, executing SQL, and examining database
|
||||
* errors.
|
||||
*/
|
||||
[scriptable, uuid(e42f0655-cdc3-47c6-824a-cdb88c731cb9)]
|
||||
[scriptable, uuid(ffddc17b-cec3-492c-b13e-d5393c4b1595)]
|
||||
interface mozIStorageConnection : nsISupports {
|
||||
/*
|
||||
* Initialization and status
|
||||
|
|
@ -275,20 +275,4 @@ interface mozIStorageConnection : nsISupports {
|
|||
*/
|
||||
nsIFile backupDB(in AString aFileName,
|
||||
[optional] in nsIFile aParentDirectory);
|
||||
|
||||
/**
|
||||
* This is used to preload the database cache. It loads pages from the
|
||||
* start of the database file until the memory cache (specified by
|
||||
* "PRAGMA cache_size=") is full or the entire file is read.
|
||||
*
|
||||
* The cache MUST be active on the database for this to work. This means
|
||||
* that you must have a transaction open on the connection, or have a
|
||||
* transaction open on another connection that shares the same pager cache.
|
||||
* This cached data will go away when the transaction is closed.
|
||||
*
|
||||
* This preload operation can dramatically speed up read operations because
|
||||
* the data is loaded as one large block. Normally, pages are read in on
|
||||
* demand, which can cause many disk seeks.
|
||||
*/
|
||||
void preload();
|
||||
};
|
||||
|
|
|
|||
|
|
@ -870,20 +870,6 @@ mozStorageConnection::BackupDB(const nsAString &aFileName,
|
|||
return mDatabaseFile->CopyTo(parentDir, fileName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Mozilla-specific sqlite function to preload the DB into the cache. See the
|
||||
* IDL and sqlite3.h
|
||||
*/
|
||||
nsresult
|
||||
mozStorageConnection::Preload()
|
||||
{
|
||||
/*
|
||||
int srv = sqlite3Preload(mDBConn);
|
||||
return ConvertResultCode(srv);
|
||||
*/
|
||||
return NS_OK; // XXX restore after sqlite upgrade
|
||||
}
|
||||
|
||||
/**
|
||||
** Other bits
|
||||
**/
|
||||
|
|
|
|||
|
|
@ -749,15 +749,6 @@ nsNavHistory::InitDB(PRBool *aDoImport)
|
|||
|
||||
// --- PUT SCHEMA-MODIFYING THINGS (like create table) ABOVE THIS LINE ---
|
||||
|
||||
// This causes the database data to be preloaded up to the maximum cache size
|
||||
// set above. This dramatically speeds up some later operations. Failures
|
||||
// here are not fatal since we can run fine without this.
|
||||
if (cachePages > 0) {
|
||||
rv = mDBConn->Preload();
|
||||
if (NS_FAILED(rv))
|
||||
NS_WARNING("Preload of database failed");
|
||||
}
|
||||
|
||||
// DO NOT PUT ANY SCHEMA-MODIFYING THINGS HERE
|
||||
|
||||
rv = InitStatements();
|
||||
|
|
|
|||
|
|
@ -544,10 +544,6 @@ nsFormHistory::StartCache()
|
|||
rv = mDummyConnection->ExecuteSimpleSQL(cacheSizePragma);
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
// preload the cache
|
||||
rv = mDummyConnection->Preload();
|
||||
NS_ENSURE_SUCCESS(rv, rv);
|
||||
|
||||
return NS_OK;
|
||||
}
|
||||
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче