mozilla
/
releases-comm-central
зеркало из https://github.com/mozilla/releases-comm-central.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 466227 - gloda should index IMAP messages that are not offline too (headers only). Implement indexing of messages in non-offline IMAP folders, and add tests for both offline and non-offline IMAP messages. r=asuth+bienvenu, sr=bienvenu. 

This patch includes a fix for a bug in maild.js where the input stream gets GCed early, and also has asuth's "v1 make test_query_core.js xpcshell test work on linux" fix.

--HG--
extra : rebase_source : 36a930adcbef55175915f655394a2df44d1acd5f
This commit is contained in:
Siddharth Agarwal 2009-09-04 23:26:52 +05:30
Родитель 372875b505
Коммит 8ad2e059b6
26 изменённых файлов: 887 добавлений и 319 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

3
mailnews/db/gloda/modules/datamodel.js
Просмотреть файл

@ -449,7 +449,8 @@ GlodaMessage.prototype = {
_clone: function gloda_message_clone() {
return new GlodaMessage(this._datastore, this._id, this._folderID,
this._messageKey, this._conversationID, this._conversation, this._date,
this._headerMessageID, this._deleted);
this._headerMessageID, this._deleted, this._jsonText, this._notability,
this._subject, this._indexedBodyText, this._attachmentNames);
},
_ghost: function gloda_message_ghost() {

263
mailnews/db/gloda/modules/datastore.js
Просмотреть файл

@ -20,6 +20,7 @@
*
* Contributor(s):
* Andrew Sutherland <asutherland@asutherland.org>
* Siddharth Agarwal <sid.bugzilla@gmail.com>
*
* Alternatively, the contents of this file may be used under the terms of
* either the GNU General Public License Version 2 or later (the "GPL"), or
@ -53,52 +54,6 @@ Cu.import("resource://app/modules/gloda/datamodel.js");
Cu.import("resource://app/modules/gloda/databind.js");
Cu.import("resource://app/modules/gloda/collection.js");
let MBM_LOG = Log4Moz.repository.getLogger("gloda.ds.mbm");
/**
* @class This callback handles processing the asynchronous query results of
* GlodaDatastore.getMessagesByMessageID. Because that method is only
* called as part of the indexing process, we are guaranteed that there will
* be no real caching ramifications. Accordingly, we can also defer our cache
* processing (via GlodaCollectionManager) until the query completes.
*
* @param aMsgIDToIndex Map from message-id to the desired
*
* @constructor
*/
function MessagesByMessageIdCallback(aMsgIDToIndex, aResults,
aCallback, aCallbackThis) {
this.msgIDToIndex = aMsgIDToIndex;
this.results = aResults;
this.callback = aCallback;
this.callbackThis = aCallbackThis;
}
MessagesByMessageIdCallback.prototype = {
onItemsAdded: function gloda_ds_mbmi_onItemsAdded(aItems, aCollection) {
// just outright bail if we are shutdown
if (GlodaDatastore.datastoreIsShutdown)
return;
MBM_LOG.debug("getting results...");
for each (let [, message] in Iterator(aItems)) {
this.results[this.msgIDToIndex[message.headerMessageID]].push(message);
}
},
onItemsModified: function () {},
onItemsRemoved: function () {},
onQueryCompleted: function gloda_ds_mbmi_onQueryCompleted(aCollection) {
// just outright bail if we are shutdown
if (GlodaDatastore.datastoreIsShutdown)
return;
MBM_LOG.debug("query completed, notifying... " + this.results);
// we no longer need to unify; it is done for us.
this.callback.call(this.callbackThis, this.results);
}
};
let PCH_LOG = Log4Moz.repository.getLogger("gloda.ds.pch");
function PostCommitHandler(aCallbacks) {
@ -1888,34 +1843,46 @@ var GlodaDatastore = {
this.asyncConnection.lastErrorString + " - " + ex);
}
// we only create the full-text row if the body is non-null.
// so, even though body might be null, we still want to create the
// full-text search row
if (aMessage._bodyLines) {
if (aMessage._content && aMessage._content.hasContent())
aMessage._indexedBodyText = aMessage._content.getContentString(true);
else
aMessage._indexedBodyText = aMessage._bodyLines.join("\n");
// we create the full-text row for any message that isn't a ghost,
// whether we have the body or not
if (aMessage.folderID !== null)
this._insertMessageText(aMessage);
},
let imts = this._insertMessageTextStatement;
imts.bindInt64Parameter(0, aMessage.id);
imts.bindStringParameter(1, aMessage._subject);
/**
* Inserts a full-text row. This should only be called if you're sure you want
* to insert a row into the table.
*/
_insertMessageText: function gloda_ds__insertMessageText(aMessage) {
if (aMessage._content && aMessage._content.hasContent())
aMessage._indexedBodyText = aMessage._content.getContentString(true);
else if (aMessage._bodyLines)
aMessage._indexedBodyText = aMessage._bodyLines.join("\n");
else
aMessage._indexedBodyText = null;
let imts = this._insertMessageTextStatement;
imts.bindInt64Parameter(0, aMessage.id);
imts.bindStringParameter(1, aMessage._subject);
if (aMessage._indexedBodyText == null)
imts.bindNullParameter(2);
else
imts.bindStringParameter(2, aMessage._indexedBodyText);
if (aMessage._attachmentNames === null)
imts.bindNullParameter(3);
else
imts.bindStringParameter(3, aMessage._attachmentNames.join("\n"));
imts.bindStringParameter(4, aMessage._indexAuthor);
imts.bindStringParameter(5, aMessage._indexRecipients);
if (aMessage._attachmentNames === null)
imts.bindNullParameter(3);
else
imts.bindStringParameter(3, aMessage._attachmentNames.join("\n"));
try {
imts.executeAsync(this.trackAsync());
}
catch(ex) {
throw("error executing fulltext statement... " +
this.asyncConnection.lastError + ": " +
this.asyncConnection.lastErrorString + " - " + ex);
}
imts.bindStringParameter(4, aMessage._indexAuthor);
imts.bindStringParameter(5, aMessage._indexRecipients);
try {
imts.executeAsync(this.trackAsync());
}
catch(ex) {
throw("error executing fulltext statement... " +
this.asyncConnection.lastError + ": " +
this.asyncConnection.lastErrorString + " - " + ex);
}
},
@ -1933,10 +1900,24 @@ var GlodaDatastore = {
return this._updateMessageStatement;
},
get _updateMessageTextStatement() {
let statement = this._createAsyncStatement(
"UPDATE messagesText SET body = ?1, \
attachmentNames = ?2 \
WHERE docid = ?3");
this.__defineGetter__("_updateMessageTextStatement", function() statement);
return this._updateMessageTextStatement;
},
/**
* Update the database row associated with the message. If aBody is supplied,
* the associated full-text row is created; it is assumed that it did not
* previously exist.
* Update the database row associated with the message. If the message is
* not a ghost and has _isNew defined, messagesText is affected.
*
* aMessage._isNew is currently equivalent to the fact that there is no
* full-text row associated with this message, and we work with this
* assumption here. Note that if aMessage._isNew is not defined, then
* we don't do anything.
*/
updateMessage: function gloda_ds_updateMessage(aMessage) {
let ums = this._updateMessageStatement;
@ -1963,31 +1944,11 @@ var GlodaDatastore = {
ums.executeAsync(this.trackAsync());
if (aMessage._isNew && aMessage._bodyLines) {
if (aMessage._content && aMessage._content.hasContent())
aMessage._indexedBodyText = aMessage._content.getContentString(true);
if (aMessage.folderID !== null) {
if (aMessage._isNew === true)
this._insertMessageText(aMessage);
else
aMessage._indexedBodyText = aMessage._bodyLines.join("\n");
let imts = this._insertMessageTextStatement;
imts.bindInt64Parameter(0, aMessage.id);
imts.bindStringParameter(1, aMessage._subject);
imts.bindStringParameter(2, aMessage._indexedBodyText);
if (aMessage._attachmentNames === null)
imts.bindNullParameter(3);
else
imts.bindStringParameter(3, aMessage._attachmentNames.join("\n"));
imts.bindStringParameter(4, aMessage._indexAuthor);
imts.bindStringParameter(5, aMessage._indexRecipients);
try {
imts.executeAsync(this.trackAsync());
}
catch(ex) {
throw("error executing fulltext statement... " +
this.asyncConnection.lastError + ": " +
this.asyncConnection.lastErrorString + " - " + ex);
}
this._updateMessageText(aMessage);
}
// In completely abstract theory, this is where we would call
@ -1996,6 +1957,51 @@ var GlodaDatastore = {
// handles it.)
},
/**
* Updates the full-text row associated with this message. This only performs
* the UPDATE query if the indexed body text has changed, which means that if
* the body hasn't changed but the attachments have, we don't update.
*/
_updateMessageText: function gloda_ds__updateMessageText(aMessage) {
let newIndexedBodyText;
if (aMessage._content && aMessage._content.hasContent())
newIndexedBodyText = aMessage._content.getContentString(true);
else if (aMessage._bodyLines)
newIndexedBodyText = aMessage._bodyLines.join("\n");
else
newIndexedBodyText = null;
// If the body text matches, don't perform an update
if (newIndexedBodyText == aMessage._indexedBodyText) {
this._log.debug("in _updateMessageText, skipping update because body matches");
return;
}
aMessage._indexedBodyText = newIndexedBodyText;
let umts = this._updateMessageTextStatement;
umts.bindInt64Parameter(2, aMessage.id);
if (aMessage._indexedBodyText == null)
umts.bindNullParameter(0);
else
umts.bindStringParameter(0, aMessage._indexedBodyText);
if (aMessage._attachmentNames == null)
umts.bindNullParameter(1);
else
umts.bindStringParameter(1, aMessage._attachmentNames.join("\n"));
try {
umts.executeAsync(this.trackAsync());
}
catch(ex) {
throw("error executing fulltext statement... " +
this.asyncConnection.lastError + ": " +
this.asyncConnection.lastErrorString + " - " + ex);
}
},
get _updateMessageLocationStatement() {
let statement = this._createAsyncStatement(
"UPDATE messages SET folderID = ?1, messageKey = ?2 WHERE id = ?3");
@ -2212,49 +2218,6 @@ var GlodaDatastore = {
return messageIDs;
},
/**
* Given a list of Message-ID's, return a matching list of lists of messages
* matching those Message-ID's. So if you pass an array with three
* Message-ID's ["a", "b", "c"], you would get back an array containing
* 3 lists, where the first list contains all the messages with a message-id
* of "a", and so forth. The reason a list is returned rather than null/a
* message is that we accept the reality that we have multiple copies of
* messages with the same ID.
* This call is asynchronous because it depends on previously created messages
* to be reflected in our results, which requires us to execute on the async
* thread where all our writes happen. This also turns out to be a
* reasonable thing because we could imagine pathological cases where there
* could be a lot of message-id's and/or a lot of messages with those
* message-id's.
*/
getMessagesByMessageID: function gloda_ds_getMessagesByMessageID(aMessageIDs,
aCallback, aCallbackThis) {
let msgIDToIndex = {};
let results = [];
for (let iID = 0; iID < aMessageIDs.length; ++iID) {
let msgID = aMessageIDs[iID];
results.push([]);
msgIDToIndex[msgID] = iID;
}
// Unfortunately, IN doesn't work with statement binding mechanisms, and
// a chain of ORed tests really can't be bound unless we create one per
// value of N (seems silly).
let quotedIDs = ["'" + msgID.replace("'", "''", "g") + "'" for each
([i, msgID] in Iterator(aMessageIDs))]
let sqlString = "SELECT * FROM messages WHERE headerMessageID IN (" +
quotedIDs + ")";
let nounDef = GlodaMessage.prototype.NOUN_DEF;
let listener = new MessagesByMessageIdCallback(msgIDToIndex, results,
aCallback, aCallbackThis);
// Use a null query because we don't want any update notifications about our
// collection. They would just confuse and anger the listener.
let query = new nounDef.nullQueryClass();
return this._queryFromSQLString(sqlString, [], nounDef,
query, listener);
},
get _updateMessagesMarkDeletedByFolderID() {
let statement = this._createAsyncStatement(
"UPDATE messages SET folderID = NULL, messageKey = NULL, \
@ -2763,8 +2726,12 @@ var GlodaDatastore = {
// return. For example, in the case of messages, deleted or ghost
// messages should not be returned by this query layer. We require
// hand-rolled SQL to do that for now.
let validityConstraintSuffix =
nounDef.dbQueryValidityConstraintSuffix || "";
let validityConstraintSuffix;
if (nounDef.dbQueryValidityConstraintSuffix &&
!aQuery.options.noDbQueryValidityConstraints)
validityConstraintSuffix = nounDef.dbQueryValidityConstraintSuffix;
else
validityConstraintSuffix = "";
for (let iUnion = 0; iUnion < unionQueries.length; iUnion++) {
let curQuery = unionQueries[iUnion];
@ -2912,8 +2879,14 @@ var GlodaDatastore = {
}
let sqlString = "SELECT * FROM " + nounDef.tableName;
if (nounDef.dbQueryJoinMagic && !aQuery.options.noMagic)
sqlString += nounDef.dbQueryJoinMagic;
if (!aQuery.options.noMagic) {
if (aQuery.options.noDbQueryValidityConstraints &&
nounDef.dbQueryJoinMagicWithNoValidityConstraints)
sqlString += nounDef.dbQueryJoinMagicWithNoValidityConstraints;
else if (nounDef.dbQueryJoinMagic)
sqlString += nounDef.dbQueryJoinMagic;
}
if (whereClauses.length)
sqlString += " WHERE (" + whereClauses.join(") OR (") + ")";

51
mailnews/db/gloda/modules/fundattr.js
Просмотреть файл

@ -49,7 +49,7 @@ Cu.import("resource://app/modules/gloda/gloda.js");
Cu.import("resource://app/modules/gloda/datastore.js");
Cu.import("resource://app/modules/gloda/noun_mimetype.js");
Cu.import("resource://app/modules/gloda/connotent.js");
/**
* @namespace The Gloda Fundamental Attribute provider is a special attribute
@ -100,6 +100,7 @@ var GlodaFundAttr = {
_attrCc: null,
_attrCcMe: null,
_attrDate: null,
_attrHeaderMessageID: null,
defineAttributes: function() {
/* ***** Conversations ***** */
@ -286,6 +287,19 @@ var GlodaFundAttr = {
objectNoun: Gloda.NOUN_DATE,
}); // tested-by: test_attributes_fundamental
// Header message ID.
this._attrHeaderMessageID = Gloda.defineAttribute({
provider: this,
extensionName: Gloda.BUILT_IN,
attributeType: Gloda.kAttrFundamental,
attributeName: "headerMessageID",
singular: true,
special: Gloda.kSpecialColumn,
specialColumnName: "headerMessageID",
subjectNouns: [Gloda.NOUN_MESSAGE],
objectNoun: Gloda.NOUN_STRING,
}); // tested-by: test_attributes_fundamental
// Attachment MIME Types
this._attrAttachmentTypes = Gloda.defineAttribute({
provider: this,
@ -432,19 +446,21 @@ var GlodaFundAttr = {
aGlodaMessage.cc = ccIdentities;
// -- Attachments
let attachmentTypes = [];
for each (let [, attachment] in Iterator(aMimeMsg.allAttachments)) {
// We don't care about would-be attachments that are not user-intended
// attachments but rather artifacts of the message content.
// We also want to avoid dealing with obviously bogus mime types.
// (If you don't have a "/", you are probably bogus.)
if (attachment.isRealAttachment &&
(attachment.contentType.indexOf("/") != -1)) {
attachmentTypes.push(MimeTypeNoun.getMimeType(attachment.contentType));
if (aMimeMsg) {
let attachmentTypes = [];
for each (let [, attachment] in Iterator(aMimeMsg.allAttachments)) {
// We don't care about would-be attachments that are not user-intended
// attachments but rather artifacts of the message content.
// We also want to avoid dealing with obviously bogus mime types.
// (If you don't have a "/", you are probably bogus.)
if (attachment.isRealAttachment &&
(attachment.contentType.indexOf("/") != -1)) {
attachmentTypes.push(MimeTypeNoun.getMimeType(attachment.contentType));
}
}
if (attachmentTypes.length) {
aGlodaMessage.attachmentTypes = attachmentTypes;
}
}
if (attachmentTypes.length) {
aGlodaMessage.attachmentTypes = attachmentTypes;
}
// TODO: deal with mailing lists, including implicit-to. this will require
@ -565,9 +581,12 @@ var GlodaFundAttr = {
if (fromMeCc.length)
aGlodaMessage.fromMeCc = fromMeCc;
if (aRawReps.bodyLines &&
this.contentWhittle({}, aRawReps.bodyLines, aRawReps.content)) {
// we were going to do something here?
// Content
if (aRawReps.bodyLines) {
aGlodaMessage._content = new GlodaContent();
if (this.contentWhittle({}, aRawReps.bodyLines, aGlodaMessage._content)) {
// we were going to do something here?
}
}
yield Gloda.kWorkDone;

86
mailnews/db/gloda/modules/gloda.js
Просмотреть файл

@ -54,6 +54,49 @@ Cu.import("resource://app/modules/gloda/utils.js");
Cu.import("resource://app/modules/iteratorUtils.jsm");
let MBM_LOG = Log4Moz.repository.getLogger("gloda.NS.mbm");
/**
* @class This callback handles processing the asynchronous query results of
* Gloda.getMessagesByMessageID.
*
* @param aMsgIDToIndex Map from message-id to the desired
*
* @constructor
*/
function MessagesByMessageIdCallback(aMsgIDToIndex, aResults,
aCallback, aCallbackThis) {
this.msgIDToIndex = aMsgIDToIndex;
this.results = aResults;
this.callback = aCallback;
this.callbackThis = aCallbackThis;
}
MessagesByMessageIdCallback.prototype = {
onItemsAdded: function gloda_ds_mbmi_onItemsAdded(aItems, aCollection) {
// just outright bail if we are shutdown
if (GlodaDatastore.datastoreIsShutdown)
return;
MBM_LOG.debug("getting results...");
for each (let [, message] in Iterator(aItems)) {
this.results[this.msgIDToIndex[message.headerMessageID]].push(message);
}
},
onItemsModified: function () {},
onItemsRemoved: function () {},
onQueryCompleted: function gloda_ds_mbmi_onQueryCompleted(aCollection) {
// just outright bail if we are shutdown
if (GlodaDatastore.datastoreIsShutdown)
return;
MBM_LOG.debug("query completed, notifying... " + this.results);
// we no longer need to unify; it is done for us.
this.callback.call(this.callbackThis, this.results);
}
};
/**
* Provides the user-visible (and extension visible) global database
* functionality. There is currently a dependency/ordering
@ -326,6 +369,45 @@ var Gloda = {
return query.getCollection(aListener, aData);
},
/**
* Given a list of Message-ID's, return a matching list of lists of messages
* matching those Message-ID's. So if you pass an array with three
* Message-ID's ["a", "b", "c"], you would get back an array containing
* 3 lists, where the first list contains all the messages with a message-id
* of "a", and so forth. The reason a list is returned rather than null/a
* message is that we accept the reality that we have multiple copies of
* messages with the same ID.
* This call is asynchronous because it depends on previously created messages
* to be reflected in our results, which requires us to execute on the async
* thread where all our writes happen. This also turns out to be a
* reasonable thing because we could imagine pathological cases where there
* could be a lot of message-id's and/or a lot of messages with those
* message-id's.
*/
getMessagesByMessageID: function gloda_ns_getMessagesByMessageID(aMessageIDs,
aCallback, aCallbackThis) {
let msgIDToIndex = {};
let results = [];
for (let iID = 0; iID < aMessageIDs.length; ++iID) {
let msgID = aMessageIDs[iID];
results.push([]);
msgIDToIndex[msgID] = iID;
}
let quotedIDs = ["'" + msgID.replace("'", "''", "g") + "'" for each
([i, msgID] in Iterator(aMessageIDs))];
let query = Gloda.newQuery(Gloda.NOUN_MESSAGE, {
noDbQueryValidityConstraints: true,
});
query.headerMessageID.apply(query, quotedIDs);
query.frozen = true;
let listener = new MessagesByMessageIdCallback(msgIDToIndex, results,
aCallback, aCallbackThis);
return query.getCollection(listener);
},
/**
* @testpoint gloda.ns.getMessageContent
*/
@ -1077,6 +1159,10 @@ var Gloda = {
dbAttribAdjuster: GlodaDatastore.adjustMessageAttributes,
dbQueryValidityConstraintSuffix:
" AND +deleted = 0 AND +folderID IS NOT NULL AND +messageKey IS NOT NULL",
// This is what's used when we have no validity constraints, i.e. we allow
// for ghost messages, which do not have a row in the messagesText table.
dbQueryJoinMagicWithNoValidityConstraints:
" LEFT JOIN messagesText ON messages.id = messagesText.rowid",
objInsert: GlodaDatastore.insertMessage,
objUpdate: GlodaDatastore.updateMessage,
toParamAndValue: function(aMessage) {

126
mailnews/db/gloda/modules/indexer.js
Просмотреть файл

@ -21,6 +21,7 @@
* Contributor(s):
* Andrew Sutherland <asutherland@asutherland.org>
* Kent James <kent@caspia.com>
* Siddharth Agarwal <sid.bugzilla@gmail.com>
*
* Alternatively, the contents of this file may be used under the terms of
* either the GNU General Public License Version 2 or later (the "GPL"), or
@ -243,7 +244,8 @@ IndexingJob.prototype = {
* meta-data.
*
* Indexing Message Control
* - We index IMAP messages that are offline. We index all local messages.
* - We index the headers of all IMAP messages. We index the bodies of all IMAP
* messages that are offline. We index all local messages.
* We plan to avoid indexing news messages.
* - We would like a way to express desires about indexing that either don't
* confound offline storage with indexing, or actually allow some choice.
@ -1061,8 +1063,8 @@ var GlodaIndexer = {
// The basic search expression is:
// ((GLODA_MESSAGE_ID_PROPERTY Is 0) || (GLODA_DIRTY_PROPERTY Isnt 0))
// If the folder !isLocal we add the terms:
// && (Status Is nsMsgMessageFlags.Offline)
// && (Status Isnt nsMsgMessageFlags.Expunged)
// - if the folder is offline -- && (Status Is nsMsgMessageFlags.Offline)
// - && (Status Isnt nsMsgMessageFlags.Expunged)
let searchSession = Cc["@mozilla.org/messenger/searchSession;1"]
.createInstance(Ci.nsIMsgSearchSession);
@ -1081,7 +1083,7 @@ var GlodaIndexer = {
searchTerm.beginsGrouping = true;
searchTerm.attrib = nsMsgSearchAttrib.Uint32HdrProperty;
searchTerm.op = nsMsgSearchOp.Is;
value = searchTerm.value;
let value = searchTerm.value;
value.attrib = searchTerm.attrib;
value.status = 0;
searchTerm.value = value;
@ -1103,16 +1105,19 @@ var GlodaIndexer = {
if (!isLocal)
{
// third term: && Status Is nsMsgMessageFlags.Offline
searchTerm = searchSession.createTerm();
searchTerm.booleanAnd = true;
searchTerm.attrib = nsMsgSearchAttrib.MsgStatus;
searchTerm.op = nsMsgSearchOp.Is;
value = searchTerm.value;
value.attrib = searchTerm.attrib;
value.status = Ci.nsMsgMessageFlags.Offline;
searchTerm.value = value;
searchTerms.appendElement(searchTerm, false);
// If the folder is offline, then the message should be too
if (this._indexingFolder.flags & Ci.nsMsgFolderFlags.Offline) {
// third term: && Status Is nsMsgMessageFlags.Offline
searchTerm = searchSession.createTerm();
searchTerm.booleanAnd = true;
searchTerm.attrib = nsMsgSearchAttrib.MsgStatus;
searchTerm.op = nsMsgSearchOp.Is;
value = searchTerm.value;
value.attrib = searchTerm.attrib;
value.status = Ci.nsMsgMessageFlags.Offline;
searchTerm.value = value;
searchTerms.appendElement(searchTerm, false);
}
// fourth term: && Status Isnt nsMsgMessageFlags.Expunged
searchTerm = searchSession.createTerm();
@ -1652,10 +1657,10 @@ var GlodaIndexer = {
Ci.nsIMsgFolder);
if (!this.shouldIndexFolder(folder))
continue;
// we could also check nsMsgFolderFlags.Mail conceivably...
let isLocal = folder instanceof Ci.nsIMsgLocalMailFolder;
// we only index local folders or IMAP folders that are marked offline
if (!isLocal && !(folder.flags & Ci.nsMsgFolderFlags.Offline))
// we only index local or IMAP folders
if (!(folder instanceof Ci.nsIMsgLocalMailFolder) &&
!(folder instanceof Ci.nsIMsgImapMailFolder))
continue;
let glodaFolder = Gloda.getFolderForFolder(folder);
@ -1734,7 +1739,6 @@ var GlodaIndexer = {
// don't do something. so we will yield with kWorkSync for every block.
const HEADER_CHECK_BLOCK_SIZE = 25;
let isLocal = this._indexingFolder instanceof Ci.nsIMsgLocalMailFolder;
// we can safely presume if we are here that this folder has been selected
// for offline processing...
@ -1835,8 +1839,6 @@ var GlodaIndexer = {
// if we are already in the correct folder, our "get in the folder" clause
// will not execute, so we need to make sure this value is accurate in
// that case. (and we want to avoid multiple checks...)
let folderIsLocal =
this._indexingFolder instanceof Ci.nsIMsgLocalMailFolder;
for (; aJob.offset < aJob.items.length; aJob.offset++) {
let item = aJob.items[aJob.offset];
// item is either [folder ID, message key] or
@ -1850,9 +1852,6 @@ var GlodaIndexer = {
// stay out of folders we should not be in!
if (!this.shouldIndexFolder(this._indexingFolder))
continue;
folderIsLocal =
this._indexingFolder instanceof Ci.nsIMsgLocalMailFolder;
}
let msgHdr;
@ -1863,12 +1862,9 @@ var GlodaIndexer = {
// TODO fixme to not assume singular message-id's.
msgHdr = this._indexingDatabase.getMsgHdrForMessageID(item[1]);
// it needs a header, the header needs to not be expunged, plus, the
// message needs to be considered offline.
// it needs a header, and the header needs to not be expunged.
if (msgHdr &&
!(msgHdr.flags & Components.interfaces.nsMsgMessageFlags.Expunged) &&
(folderIsLocal ||
(msgHdr.flags & Components.interfaces.nsMsgMessageFlags.Offline)))
!(msgHdr.flags & Components.interfaces.nsMsgMessageFlags.Expunged))
yield this._callbackHandle.pushAndGo(this._indexMessage(msgHdr,
this._callbackHandle));
else
@ -2069,9 +2065,16 @@ var GlodaIndexer = {
let msgFolder = aMsgHdr.folder;
if (!this.indexer.shouldIndexFolder(msgFolder))
return;
// Make sure the message is eligible for indexing.
// We index local messages, IMAP messages that are offline, and IMAP
// messages that aren't offline but whose folders aren't offline either
let isFolderLocal = msgFolder instanceof Ci.nsIMsgLocalMailFolder;
if (!isFolderLocal && !(msgFolder.flags&Ci.nsMsgFolderFlags.Offline))
return;
if (!isFolderLocal) {
if (!(aMsgHdr.flags & Ci.nsMsgMessageFlags.Offline) &&
(msgFolder.flags & Ci.nsMsgFolderFlags.Offline))
return;
}
// mark the folder dirty so we know to look in it, but there is no need
// to mark the message because it will lack a gloda-id anyways.
@ -2379,9 +2382,16 @@ var GlodaIndexer = {
let msgFolder = aMsgHdr.folder;
if (!this.indexer.shouldIndexFolder(msgFolder))
return;
// Make sure the message is eligible for indexing.
// We index local messages, IMAP messages that are offline, and IMAP
// messages that aren't offline but whose folders aren't offline either
let isFolderLocal = msgFolder instanceof Ci.nsIMsgLocalMailFolder;
if (!isFolderLocal && !(msgFolder.flags&Ci.nsMsgFolderFlags.Offline))
return;
if (!isFolderLocal) {
if (!(aMsgHdr.flags & Ci.nsMsgMessageFlags.Offline) &&
(msgFolder.flags & Ci.nsMsgFolderFlags.Offline))
return;
}
// mark the message as dirty
// (We could check for the presence of the gloda message id property
@ -2399,13 +2409,15 @@ var GlodaIndexer = {
}
// only queue the message if we haven't overflowed our event-driven budget
if (this.indexer._pendingAddJob.items.length <
this.indexer._indexMaxEventQueueMessages)
this.indexer._indexMaxEventQueueMessages) {
this.indexer._pendingAddJob.items.push(
[GlodaDatastore._mapFolder(msgFolder).id,
aMsgHdr.messageKey]);
else
this.indexer.indexing = true;
}
else {
this.indexer.indexingSweepNeeded = true;
this.indexer.indexing = true;
}
},
OnItemAdded: function gloda_indexer_OnItemAdded(aParentItem, aItem) {
@ -2485,13 +2497,27 @@ var GlodaIndexer = {
_indexMessage: function gloda_indexMessage(aMsgHdr, aCallbackHandle) {
let logDebug = this._log.level <= Log4Moz.Level.Debug;
if (logDebug)
this._log.debug("*** Indexing message: " + aMsgHdr.messageKey + " : " +
aMsgHdr.subject);
MsgHdrToMimeMessage(aMsgHdr, aCallbackHandle.callbackThis,
aCallbackHandle.callback);
let [,aMimeMsg] = yield this.kWorkAsync;
// If the message is offline, then get the message body as well
let isMsgOffline = false;
let aMimeMsg;
if ((aMsgHdr.flags & Ci.nsMsgMessageFlags.Offline) ||
(aMsgHdr.folder instanceof Ci.nsIMsgLocalMailFolder)) {
isMsgOffline = true;
MsgHdrToMimeMessage(aMsgHdr, aCallbackHandle.callbackThis,
aCallbackHandle.callback);
[,aMimeMsg] = yield this.kWorkAsync;
}
else {
if (logDebug)
this._log.debug(" * Message is not offline -- only headers indexed");
}
if (logDebug)
this._log.debug(" * Got message, subject " + aMsgHdr.subject);
if (this._unitTestSuperVerbose) {
if (aMimeMsg)
@ -2511,8 +2537,8 @@ var GlodaIndexer = {
// also see if we already know about the message...
references.push(aMsgHdr.messageId);
this._datastore.getMessagesByMessageID(references, aCallbackHandle.callback,
aCallbackHandle.callbackThis);
Gloda.getMessagesByMessageID(references, aCallbackHandle.callback,
aCallbackHandle.callbackThis);
// (ancestorLists has a direct correspondence to the message ids)
let ancestorLists = yield this.kWorkAsync;
@ -2668,7 +2694,7 @@ var GlodaIndexer = {
let bodyPlain = aMimeMsg.coerceBodyToPlaintext(aMsgHdr.folder);
if (bodyPlain) {
curMsg._bodyLines = bodyPlain.split(/\r?\n/);
curMsg._content = new GlodaContent();
// curMsg._content gets set by fundattr.js
}
}
@ -2676,20 +2702,20 @@ var GlodaIndexer = {
curMsg._isNew = true;
// curMsg._indexedBodyText is set by GlodaDatastore.insertMessage or
// GlodaDatastore.updateMessage
curMsg._subject = aMsgHdr.mime2DecodedSubject;
curMsg._attachmentNames = attachmentNames;
// curMsg._indexAuthor gets set by fundattr.js
// curMsg._indexRecipients gets set by fundattr.js
}
curMsg._subject = aMsgHdr.mime2DecodedSubject;
curMsg._attachmentNames = attachmentNames;
// curMsg._indexAuthor gets set by fundattr.js
// curMsg._indexRecipients gets set by fundattr.js
// zero the notability so everything in grokNounItem can just increment
curMsg.notability = 0;
yield aCallbackHandle.pushAndGo(
Gloda.grokNounItem(curMsg,
{header: aMsgHdr, mime: aMimeMsg,
bodyLines: curMsg._bodyLines, content: curMsg._content},
{header: aMsgHdr, mime: aMimeMsg, bodyLines: curMsg._bodyLines},
isConceptuallyNew, isRecordNew,
aCallbackHandle));

3
mailnews/db/gloda/modules/query.js
Просмотреть файл

@ -73,6 +73,9 @@ Cu.import("resource://app/modules/gloda/datastore.js");
* this makes things more readable but is unlikely to improve
* performance. (Namely, my use of 'offsets' for full-text stuff
* ends up in the EXPLAIN plan twice despite this.)
* - noDbQueryValidityConstraints: Indicates that any validity constraints
* should be ignored. This should be used when you need to get every
* match regardless of whether it's valid.
*
* @property _owner The query instance that holds the list of unions...
* @property _constraints A list of (lists of OR constraints) that are ANDed

303
mailnews/db/gloda/test/unit/resources/glodaTestHelper.js
Просмотреть файл

@ -20,6 +20,7 @@
*
* Contributor(s):
* Andrew Sutherland <asutherland@asutherland.org>
* Siddharth Agarwal <sid.bugzilla@gmail.com>
*
* Alternatively, the contents of this file may be used under the terms of
* either the GNU General Public License Version 2 or later (the "GPL"), or
@ -35,8 +36,9 @@
*
* ***** END LICENSE BLOCK ***** */
// -- Pull in the POP3 fake-server / local account helper code
load("../../test_mailnewslocal/unit/head_maillocal.js");
// Import the main scripts that mailnews tests need to set up and tear down
load("../../mailnews/resources/mailDirService.js");
load("../../mailnews/resources/mailTestUtils.js");
/**
* Create a 'me' identity of "me@localhost" for the benefit of Gloda. At the
@ -217,6 +219,8 @@ const INJECT_FAKE_SERVER = 1;
const INJECT_MBOX = 2;
/** Inject messages using addMessage. */
const INJECT_ADDMESSAGE = 3;
/** Inject messages using the IMAP fakeserver. */
const INJECT_IMAP_FAKE_SERVER = 4;
/**
* Convert a list of synthetic messages to a form appropriate to feed to the
@ -227,7 +231,32 @@ function _synthMessagesToFakeRep(aSynthMessages) {
(msg in aSynthMessages)];
}
function lobotomizeAdaptiveIndexer() {
// The indexer doesn't need to worry about load; zero his rescheduling time.
GlodaIndexer._INDEX_INTERVAL = 0;
let realIdleService = GlodaIndexer._idleService;
// pretend we are always idle
GlodaIndexer._idleService = {
idleTime: 1000,
addIdleObserver: function() {
realIdleService.addIdleObserver.apply(realIdleService, arguments);
},
removeIdleObserver: function() {
realIdleService.removeIdleObserver.apply(realIdleService, arguments);
}
};
// Lobotomize the adaptive indexer
GlodaIndexer._cpuTargetIndexTime = 10000;
GlodaIndexer._CPU_TARGET_INDEX_TIME_ACTIVE = 10000;
GlodaIndexer._CPU_TARGET_INDEX_TIME_IDLE = 10000;
GlodaIndexer._CPU_IS_BUSY_TIME = 10000;
GlodaIndexer._PAUSE_LATE_IS_BUSY_TIME = 10000;
}
function imsInit() {
dump("Initializing index message state\n");
let ims = indexMessageState;
if (!ims.inited) {
@ -246,29 +275,12 @@ function imsInit() {
// Make the indexer be more verbose about indexing for us...
GlodaIndexer._unitTestSuperVerbose = true;
// The indexer doesn't need to worry about load; zero his rescheduling time.
GlodaIndexer._INDEX_INTERVAL = 0;
let realIdleService = GlodaIndexer._idleService;
// pretend we are always idle
GlodaIndexer._idleService = {
idleTime: 1000,
addIdleObserver: function() {
realIdleService.addIdleObserver.apply(realIdleService, arguments);
},
removeIdleObserver: function() {
realIdleService.removeIdleObserver.apply(realIdleService, arguments);
}
};
// Lobotomize the adaptive indexer
GlodaIndexer._cpuTargetIndexTime = 10000;
GlodaIndexer._CPU_TARGET_INDEX_TIME_ACTIVE = 10000;
GlodaIndexer._CPU_TARGET_INDEX_TIME_IDLE = 10000;
GlodaIndexer._CPU_IS_BUSY_TIME = 10000;
GlodaIndexer._PAUSE_LATE_IS_BUSY_TIME = 10000;
lobotomizeAdaptiveIndexer();
if (ims.injectMechanism == INJECT_FAKE_SERVER) {
// -- Pull in the POP3 fake-server / local account helper code
load("../../test_mailnewslocal/unit/head_maillocal.js");
// set up POP3 fakeserver to feed things in...
[ims.daemon, ims.server] = setupServerDaemon();
// (this will call loadLocalMailAccount())
@ -285,6 +297,42 @@ function imsInit() {
// we need an inbox
loadLocalMailAccount();
}
else if (ims.injectMechanism == INJECT_IMAP_FAKE_SERVER) {
// Pull in the IMAP fake server code
load("../../test_imap/unit/head_server.js");
// set up IMAP fakeserver and incoming server
ims.daemon = new imapDaemon();
ims.server = makeServer(ims.daemon, "");
ims.incomingServer = createLocalIMAPServer();
// we need a local account for the IMAP server to have its sent messages in
loadLocalMailAccount();
// We need an identity so that updateFolder doesn't fail
let acctMgr = Cc["@mozilla.org/messenger/account-manager;1"]
.getService(Ci.nsIMsgAccountManager);
let localAccount = acctMgr.createAccount();
let identity = acctMgr.createIdentity();
localAccount.addIdentity(identity);
localAccount.defaultIdentity = identity;
localAccount.incomingServer = gLocalIncomingServer;
acctMgr.defaultAccount = localAccount;
// Let's also have another account, using the same identity
let imapAccount = acctMgr.createAccount();
imapAccount.addIdentity(identity);
imapAccount.defaultIdentity = identity;
imapAccount.incomingServer = ims.incomingServer;
// The server doesn't support more than one connection
prefSvc.setIntPref("mail.server.server1.max_cached_connections", 1);
// We aren't interested in downloading messages automatically
prefSvc.setBoolPref("mail.server.server1.download_on_biff", false);
// Set the inbox to not be offline
ims.imapInbox = ims.incomingServer.rootFolder.getChildNamed("Inbox");
ims.imapInbox.flags &= ~Ci.nsMsgFolderFlags.Offline;
}
ims.inited = true;
}
@ -312,16 +360,11 @@ function imsInit() {
*/
function indexMessages(aSynthMessages, aVerifier, aOnDone) {
let ims = indexMessageState;
ims.inputMessages = aSynthMessages;
ims.glodaMessages = [];
ims.verifier = aVerifier;
ims.previousValue = undefined;
ims.onDone = aOnDone;
ims.expectMessages(aSynthMessages, aVerifier, aOnDone);
if (ims.injectMechanism == INJECT_FAKE_SERVER) {
ims.daemon.setMessages(_synthMessagesToFakeRep(aSynthMessages));
do_timeout(0, "driveFakeServer();");
do_timeout(0, "drivePOP3FakeServer();");
}
else if (ims.injectMechanism == INJECT_MBOX) {
ims.mboxName = "injecty" + ims.nextMboxNumber++;
@ -344,7 +387,20 @@ function indexMessages(aSynthMessages, aVerifier, aOnDone) {
localFolder.addMessage(msg.toMboxString());
}
}
else if (ims.injectMechanism == INJECT_IMAP_FAKE_SERVER) {
let ioService = Cc["@mozilla.org/network/io-service;1"]
.getService(Ci.nsIIOService);
let serverInbox = ims.daemon.getMailbox("INBOX");
for (let [, msg] in Iterator(aSynthMessages)) {
// Generate a URI out of the message
let URI = ioService.newURI("data:text/plain;base64," + btoa(msg.toMessageString()), null, null);
// Add it to the server
serverInbox.addMessage(new imapMessage(URI.spec, serverInbox.uidnext++, []));
}
// Time to do stuff with the fakeserver
driveIMAPFakeServer();
}
}
function injectMessagesUsing(aInjectMechanism) {
@ -354,6 +410,8 @@ function injectMessagesUsing(aInjectMechanism) {
var indexMessageState = {
/** have we been initialized (hooked listeners, etc.) */
inited: false,
/** whether we're due for any index notifications */
expectingIndexNotifications: false,
/** our catch-all message collection that nets us all messages passing by */
catchAllCollection: null,
/** the set of synthetic messages passed in to indexMessages */
@ -370,20 +428,49 @@ var indexMessageState = {
injectMechanism: INJECT_ADDMESSAGE,
/* === Fake Server State === */
/** nsMailServer instance with POP3_RFC1939 handler */
/** nsMailServer instance (if POP3, with POP3_RFC1939 handler) */
server: null,
serverStarted: false,
/** pop3Daemon instance */
/** pop3Daemon/imapDaemon instance */
daemon: null,
/** incoming pop3 server */
/** incoming pop3/imap server */
incomingServer: null,
/** pop3 service */
/** pop3 service (not used for imap) */
pop3Service: null,
/** IMAP inbox */
imapInbox: null,
/* === MBox Injection State === */
nextMboxNumber: 0,
mboxName: null,
/**
* Sets up messages to expect index notifications for.
*
* @param aSynthMessages The synthetic messages to expect notifications
* for. We currently don't do anything with these other than count them,
* so pass whatever you want and it will be the 'source message' (1st
* argument) to your verifier function.
* @param aVerifier The function to call to verify that the indexing had the
* desired result. Takes arguments aSynthMessage (the synthetic message
* just indexed), aGlodaMessage (the gloda message representation of the
* indexed message), and aPreviousResult (the value last returned by the
* verifier function for this given set of messages, or undefined if it is
* the first message.)
* @param aOnDone The function to call when we complete processing this set of
* messages.
*/
expectMessages: function indexMessageState_expectMessages(aSynthMessages, aVerifier,
aOnDone) {
dump("^^^ setting up " + aSynthMessages.length + " message(s) to expect\n");
this.inputMessages = aSynthMessages;
this.expectingIndexNotifications = true;
this.glodaMessages = [];
this.verifier = aVerifier;
this.previousValue = undefined;
this.onDone = aOnDone;
},
/**
* Listener to handle the completion of the POP3 message retrieval (one way or
* the other.)
@ -418,38 +505,12 @@ var indexMessageState = {
}
};
/**
* Indicate that we should expect some modified messages to be indexed.
*
* @param aMessages The messages that will be modified and we should expect
* notifications about. We currently don't do anything with these other than
* count them, so pass whatever you want and it will be the 'source message'
* (1st argument) to your verifier function.
* @param aVerifier See indexMessage's aVerifier argument.
* @param aDone The (optional) callback to call on completion.
* Perform POP3 mail fetching, seeing it through to completion.
*/
function expectModifiedMessages(aMessages, aVerifier, aOnDone) {
function drivePOP3FakeServer() {
let ims = indexMessageState;
ims.inputMessages = aMessages;
ims.glodaMessages = [];
ims.verifier = aVerifier;
ims.previousValue = undefined;
ims.onDone = aOnDone;
// we don't actually need to do anything. the caller is going to be
// triggering a notification which will spur the indexer into action. the
// indexer uses its own scheduling mechanism to drive itself, so as long
// as an event loop is active, we're good.
}
/**
* Perform the mail fetching, seeing it through to completion.
*/
function driveFakeServer() {
let ims = indexMessageState;
dump(">>> enter driveFakeServer\n");
dump(">>> enter drivePOP3FakeServer\n");
// Handle the server in a try/catch/finally loop so that we always will stop
// the server if something fails.
try {
@ -480,15 +541,48 @@ dump(">>> enter driveFakeServer\n");
while (thread.hasPendingEvents())
thread.processNextEvent(true);
}
dump("<<< exit driveFakeServer\n");
dump("<<< exit drivePOP3FakeServer\n");
}
/**
* Perform an IMAP mail fetch, seeing it through to completion
*/
function driveIMAPFakeServer() {
dump(">>> enter driveIMAPFakeServer\n");
let ims = indexMessageState;
// Handle the server in a try/catch/finally loop so that we always will stop
// the server if something fails.
try {
dump(" resetting fake server\n");
ims.server.resetTest();
// Update the inbox
dump(" issuing updateFolder\n");
ims.imapInbox.updateFolder(null);
// performTest won't work here because that seemingly blocks until the
// socket is closed, which is something undesirable here
}
catch (e) {
ims.server.stop();
do_throw(e);
}
finally {
dump(" draining events\n");
let thread = gThreadManager.currentThread;
while (thread.hasPendingEvents())
thread.processNextEvent(true);
}
dump("<<< exit driveIMAPFakeServer\n");
}
/**
* Tear down the fake server. This is very important to avoid things getting
* upset during shutdown. (Namely, XPConnect will get mad about running in
* a context without "Components" defined.)
*/
function killFakeServer() {
dump("Killing fake server\n");
let ims = indexMessageState;
ims.incomingServer.closeCachedConnections();
@ -499,6 +593,8 @@ function killFakeServer() {
var thread = gThreadManager.currentThread;
while (thread.hasPendingEvents())
thread.processNextEvent(true);
do_test_finished();
}
/**
@ -549,30 +645,46 @@ var messageIndexerListener = {
callbackOnDone: null,
onIndexNotification: function(aStatus, aPrettyName, aJobIndex, aJobTotal,
aJobItemIndex, aJobItemGoal) {
dump("((( Index listener notified! aStatus = " + aStatus + "\n");
// Ignore moving/removing notifications
if (aStatus == Gloda.kIndexerMoving || aStatus == Gloda.kIndexerRemoving)
return;
let ims = indexMessageState;
// If we shouldn't be receiving notifications and we receive one with aStatus
// != kIndexerIdle. throw.
if (!ims.expectingIndexNotifications) {
if (aStatus == Gloda.kIndexerIdle) {
dump("((( Ignoring indexing notification since it's just kIndexerIdle.\n");
return;
}
else {
do_throw("Exception during index notification -- we weren't " +
"expecting one.");
}
}
// we only care if indexing has just completed...
if (!GlodaIndexer.indexing) {
if (aStatus == Gloda.kIndexerIdle) {
if (messageIndexerListener.callbackOnDone) {
let callback = messageIndexerListener.callbackOnDone;
messageIndexerListener.callbackOnDone = null;
callback();
}
let ims = indexMessageState;
// this is just the synthetic notification if inputMessages is null
if (ims.inputMessages === null) {
dump("((( ignoring indexing notification, assuming synthetic " +
"notification.\n");
return;
}
// if we haven't seen all the messages we should see, assume that the
// rest are on their way, and are just coming in a subsequent job...
// (Also, the first time we register our listener, we will get a synthetic
// idle status; at least if the indexer is idle.)
if (ims.glodaMessages.length < ims.inputMessages.length) {
let glodaLen = ims.glodaMessages.length, inputLen =
ims.inputMessages.length;
if (glodaLen < inputLen) {
dump("((( indexing is no longer indexing, but we're still expecting " +
"more results, ignoring.\n");
inputLen + " - " + glodaLen + " = " + (inputLen - glodaLen) +
" more results, ignoring.\n");
// If we're running IMAP, then update the folder once more
if (ims.imapInbox)
ims.imapInbox.updateFolder(null);
return;
}
@ -580,6 +692,8 @@ var messageIndexerListener = {
" messages) about to verify: " +
(ims.verifier ? ims.verifier.name : "none") + " and complete: " +
(ims.onDone ? ims.onDone.name : "none") + "\n");
// If we're verifying messages, we shouldn't be expecting them
ims.expectingIndexNotifications = false;
// call the verifier. (we expect them to generate an exception if the
// verification fails, using do_check_*/do_throw; we don't care about
@ -600,6 +714,7 @@ var messageIndexerListener = {
}
}
dump("((( Verification complete\n");
if (ims.onDone) {
try {
ims.onDone();
@ -770,9 +885,8 @@ function twiddleAndTest(aSynthMsg, aActionsAndTests) {
// the underlying nsIMsgDBHdr should exist at this point...
do_check_neq(gmsg.folderMessage, null);
// prepare
expectModifiedMessages([gmsg.folderMessage], verify_next_attr);
indexMessageState.expectMessages([gmsg.folderMessage], verify_next_attr);
// tell the function to perform its mutation to the desired state
dump("twiddling: " + twiddleFunc.name + ": " + desiredState + "\n");
twiddleFunc(gmsg.folderMessage, desiredState);
}
function verify_next_attr(smsg, gmsg) {
@ -1021,8 +1135,11 @@ function _gh_test_iterator() {
}
}
if (indexMessageState.injectMechanism == INJECT_FAKE_SERVER) {
killFakeServer();
if (indexMessageState.injectMechanism == INJECT_FAKE_SERVER ||
indexMessageState.injectMechanism == INJECT_IMAP_FAKE_SERVER) {
do_test_pending();
// Give a bit of time for any remaining notifications to go through.
do_timeout(500, "killFakeServer()");
}
do_test_finished();
@ -1072,15 +1189,23 @@ function parameterizeTest(aTestFunc, aParameters) {
*
* @param aTests A list of test functions to call.
* @param aLongestTestRunTimeConceivableInSecs Optional parameter
* @param aDontInitIMS Optional parameter. If this is specified then the index
* message state is not initialized
*/
function glodaHelperRunTests(aTests, aLongestTestRunTimeConceivableInSecs) {
function glodaHelperRunTests(aTests, aLongestTestRunTimeConceivableInSecs,
aDontInitIMS) {
if (aLongestTestRunTimeConceivableInSecs == null)
aLongestTestRunTimeConceivableInSecs =
DEFAULT_LONGEST_TEST_RUN_CONCEIVABLE_SECS;
do_timeout(aLongestTestRunTimeConceivableInSecs * 1000,
"do_throw('Timeout running test, and we want you to have the log.');");
imsInit();
if (!aDontInitIMS)
imsInit();
// even if we don't want to init IMS, we want to avoid anything adaptive
// happening.
else
lobotomizeAdaptiveIndexer();
glodaHelperTests = aTests;
glodaHelperIterator = _gh_test_iterator();
next_test();
@ -1135,3 +1260,19 @@ function nukeGlodaCachesAndCollections() {
GlodaCollectionManager.defineCache(cache._nounDef, cache._maxCacheSize);
}
}
/**
* Given an IMAP folder, marks it offline and downloads its messages.
*
* @param aFolder an IMAP message folder
* @param aSynthMessages see indexMessageState.expectMessages
* @param aVerifier see indexMessageState.expectMessages
* @param aDone see indexMessageState.expectMessages
*/
function imapDownloadAllMessages(aFolder, aSynthMessages, aVerifier, aDone) {
// Let the message state know that we're expecting messages
indexMessageState.expectMessages(aSynthMessages, aVerifier, aDone);
aFolder.setFlag(Ci.nsMsgFolderFlags.Offline);
aFolder.downloadAllForOffline(null, null);
// The indexer listener is going to call aDone, so our job is done here
}

25
mailnews/db/gloda/test/unit/test_gloda_content.js
Просмотреть файл

@ -97,6 +97,17 @@ var messageInfos = [
/* ===== Tests ===== */
/**
* Hooks for pre/post setup message, used for the IMAP tests.
*/
var pre_inject_message_hook = function default_pre_inject_message_hook() {
next_test();
};
var post_inject_message_hook = function default_post_inject_message_hook() {
next_test();
};
function setup_create_message(info) {
info.body = {body: [tupe[1] for each
([, tupe] in Iterator(info.bode))].join("\r\n")};
@ -125,10 +136,12 @@ function glodaInfoStasher(aSynthMessage, aGlodaMessage) {
/**
* Actually inject all the messages we created above.
*/
var gSynMessages;
function setup_inject_messages() {
let synMessages = [info._synMsg for each
([, info] in Iterator(messageInfos))];
indexMessages(synMessages, glodaInfoStasher, next_test);
gSynMessages = [info._synMsg for each
([, info] in Iterator(messageInfos))];
indexMessages(gSynMessages, glodaInfoStasher, next_test);
}
function test_stream_message(info) {
@ -166,11 +179,15 @@ function verify_message_content(aInfo, aSynMsg, aGlodaMsg, aMsgHdr, aMimeMsg) {
var tests = [
parameterizeTest(setup_create_message, messageInfos),
function pre_inject_message() { pre_inject_message_hook(); },
setup_inject_messages,
function post_inject_message() { post_inject_message_hook(); },
// disable_index_notifications,
parameterizeTest(test_stream_message, messageInfos),
];
function run_test() {
injectMessagesUsing(INJECT_MBOX);
glodaHelperRunTests(tests);
}
injectMessagesUsing(INJECT_MBOX);

17
mailnews/db/gloda/test/unit/test_gloda_content_imap_originally_offline.js Normal file
Просмотреть файл

@ -0,0 +1,17 @@
/* -*- Mode: JavaScript; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/**
* Tests the operation of the GlodaContent (in connotent.js) and its exposure
* via Gloda.getMessageContent for IMAP messages that are originally offline.
*/
load("test_gloda_content.js");
/**
* Set the imap folder to offline before adding the messages.
*/
var pre_inject_message_hook = function imap_pre_inject_message_hook() {
indexMessageState.imapInbox.setFlag(Ci.nsMsgFolderFlags.Offline);
next_test();
};
injectMessagesUsing(INJECT_IMAP_FAKE_SERVER);

19
mailnews/db/gloda/test/unit/test_gloda_content_imap_switched_to_offline.js Normal file
Просмотреть файл

@ -0,0 +1,19 @@
/* -*- Mode: JavaScript; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/**
* Tests the operation of the GlodaContent (in connotent.js) and its exposure
* via Gloda.getMessageContent for IMAP messages that were not originally
* offline, but were later made offline.
*/
load("test_gloda_content.js");
/**
* Set the imap folder to offline after adding the messages, then force a
* download of all messages.
*/
var post_inject_message_hook = function imap_post_inject_message_hook() {
imapDownloadAllMessages(indexMessageState.imapInbox, gSynMessages,
glodaInfoStasher, next_test);
};
injectMessagesUsing(INJECT_IMAP_FAKE_SERVER);

44
mailnews/db/gloda/test/unit/test_index_messages.js
Просмотреть файл

@ -10,6 +10,9 @@
load("../../mailnews/resources/messageGenerator.js");
load("resources/glodaTestHelper.js");
// Whether we can expect fulltext results
var expectFulltextResults = true;
// Create a message generator
var msgGen = new MessageGenerator();
// Create a message scenario generator using that message generator
@ -17,13 +20,25 @@ var scenarios = new MessageScenarioFactory(msgGen);
/* ===== Threading / Conversation Grouping ===== */
var gSynMessages = [];
function allMessageInSameConversation(aSynthMessage, aGlodaMessage, aConvID) {
if (aConvID === undefined)
return aGlodaMessage.conversationID;
do_check_eq(aConvID, aGlodaMessage.conversationID);
// Cheat and stash the synthetic message (we need them for one of the IMAP
// tests)
gSynMessages.push(aSynthMessage);
return aConvID;
}
// These are overridden by the IMAP tests as needed
var pre_test_threading_hook = function default_pre_test_threading_hook() {
next_test();
};
var post_test_threading_hook = function default_post_test_threading_hook() {
next_test();
};
/**
* Test our conversation/threading logic in the straight-forward direct
* reply case, the missing intermediary case, and the siblings with missing
@ -72,12 +87,17 @@ function test_attributes_fundamental() {
indexMessages([smsg], verify_attributes_fundamental, next_test);
}
// Overridden by test_index_imap_mesasges
var get_expected_folder_URI = function local_get_expected_folder_URI() {
return gLocalInboxFolder.URI;
};
function verify_attributes_fundamental(smsg, gmsg) {
try {
// save off the message id for test_attributes_fundamental_from_disk
fundamentalGlodaMessageId = gmsg.id;
do_check_eq(gmsg.folderURI, gLocalInboxFolder.URI);
do_check_eq(gmsg.folderURI, get_expected_folder_URI());
// -- subject
do_check_eq(smsg.subject, gmsg.conversation.subject);
@ -97,12 +117,22 @@ function verify_attributes_fundamental(smsg, gmsg) {
// date
do_check_eq(smsg.date.valueOf(), gmsg.date.valueOf());
// -- message ID
do_check_eq(smsg.messageId, gmsg.headerMessageID);
// -- attachments
do_check_eq(gmsg.attachmentTypes.length, 1);
do_check_eq(gmsg.attachmentTypes[0], "text/plain");
do_check_eq(gmsg.attachmentNames.length, 1);
do_check_eq(gmsg.attachmentNames[0], "bob.txt");
// -- attachments. We won't have these if we don't have fulltext results
if (expectFulltextResults) {
do_check_eq(gmsg.attachmentTypes.length, 1);
do_check_eq(gmsg.attachmentTypes[0], "text/plain");
do_check_eq(gmsg.attachmentNames.length, 1);
do_check_eq(gmsg.attachmentNames[0], "bob.txt");
}
else {
// Make sure we don't actually get attachments!
do_check_eq(gmsg.attachmentTypes, null);
do_check_eq(gmsg.attachmentNames, null);
}
}
catch (ex) {
// print out some info on the various states of the messages...
@ -229,7 +259,9 @@ function test_message_deletion() {
var tests = [
function pre_test_threading() { pre_test_threading_hook(); },
test_threading,
function post_test_threading() { post_test_threading_hook(); },
test_attributes_fundamental,
test_attributes_fundamental_from_disk,
test_attributes_explicit,

19
mailnews/db/gloda/test/unit/test_index_messages_in_bulk.js
Просмотреть файл

@ -14,22 +14,35 @@ var scenarios = new MessageScenarioFactory(msgGen);
/**
* Provide a bunch of messages to be indexed.
*/
var gSynMessages;
function test_index_a_bunch() {
// 4-children-per, 3-deep = 21
// 6-children-per, 3 deep = 43
// 7-children-per, 3-deep = 57
// 4-children-per, 4-deep = 85
// 4-children-per, 5-deep pyramid = 341
// 5-children-per, 5-deep pyramid = 781
// 4-children-per, 6-deep pyramid = 1365 messages
let messages = scenarios.fullPyramid(4, 3);
gSynMessages = scenarios.fullPyramid(6, 3);
// we have no need to verify.
indexMessages(messages, null, next_test);
indexMessages(gSynMessages, null, next_test);
}
var pre_test_hook = function default_pre_test_hook() {
next_test();
};
var post_test_hook = function default_post_test_hook() {
next_test();
};
var tests = [
function pre_test() { pre_test_hook(); },
test_index_a_bunch,
function post_test() { post_test_hook(); },
];
function run_test() {
injectMessagesUsing(INJECT_MBOX);
glodaHelperRunTests(tests);
}
injectMessagesUsing(INJECT_MBOX);

16
mailnews/db/gloda/test/unit/test_index_non_offline_imap_messages.js Normal file
Просмотреть файл

@ -0,0 +1,16 @@
/* -*- Mode: JavaScript; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/**
* Tests how well gloda indexes IMAP messages that aren't offline.
*/
// Most of the definitions are common, so just re-use those
load("test_index_messages.js");
var get_expected_folder_URI = function imap_get_expected_folder_URI() {
return indexMessageState.imapInbox.URI;
};
var expectFulltextResults = false;
// Switch to the IMAP fake server
injectMessagesUsing(INJECT_IMAP_FAKE_SERVER);

10
mailnews/db/gloda/test/unit/test_index_non_offline_imap_messages_in_bulk.js Normal file
Просмотреть файл

@ -0,0 +1,10 @@
/* -*- Mode: JavaScript; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/**
* Tests how well gloda indexes IMAP messages that aren't offline in bulk.
*/
// The definitions are common, so just re-use those
load("test_index_messages_in_bulk.js");
// Switch to the IMAP fake server
injectMessagesUsing(INJECT_IMAP_FAKE_SERVER);

19
mailnews/db/gloda/test/unit/test_index_originally_offline_imap_messages.js Normal file
Просмотреть файл

@ -0,0 +1,19 @@
/* -*- Mode: JavaScript; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/**
* Tests how well gloda indexes IMAP messages that are offline from the start.
*/
// Most of the definitions are common, so just re-use those
load("test_index_messages.js");
var get_expected_folder_URI = function imap_get_expected_folder_URI() {
return indexMessageState.imapInbox.URI;
};
var pre_test_threading_hook = function imap_pre_test_threading_hook() {
indexMessageState.imapInbox.setFlag(Ci.nsMsgFolderFlags.Offline);
next_test();
};
// Switch to the IMAP fake server
injectMessagesUsing(INJECT_IMAP_FAKE_SERVER);

15
mailnews/db/gloda/test/unit/test_index_originally_offline_imap_messages_in_bulk.js Normal file
Просмотреть файл

@ -0,0 +1,15 @@
/* -*- Mode: JavaScript; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/**
* Tests how well gloda indexes IMAP messages that aren't offline in bulk.
*/
// The definitions are common, so just re-use those
load("test_index_messages_in_bulk.js");
var pre_test_hook = function imap_pre_test_hook() {
indexMessageState.imapInbox.setFlag(Ci.nsMsgFolderFlags.Offline);
next_test();
};
// Switch to the IMAP fake server
injectMessagesUsing(INJECT_IMAP_FAKE_SERVER);

21
mailnews/db/gloda/test/unit/test_index_switched_to_offline_imap_messages.js Normal file
Просмотреть файл

@ -0,0 +1,21 @@
/* -*- Mode: JavaScript; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/**
* Tests how well gloda indexes IMAP messages that are not offline at first, but
* are made offline later.
*/
// Most of the definitions are common, so just re-use those
load("test_index_messages.js");
var get_expected_folder_URI = function imap_get_expected_folder_URI() {
return indexMessageState.imapInbox.URI;
};
var post_test_threading_hook = function imap_post_test_threading_hook() {
// We aren't concerned about verification here, so just pass in null
imapDownloadAllMessages(indexMessageState.imapInbox, gSynMessages, null,
next_test);
};
// Switch to the IMAP fake server
injectMessagesUsing(INJECT_IMAP_FAKE_SERVER);

16
mailnews/db/gloda/test/unit/test_index_switched_to_offline_imap_messages_in_bulk.js Normal file
Просмотреть файл

@ -0,0 +1,16 @@
/* -*- Mode: JavaScript; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/**
* Tests how well gloda indexes IMAP messages that aren't offline in bulk.
*/
// The definitions are common, so just re-use those
load("test_index_messages_in_bulk.js");
var post_test_hook = function imap_post_test_hook() {
// We're not verifying anything
imapDownloadAllMessages(indexMessageState.imapInbox, gSynMessages,
null, next_test);
};
// Switch to the IMAP fake server
injectMessagesUsing(INJECT_IMAP_FAKE_SERVER);

22
mailnews/db/gloda/test/unit/test_query_core.js
Просмотреть файл

@ -241,7 +241,22 @@ function setup_search_ranking_idiom() {
new Widget(1, origin, "", 0, "bar baz", "bar baz bar bar"), // 6 + 0
new Widget(0, origin, "", 1, "bar baz", "bar baz bar bar") // 7 + 0
];
runOnIndexingComplete(next_test);
let indexingInProgress = false;
// Since we don't use the message indexer listener any more in this test, we
// need to add our own listener.
function genericIndexerCallback(aStatus) {
// If indexingInProgress is false, we've received the synthetic
// notification, so ignore it
if (indexingInProgress && aStatus == Gloda.kIndexerIdle) {
// We're done, so remove ourselves and move to the next test
Gloda.removeIndexerListener(genericIndexerCallback);
next_test();
}
}
Gloda.addIndexerListener(genericIndexerCallback);
indexingInProgress = true;
GenericIndexer.indexNewObjects(fooWidgets.concat(barBazWidgets));
}
@ -317,7 +332,6 @@ var tests = [
];
function run_test() {
// use mbox injection so we get multiple folders...
injectMessagesUsing(INJECT_MBOX);
glodaHelperRunTests(tests);
// Don't initialize the index message state
glodaHelperRunTests(tests, null, true);
}

55
mailnews/db/gloda/test/unit/test_query_messages.js
Просмотреть файл

@ -22,6 +22,12 @@ load("resources/glodaTestHelper.js");
var msgGen = new MessageGenerator();
// Create a message scenario generator using that message generator
var scenarios = new MessageScenarioFactory(msgGen);
// Whether we're using a single folder to test. We need to skip a few tests if
// we're doing so
var singleFolder = false;
// Whether we expect fulltext results. IMAP folders that are offline shouldn't
// have their bodies indexed.
var expectFulltextResults = true;
/* ===== Populate ===== */
var world = {
@ -201,6 +207,15 @@ function glodaInfoStasher(aSynthMessage, aGlodaMessage) {
world.glodaFolders.push(aGlodaMessage.folder);
}
// We override these for the IMAP tests
var pre_setup_populate_hook = function default_pre_setup_populate_hook() {
next_test();
};
var post_setup_populate_hook = function default_post_setup_populate_hook() {
next_test();
};
var gSynMessages = [];
// first, we must populate our message store with delicious messages.
function setup_populate() {
world.glodaHolderCollection = Gloda.explicitCollection(Gloda.NOUN_MESSAGE,
@ -216,13 +231,22 @@ function setup_populate() {
world.glodaConversationIds.push(null);
}
indexMessages(generateFolderMessages(), glodaInfoStasher,
setup_populate_phase_two);
let messages = generateFolderMessages();
gSynMessages = gSynMessages.concat(messages);
indexMessages(messages, glodaInfoStasher, setup_populate_phase_two);
}
function setup_populate_phase_two() {
world.phase++;
indexMessages(generateFolderMessages(), glodaInfoStasher, next_test);
// If we have one folder, we don't attempt to populate the other one
if (singleFolder) {
next_test();
}
else {
world.phase++;
let messages = generateFolderMessages();
gSynMessages = gSynMessages.concat(messages);
indexMessages(messages, glodaInfoStasher, next_test);
}
}
/* ===== Non-text queries ===== */
@ -294,6 +318,12 @@ var ts_folderCollections = [];
* @tests gloda.datastore.sqlgen.kConstraintIn
*/
function test_query_messages_by_folder() {
// If we have one folder to test with, we can't do this test more times
if (singleFolder && ts_folderNum >= 1) {
next_test();
return;
}
let folderNum = ts_folderNum++;
let query = Gloda.newQuery(Gloda.NOUN_MESSAGE);
query.folder(world.glodaFolders[folderNum]);
@ -307,7 +337,9 @@ function test_query_messages_by_folder() {
* @tests gloda.query.test.kConstraintIn
*/
function test_query_messages_by_folder_nonmatches() {
verify_nonMatches(ts_folderQueries, ts_folderCollections);
// No can do with one folder
if (!singleFolder)
verify_nonMatches(ts_folderQueries, ts_folderCollections);
next_test();
}
@ -458,7 +490,8 @@ function test_query_messages_by_body_text() {
let convBodyTerm = uniqueTermGenerator(
UNIQUE_OFFSET_BODY + UNIQUE_OFFSET_CONV + convNum);
query.bodyMatches(convBodyTerm);
queryExpect(query, world.conversationLists[convNum]); // calls next_test
queryExpect(query, expectFulltextResults ? world.conversationLists[convNum] :
[]); // calls next_test
}
/**
@ -473,7 +506,8 @@ function test_query_messages_by_attachment_names() {
let convUniqueAttachment = uniqueTermGenerator(
UNIQUE_OFFSET_ATTACHMENT + UNIQUE_OFFSET_CONV + convNum);
query.attachmentNamesMatch(convUniqueAttachment);
queryExpect(query, world.conversationLists[convNum]); // calls next_test
queryExpect(query, expectFulltextResults ? world.conversationLists[convNum] :
[]); // calls next_test
}
/**
@ -595,7 +629,9 @@ function test_query_identities_by_kind_and_value_nonmatches() {
/* ===== Driver ===== */
var tests = [
function pre_setup_populate() { pre_setup_populate_hook(); },
setup_populate,
function post_setup_populate() { post_setup_populate_hook(); },
test_query_messages_by_conversation,
test_query_messages_by_conversation,
test_query_messages_by_conversation_nonmatches,
@ -629,7 +665,8 @@ var tests = [
];
function run_test() {
// use mbox injection so we get multiple folders...
injectMessagesUsing(INJECT_MBOX);
glodaHelperRunTests(tests);
}
// use mbox injection so we get multiple folders...
injectMessagesUsing(INJECT_MBOX);

10
mailnews/db/gloda/test/unit/test_query_non_offline_imap_messages.js Normal file
Просмотреть файл

@ -0,0 +1,10 @@
/* -*- Mode: JavaScript; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/**
* Test query support for IMAP messages that aren't offline.
*/
load("test_query_messages.js");
var expectFulltextResults = false;
// TODO: Make this use multiple folders, like the local folders test
var singleFolder = true;
injectMessagesUsing(INJECT_IMAP_FAKE_SERVER);

16
mailnews/db/gloda/test/unit/test_query_originally_offline_imap_messages.js Normal file
Просмотреть файл

@ -0,0 +1,16 @@
/* -*- Mode: JavaScript; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/**
* Test query support for IMAP messages that were offline before they were
* indexed.
*/
load("test_query_messages.js");
// Set the inbox to offline before proceeding
var pre_setup_populate_hook = function imap_pre_setup_populate_hook() {
indexMessageState.imapInbox.setFlag(Ci.nsMsgFolderFlags.Offline);
next_test();
};
// TODO: Make this use multiple folders, like the local folders test
var singleFolder = true;
injectMessagesUsing(INJECT_IMAP_FAKE_SERVER);

19
mailnews/db/gloda/test/unit/test_query_switched_to_offline_imap_messages.js Normal file
Просмотреть файл

@ -0,0 +1,19 @@
/* -*- Mode: JavaScript; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/**
* Test query support for IMAP messages that were indexed, then made available
* offline.
*/
load("test_query_messages.js");
/**
* Set the imap folder to offline after adding the messages, then force a
* download of all messages.
*/
var post_setup_populate_hook = function imap_post_setup_populate_hook() {
imapDownloadAllMessages(indexMessageState.imapInbox, gSynMessages, null,
next_test);
};
// TODO: Make this use multiple folders, like the local folders test
var singleFolder = true;
injectMessagesUsing(INJECT_IMAP_FAKE_SERVER);

14
mailnews/test/fakeserver/maild.js
Просмотреть файл

@ -39,6 +39,11 @@
// Much of the original code is taken from netwerk's httpserver implementation
// Make sure we execute this file exactly once
var gMaild_js__;
if (!gMaild_js__) {
gMaild_js__ = true;
var Cc = Components.classes;
var Ci = Components.interfaces;
var Cr = Components.results;
@ -120,6 +125,12 @@ function nsMailServer(handler) {
this._handler = handler;
this._readers = [];
this._test = false;
/**
* An array to hold refs to all the input streams below, so that they don't
* get GCed
*/
this._inputStreams = [];
}
nsMailServer.prototype = {
onSocketAccepted : function (socket, trans) {
@ -130,6 +141,7 @@ nsMailServer.prototype = {
const SEGMENT_COUNT = 1024;
var input = trans.openInputStream(0, SEGMENT_SIZE, SEGMENT_COUNT)
.QueryInterface(Ci.nsIAsyncInputStream);
this._inputStreams.push(input);
var reader = new nsMailReader(this, this._handler, trans, this._debug);
this._readers.push(reader);
@ -526,3 +538,5 @@ function server(port, handler) {
srv.performTest();
return srv.playTransaction();
}
} // gMaild_js__

7
mailnews/test/resources/mailDirService.js
Просмотреть файл

@ -9,6 +9,11 @@
* <objdir>/dist/Thunderbird{Debug}.app/Contents/MacOS/mailtest/ (on Mac OS X)
*/
// Make sure we execute this file exactly once
var gMailDirService_js__;
if (!gMailDirService_js__) {
gMailDirService_js__ = true;
Components.utils.import("resource://gre/modules/XPCOMUtils.jsm");
// Declare these globally for unit tests and be done with it.
@ -133,3 +138,5 @@ catch (e) {
}
// Always ensure the profile directory exists before we start the tests
gProfileDir.create(Ci.nsIFile.DIRECTORY_TYPE, 0700);
} // gMailDirService_js__

7
mailnews/test/resources/mailTestUtils.js
Просмотреть файл

@ -35,6 +35,11 @@
*
* ***** END LICENSE BLOCK ***** */
// Make sure we execute this file exactly once
var gMailTestUtils_js__;
if (!gMailTestUtils_js__) {
gMailTestUtils_js__ = true;
// we would like for everyone to have fixIterator and toXPComArray
Components.utils.import("resource://gre/modules/iteratorUtils.jsm");
@ -275,3 +280,5 @@ function updateFolderAndNotify(aFolder, aCallback, aCallbackThis,
aFolder.updateFolder(null);
}
} // gMailTestUtils_js__