fix 64456, cache file stream when updating multiple local messages, r=sspitzer, sr=mscott

mailnews/base/util/makefile.win

@@ -76,6 +76,7 @@ CPP_OBJS=                               \
 	.\$(OBJDIR)\nsMsgMailNewsUrl.obj	\
 	.\$(OBJDIR)\nsMsgTxn.obj			\
 	.\$(OBJDIR)\nsMsgI18N.obj			\
+	.\$(OBJDIR)\nsMsgCompactFolder.obj			\
 	$(NULL)
 
 LLIBS=								\

mailnews/base/util/nsMsgDBFolder.cpp

@@ -114,6 +114,8 @@ NS_IMETHODIMP nsMsgDBFolder::Shutdown(PRBool shutdownChildren)
 
 NS_IMETHODIMP nsMsgDBFolder::ForceDBClosed ()
 {
+   NotifyStoreClosedAllHeaders();
+
     PRUint32 cnt = 0, i;
     if (mSubFolders)
     {

mailnews/base/util/nsMsgFolder.cpp

@@ -2431,8 +2431,21 @@ NS_IMETHODIMP nsMsgFolder::EnableNotifications(PRInt32 notificationType, PRBool
 	if(notificationType == nsIMsgFolder::allMessageCountNotifications)
 	{
 		mNotifyCountChanges = enable;
+    // start and stop db batching here. This is under the theory
+    // that any time we want to enable and disable notifications,
+    // we're probably doing something that should be batched.
+    nsCOMPtr <nsIMsgDatabase> database;
+
+    nsresult rv = GetMsgDatabase(nsnull, getter_AddRefs(database));
 		if(enable)
+    {
+      if (database)
+        database->EndBatch();
 			UpdateSummaryTotals(PR_TRUE);
+    }
+    else if (database)
+      database->StartBatch();
+
 		return NS_OK;
 	}
 	return NS_ERROR_NOT_IMPLEMENTED;

mailnews/db/msgdb/public/nsIMsgDatabase.idl

@@ -204,9 +204,14 @@ interface  nsIMsgDatabase : nsIDBChangeAnnouncer {
   // but also useful to tell the summary to mark itself invalid
   void SetSummaryValid(in boolean valid);
 
+  // batching - can be used to cache file stream for local mail,
+  // and perhaps to use the mdb batching mechanism as well.
+  void StartBatch();
+  void EndBatch();
   // offline operations
   [noscript] void ListAllOfflineOpIds(in nsMsgKeyArrayPtr offlineOpIds);
   [noscript] void ListAllOfflineDeletes(in nsMsgKeyArrayPtr offlineDeletes);
+  [noscript] void ListAllOfflineMsgs(in nsMsgKeyArrayPtr offlineMsgs);
 
   readonly attribute nsMsgKey LowWaterArticleNum;
   readonly attribute nsMsgKey HighWaterArticleNum;

mailnews/db/msgdb/public/nsImapMailDatabase.h

@@ -34,13 +34,17 @@ public:
 	virtual ~nsImapMailDatabase();
 	
 	NS_IMETHOD		Open(nsIFileSpec *folderName, PRBool create, PRBool upgrading, nsIMsgDatabase** pMessageDB);
-	
-	NS_IMETHOD			SetSummaryValid(PRBool valid = TRUE);
+
+  NS_IMETHOD    StartBatch();
+  NS_IMETHOD    EndBatch();
+	NS_IMETHOD    SetSummaryValid(PRBool valid = TRUE);
+  NS_IMETHOD    DeleteMessages(nsMsgKeyArray* nsMsgKeys, nsIDBChangeListener *instigator);
 	
 protected:
 	// IMAP does not set local file flags, override does nothing
 	virtual void	UpdateFolderFlag(nsIMsgDBHdr *msgHdr, PRBool bSet, 
 									 MsgFlags flag, nsIOFileStream **ppFileStream);
+
 };
 
 

mailnews/db/msgdb/public/nsMailDatabase.h

@@ -52,6 +52,9 @@ public:
 
 	NS_IMETHOD DeleteMessages(nsMsgKeyArray* nsMsgKeys, nsIDBChangeListener *instigator);
 
+  NS_IMETHOD StartBatch();
+  NS_IMETHOD EndBatch();
+
 	static  nsresult		SetFolderInfoValid(nsFileSpec *folderSpec, int num, int numunread);
 	nsresult				GetFolderName(nsString &folderName);
 	virtual nsMailDatabase	*GetMailDB() {return this;}

mailnews/db/msgdb/public/nsMsgDatabase.h

@@ -61,6 +61,7 @@ public:
   virtual nsresult GetThreadForMsgKey(nsMsgKey msgKey, nsIMsgThread **result);
   virtual nsresult EnumerateUnreadMessages(nsISimpleEnumerator* *result);
   virtual nsresult EnumerateReadMessages(nsISimpleEnumerator* *result);
+  virtual nsresult EnumerateMessagesWithFlag(nsISimpleEnumerator* *result, PRUint32 flag);
 
   // this might just be for debugging - we'll see.
   nsresult ListAllThreads(nsMsgKeyArray *threadIds);

mailnews/db/msgdb/src/nsImapMailDatabase.cpp

@@ -160,3 +160,19 @@ void	nsImapMailDatabase::UpdateFolderFlag(nsIMsgDBHdr * /* msgHdr */, PRBool /*
 {
 }
 
+// override so nsMailDatabase methods that deal with m_folderStream are *not* called
+NS_IMETHODIMP nsImapMailDatabase::StartBatch()
+{
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsImapMailDatabase::EndBatch()
+{
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsImapMailDatabase::DeleteMessages(nsMsgKeyArray* nsMsgKeys, nsIDBChangeListener *instigator)
+{
+	return nsMsgDatabase::DeleteMessages(nsMsgKeys, instigator);
+}
+

mailnews/db/msgdb/src/nsMailDatabase.cpp

@@ -176,10 +176,28 @@ nsresult nsMailDatabase::OnNewPath (nsFileSpec &newPath)
 	return ret;
 }
 
-nsresult nsMailDatabase::DeleteMessages(nsMsgKeyArray* nsMsgKeys, nsIDBChangeListener *instigator)
+// cache m_folderStream to make updating mozilla status flags fast
+NS_IMETHODIMP nsMailDatabase::StartBatch()
+{
+  if (!m_folderStream)
+	  m_folderStream = new nsIOFileStream(nsFileSpec(m_dbName));
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsMailDatabase::EndBatch()
+{
+	if (m_folderStream)
+		delete m_folderStream;
+	m_folderStream = NULL;
+  return NS_OK;
+}
+
+
+NS_IMETHODIMP nsMailDatabase::DeleteMessages(nsMsgKeyArray* nsMsgKeys, nsIDBChangeListener *instigator)
 {
 	nsresult ret = NS_OK;
-	m_folderStream = new nsIOFileStream(nsFileSpec(m_dbName));
+  if (!m_folderStream)
+	  m_folderStream = new nsIOFileStream(nsFileSpec(m_dbName));
 	ret = nsMsgDatabase::DeleteMessages(nsMsgKeys, instigator);
 	if (m_folderStream)
 		delete m_folderStream;

mailnews/db/msgdb/src/nsMsgDatabase.cpp

@@ -1293,6 +1293,16 @@ NS_IMETHODIMP nsMsgDatabase::GetMsgHdrForKey(nsMsgKey key, nsIMsgDBHdr **pmsgHdr
 	return err;
 }
 
+NS_IMETHODIMP nsMsgDatabase::StartBatch()
+{
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsMsgDatabase::EndBatch()
+{
+  return NS_OK;
+}
+
 NS_IMETHODIMP nsMsgDatabase::DeleteMessage(nsMsgKey key, nsIDBChangeListener *instigator, PRBool commit)
 {
 	nsresult	err = NS_OK;
@@ -2432,44 +2442,16 @@ nsMsgDatabase::EnumerateThreads(PRUint32 viewType, nsISimpleEnumerator* *result)
     return NS_OK;
 }
 
-
-#if 0
-// convenience function to iterate through a db only looking for unread messages.
-// It turns out to be possible to do this more efficiently in news, since read/unread
-// status is kept in a much more compact format (the newsrc format).
-// So this is the base implementation, which news databases override.
-nsresult nsMsgDatabase::ListNextUnread(ListContext **pContext, nsMsgHdr **pResult)
+// only return headers with a particular flag set
+static nsresult
+nsMsgFlagSetFilter(nsIMsgDBHdr *msg, void *closure)
 {
-	nsMsgHdr	*pHeader;
-	nsresult			dbErr = NS_OK;
-	PRBool			lastWasRead = TRUE;
-	*pResult = NULL;
-
-	while (PR_TRUE)
-	{
-		if (*pContext == NULL)
-			dbErr = ListFirst (pContext, &pHeader);
-		else
-			dbErr = ListNext(*pContext, &pHeader);
-
-		if (dbErr != NS_OK)
-		{
-			ListDone(*pContext);
-			break;
-		}
-
-		else if (dbErr != NS_OK)	 
-			break;
-		if (IsHeaderRead(pHeader, &lastWasRead) == NS_OK && !lastWasRead)
-			break;
-		else
-			NS_RELEASE(pHeader);
-	}
-	if (!lastWasRead)
-		*pResult = pHeader;
-	return dbErr;
+  PRUint32 msgFlags, desiredFlags;
+  desiredFlags = * (PRUint32 *) closure;
+  msg->GetFlags(&msgFlags);
+  return (msgFlags & desiredFlags) ? NS_OK : NS_COMFALSE;
 }
-#else
+
 static nsresult
 nsMsgUnreadFilter(nsIMsgDBHdr* msg, void* closure)
 {
@@ -2492,6 +2474,17 @@ nsMsgDatabase::EnumerateUnreadMessages(nsISimpleEnumerator* *result)
     return NS_OK;
 }
 
+nsresult  
+nsMsgDatabase::EnumerateMessagesWithFlag(nsISimpleEnumerator* *result, PRUint32 flag)
+{
+    nsMsgDBEnumerator* e = new nsMsgDBEnumerator(this, nsMsgFlagSetFilter, &flag);
+    if (e == nsnull)
+        return NS_ERROR_OUT_OF_MEMORY;
+    NS_ADDREF(e);
+    *result = e;
+    return NS_OK;
+}
+
 static nsresult
 nsMsgReadFilter(nsIMsgDBHdr* msg, void* closure)
 {
@@ -2503,6 +2496,8 @@ nsMsgReadFilter(nsIMsgDBHdr* msg, void* closure)
     return wasRead ? NS_OK : NS_COMFALSE;
 }
 
+// note that we can't just use EnumerateMessagesWithFlag(MSG_FLAG_READ) because we need
+// to call IsHeaderRead.
 nsresult
 nsMsgDatabase::EnumerateReadMessages(nsISimpleEnumerator* *result)
 {
@@ -2513,7 +2508,6 @@ nsMsgDatabase::EnumerateReadMessages(nsISimpleEnumerator* *result)
     *result = e;
     return NS_OK;
 }
-#endif
 
 NS_IMETHODIMP nsMsgDatabase::CreateNewHdr(nsMsgKey key, nsIMsgDBHdr **pnewHdr)
 {
@@ -3311,6 +3305,35 @@ nsresult nsMsgDatabase::ListAllThreads(nsMsgKeyArray *threadIds)
 	return rv;
 }
 
+NS_IMETHODIMP nsMsgDatabase::ListAllOfflineMsgs(nsMsgKeyArray *outputKeys)
+{
+  nsCOMPtr <nsISimpleEnumerator> enumerator;
+  nsresult rv = EnumerateMessagesWithFlag(getter_AddRefs(enumerator), MSG_FLAG_OFFLINE);
+  if (NS_SUCCEEDED(rv) && enumerator)
+  {
+		PRBool hasMoreElements;
+		while(NS_SUCCEEDED(enumerator->HasMoreElements(&hasMoreElements)) && hasMoreElements)
+		{
+			nsCOMPtr<nsISupports> childSupports;
+			rv = enumerator->GetNext(getter_AddRefs(childSupports));
+			if(NS_FAILED(rv))
+				return rv;
+
+      // clear out db hdr, because it won't be valid when we get rid of the .msf file
+		  nsCOMPtr<nsIMsgDBHdr> dbMessage(do_QueryInterface(childSupports, &rv));
+		  if(NS_SUCCEEDED(rv) && dbMessage)
+      {
+        nsMsgKey msgKey;
+			  dbMessage->GetMessageKey(&msgKey);
+        outputKeys->Add(msgKey);
+      }
+    }
+  }
+  outputKeys->QuickSort();
+
+  return rv;
+}
+
 NS_IMETHODIMP nsMsgDatabase::ListAllOfflineOpIds(nsMsgKeyArray *offlineOpIds)
 {
 	nsresult ret = NS_OK;