Bug 480843 - Implement a memory reporter for mailnews code, r=irving.

Unfortunately, due to issues with our build system, this patch makes the "Dark
Matter Detector" reports incorrect since it doesn't realize that our memory
reporter is reporting memory. This should be fixed as part of the build system
overhaul in bug 846540, however.

db/mork/public/mdb.h

@@ -460,7 +460,8 @@ public:
     
   NS_IMETHOD Free(nsIMdbEnv* ev, // free block from Alloc or Resize()
     void* ioBlock) = 0;     // block to be destroyed/deallocated
-    
+
+  virtual size_t GetUsedSize() = 0;
 // } ===== end nsIMdbHeap methods =====
 };
 

db/mork/src/morkZone.h

@@ -279,6 +279,8 @@ public:
     
   NS_IMETHOD Free(nsIMdbEnv* ev, // free block allocated earlier by Alloc()
     void* inBlock);
+
+  virtual size_t GetUsedSize() { return mZone_Heap->GetUsedSize(); }
 // } ===== end nsIMdbHeap methods =====
   
 // { ===== begin morkNode interface =====

db/mork/src/orkinHeap.cpp

@@ -19,22 +19,15 @@
 #include "morkEnv.h"
 #endif
 
+#include "nsIMemoryReporter.h"
+
 #include <stdlib.h>
 
 //3456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789
 
 
 orkinHeap::orkinHeap() // does nothing
-#ifdef MORK_DEBUG_HEAP_STATS
-  : sHeap_AllocCount( 0 )
-  , sHeap_FreeCount( 0 )
-  , sHeap_BlockCount( 0 )
-  
-  , sHeap_BlockVolume( 0 )
-  , sHeap_HighWaterVolume( 0 )
-  , sHeap_HighWaterTenKilo( 0 )
-  , sHeap_HighWaterHundredKilo( 0 )
-#endif /*MORK_DEBUG_HEAP_STATS*/
+  : mUsedSize(0)
 {
 }
 
@@ -43,101 +36,49 @@ orkinHeap::~orkinHeap() // does nothing
 {
 }
 
+NS_MEMORY_REPORTER_MALLOC_SIZEOF_ON_ALLOC_FUN(MorkSizeOfOnAlloc)
+NS_MEMORY_REPORTER_MALLOC_SIZEOF_ON_FREE_FUN(MorkSizeOfOnFree)
+
 // { ===== begin nsIMdbHeap methods =====
 /*virtual*/ mdb_err
 orkinHeap::Alloc(nsIMdbEnv* mev, // allocate a piece of memory
-  mdb_size inSize,   // requested size of new memory block 
+  mdb_size inSize,  // requested size of new memory block
   void** outBlock)  // memory block of inSize bytes, or nil
 {
-#ifdef MORK_DEBUG_HEAP_STATS
-  mdb_size realSize = inSize;
-  inSize += 12; // sizeof(mork_u4) * 3
-  ++sHeap_AllocCount;
-#endif /*MORK_DEBUG_HEAP_STATS*/
 
   MORK_USED_1(mev);
   mdb_err outErr = NS_OK;
   void* block = malloc(inSize);
   if ( !block )
     outErr = morkEnv_kOutOfMemoryError;
-#ifdef MORK_DEBUG_HEAP_STATS
   else
-  {
-    printf("%lx allocating %d\n", this, realSize);
-    mork_u4* array = (mork_u4*) block;
-    *array++ = (mork_u4) this;
-    *array++ = realSize;
-    *array++ = orkinHeap_kTag;
-    block = array;
-    ++sHeap_BlockCount;
-    mork_num blockVol = sHeap_BlockVolume + realSize;
-    sHeap_BlockVolume = blockVol;
-    if ( blockVol > sHeap_HighWaterVolume )
-    {
-      sHeap_HighWaterVolume = blockVol;
-      
-      mork_num tenKiloVol = blockVol / (10 * 1024);
-      if ( tenKiloVol > sHeap_HighWaterTenKilo )
-      {
-        sHeap_HighWaterTenKilo = tenKiloVol;
-      
-        mork_num hundredKiloVol = blockVol / (100 * 1024);
-        if ( hundredKiloVol > sHeap_HighWaterHundredKilo )
-          sHeap_HighWaterHundredKilo = hundredKiloVol;
-      }
-    }
-  }
-#endif /*MORK_DEBUG_HEAP_STATS*/
-    
+    mUsedSize += MorkSizeOfOnAlloc(block);
+
   MORK_ASSERT(outBlock);
   if ( outBlock )
     *outBlock = block;
   return outErr;
 }
-  
+
 /*virtual*/ mdb_err
 orkinHeap::Free(nsIMdbEnv* mev, // free block allocated earlier by Alloc()
   void* inBlock)
 {
-#ifdef MORK_DEBUG_HEAP_STATS
-  ++sHeap_FreeCount;
-#endif /*MORK_DEBUG_HEAP_STATS*/
-
   MORK_USED_1(mev);
   MORK_ASSERT(inBlock);
   if ( inBlock )
   {
-#ifdef MORK_DEBUG_HEAP_STATS
-    morkEnv* ev = 0; //morkEnv::FromMdbEnv(mev);
-    mork_u4* array = (mork_u4*) inBlock;
-    if ( *--array != orkinHeap_kTag )
-    {
-      if ( ev )
-        ev->NewWarning("heap block tag not hEaP");
-    }
-    mork_u4 realSize = *--array;
-    inBlock = --array; // skip over heap ptr too.
-    
-    printf("%lx freeing %d\n", this, realSize);
-    if ( sHeap_BlockCount )
-      --sHeap_BlockCount;
-    else if ( ev ) 
-      ev->NewWarning("sHeap_BlockCount underflow");
-    
-    if ( sHeap_BlockVolume >= realSize )
-      sHeap_BlockVolume -= realSize;
-    else if ( ev )
-    {
-      sHeap_BlockVolume = 0;
-      ev->NewWarning("sHeap_BlockVolume underflow");
-    }
-#endif /*MORK_DEBUG_HEAP_STATS*/
-    
+    mUsedSize -= MorkSizeOfOnFree(inBlock);
     free(inBlock);
   }
   return NS_OK;
 }
 
+size_t
+orkinHeap::GetUsedSize()
+{
+  return mUsedSize;
+}
 // } ===== end nsIMdbHeap methods =====
 
 //3456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789

db/mork/src/orkinHeap.h

@@ -21,27 +21,9 @@
 /*| orkinHeap: 
 |*/
 class orkinHeap : public nsIMdbHeap { //
-
-#ifdef MORK_DEBUG_HEAP_STATS
 protected:
-  mork_num sHeap_AllocCount;  // number of times Alloc() is called
-  mork_num sHeap_FreeCount;   // number of times Free() is called
-  mork_num sHeap_BlockCount;  // number of outstanding blocks
-  
-  mork_num sHeap_BlockVolume; // sum of sizes for all outstanding blocks
-  mork_num sHeap_HighWaterVolume;  // largest value of sHeap_BlockVolume seen
-  mork_num sHeap_HighWaterTenKilo; // HighWaterVolume in 10K granularity
-  mork_num sHeap_HighWaterHundredKilo; // HighWaterVolume in 100K granularity
-  
-public: // getters
-  mork_num HeapAllocCount() const { return sHeap_AllocCount; }
-  mork_num HeapFreeCount() const { return sHeap_FreeCount; }
-  mork_num HeapBlockCount() const { return sHeap_AllocCount - sHeap_FreeCount; }
-  
-  mork_num HeapBlockVolume() const { return sHeap_BlockVolume; }
-  mork_num HeapHighWaterVolume() const { return sHeap_HighWaterVolume; }
-#endif /*MORK_DEBUG_HEAP_STATS*/
-  
+  size_t mUsedSize;
+
 public:
   orkinHeap(); // does nothing
   virtual ~orkinHeap(); // does nothing
@@ -59,6 +41,8 @@ public:
     
   NS_IMETHOD Free(nsIMdbEnv* ev, // free block allocated earlier by Alloc()
     void* inBlock);
+
+  virtual size_t GetUsedSize();
 // } ===== end nsIMdbHeap methods =====
 
 };

mailnews/db/msgdb/public/nsDBFolderInfo.h

@@ -59,7 +59,15 @@ public:
                                       uint64_t *propertyValue);
 
   nsTArray<nsMsgKey> m_lateredKeys; // list of latered messages
-  
+
+  virtual size_t SizeOfExcludingThis(nsMallocSizeOfFun aMallocSizeOf) const
+  {
+    return m_lateredKeys.SizeOfExcludingThis(aMallocSizeOf);
+  }
+  virtual size_t SizeOfIncludingThis(nsMallocSizeOfFun aMallocSizeOf) const
+  {
+    return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
+  }
 protected:
 
   // initialize from appropriate table and row in existing db.

mailnews/db/msgdb/public/nsMsgDatabase.h

@@ -98,6 +98,12 @@ protected:
 
 };
 
+namespace mozilla {
+namespace mailnews {
+class MsgDBReporter;
+}
+}
+
 class nsMsgDatabase : public nsIMsgDatabase
 {
 public:
@@ -407,8 +413,20 @@ protected:
   // not-reference holding array of enumerators we've handed out.
   // If a db goes away, it will clean up the outstanding enumerators.
   nsTArray<nsMsgDBEnumerator *> m_enumerators;
+
+  // Memory reporter details
+public:
+  static size_t HeaderHashSizeOf(PLDHashEntryHdr *hdr,
+                                 nsMallocSizeOfFun aMallocSizeOf,
+                                 void *arg);
+  virtual size_t SizeOfExcludingThis(nsMallocSizeOfFun aMallocSizeOf) const;
+  virtual size_t SizeOfIncludingThis(nsMallocSizeOfFun aMallocSizeOf) const
+  {
+    return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
+  }
 private:
   uint32_t m_cacheSize;
+  nsRefPtr<mozilla::mailnews::MsgDBReporter> mMemReporter;
 };
 
 class nsMsgRetentionSettings : public nsIMsgRetentionSettings

mailnews/db/msgdb/public/nsMsgHdr.h

@@ -40,6 +40,16 @@ public:
     bool        IsAncestorOf(nsIMsgDBHdr *possibleChild);
     bool        IsAncestorKilled(uint32_t ancestorsToCheck);
     void        ReparentInThread(nsIMsgThread *thread);
+
+    size_t SizeOfExcludingThis(nsMallocSizeOfFun aMallocSizeOfFun) const
+    {
+      return m_references.SizeOfExcludingThis(aMallocSizeOfFun);
+    }
+    size_t SizeOfIncludingThis(nsMallocSizeOfFun aMallocSizeOfFun) const
+    {
+      return aMallocSizeOfFun(this) + SizeOfExcludingThis(aMallocSizeOfFun);
+    }
+
 protected:
     nsresult SetStringColumn(const char *str, mdb_token token);
     nsresult SetUInt32Column(uint32_t value, mdb_token token);

mailnews/db/msgdb/src/nsMsgDatabase.cpp

@@ -47,6 +47,7 @@
 #include "nsIMsgPluggableStore.h"
 #include "nsAlgorithm.h"
 #include "nsArrayEnumerator.h"
+#include "nsIMemoryReporter.h"
 #include <algorithm>
 
 #if defined(DEBUG_sspitzer_) || defined(DEBUG_seth_)
@@ -996,6 +997,115 @@ void nsMsgDatabase::DumpCache()
   }
 }
 
+// Memory Reporting implementations
+
+size_t nsMsgDatabase::HeaderHashSizeOf(PLDHashEntryHdr *hdr,
+                                       nsMallocSizeOfFun aMallocSizeOf,
+                                       void *arg)
+{
+  MsgHdrHashElement *entry = reinterpret_cast<MsgHdrHashElement*>(hdr);
+  // Sigh, this is dangerous, but so long as this is a closed system, this is
+  // safe.
+  return static_cast<nsMsgHdr*>(entry->mHdr)->
+    SizeOfIncludingThis(aMallocSizeOf);
+}
+
+size_t nsMsgDatabase::SizeOfExcludingThis(nsMallocSizeOfFun aMallocSizeOf) const
+{
+  size_t totalSize = 0;
+  if (m_dbFolderInfo)
+    totalSize += m_dbFolderInfo->SizeOfExcludingThis(aMallocSizeOf);
+  if (m_mdbEnv)
+  {
+    nsIMdbHeap *morkHeap = nullptr;
+    m_mdbEnv->GetHeap(&morkHeap);
+    if (morkHeap)
+      totalSize += morkHeap->GetUsedSize();
+  }
+  totalSize += m_newSet.SizeOfExcludingThis(aMallocSizeOf);
+  totalSize += m_ChangeListeners.SizeOfExcludingThis(aMallocSizeOf);
+  totalSize += m_threads.SizeOfExcludingThis(aMallocSizeOf);
+  // We have two tables of header objects, but every header in m_cachedHeaders
+  // should be in m_headersInUse.
+  // double-counting...
+  size_t headerSize = 0;
+  if (m_headersInUse)
+  {
+    headerSize = PL_DHashTableSizeOfIncludingThis(m_headersInUse,
+      nsMsgDatabase::HeaderHashSizeOf, aMallocSizeOf);
+  }
+  totalSize += headerSize;
+  if (m_msgReferences)
+    totalSize += PL_DHashTableSizeOfIncludingThis(m_msgReferences, nullptr,
+      aMallocSizeOf);
+  return totalSize;
+}
+
+namespace mozilla {
+namespace mailnews {
+
+NS_MEMORY_REPORTER_MALLOC_SIZEOF_FUN(GetMallocSize)
+
+class MsgDBReporter MOZ_FINAL : public nsIMemoryReporter
+{
+  nsMsgDatabase *mDatabase;
+public:
+  MsgDBReporter(nsMsgDatabase *db) : mDatabase(db) {}
+
+  NS_DECL_ISUPPORTS
+  NS_IMETHOD GetProcess(nsACString &process)
+  {
+    process.Truncate();
+    return NS_OK;
+  }
+
+  NS_IMETHOD GetPath(nsACString &memoryPath)
+  {
+    memoryPath.AssignLiteral("explicit/maildb/database(");
+    nsCOMPtr<nsIMsgFolder> folder;
+    mDatabase->GetFolder(getter_AddRefs(folder));
+    if (folder)
+    {
+      nsAutoCString folderURL;
+      folder->GetFolderURL(folderURL);
+      folderURL.ReplaceChar('/', '\\');
+      memoryPath += folderURL;
+    } else {
+      memoryPath.AppendLiteral("UNKNOWN-FOLDER");
+    }
+    memoryPath.Append(')');
+    return NS_OK;
+  }
+
+  NS_IMETHOD GetKind(int *kind)
+  {
+    *kind = nsIMemoryReporter::KIND_HEAP;
+    return NS_OK;
+  }
+
+  NS_IMETHOD GetUnits(int *units)
+  {
+    *units = nsIMemoryReporter::UNITS_BYTES;
+    return NS_OK;
+  }
+
+  NS_IMETHOD GetAmount(int64_t *amount)
+  {
+    *amount = mDatabase->SizeOfIncludingThis(GetMallocSize);
+    return NS_OK;
+  }
+
+  NS_IMETHOD GetDescription(nsACString &desc)
+  {
+    desc.AssignLiteral("Memory used for the folder database.");
+    return NS_OK;
+  }
+};
+
+NS_IMPL_ISUPPORTS1(MsgDBReporter, nsIMemoryReporter)
+}
+}
+
 nsMsgDatabase::nsMsgDatabase()
         : m_dbFolderInfo(nullptr),
         m_nextPseudoMsgKey(kFirstPseudoKey),
@@ -1042,10 +1152,13 @@ nsMsgDatabase::nsMsgDatabase()
         m_msgReferences(nullptr),
         m_cacheSize(kMaxHdrsInCache)
 {
+  mMemReporter = new mozilla::mailnews::MsgDBReporter(this);
+  NS_RegisterMemoryReporter(mMemReporter);
 }
 
 nsMsgDatabase::~nsMsgDatabase()
 {
+  NS_UnregisterMemoryReporter(mMemReporter);
   //  Close(FALSE);  // better have already been closed.
   ClearCachedObjects(true);
   ClearEnumerators();