Bug 1711948 - Add surfaces from image containers to the memory report. r=tnikkel

An image container can keep a surface alive longer than it can remain in
the cache, if it is indeed kept in the cache. We should cross reference
our memory report generated from the SurfaceCache against any surfaces
stored in our ImageContainer objects to ensure they are all reported.

This is of particular importance for blob recordings which are not put
into SurfaceCache. While the recordings themselves have their own memory
reporting inside of WebRender, it would be good to know what recordings
we are keeping alive from the content side to help break it down.

Differential Revision: https://phabricator.services.mozilla.com/D115517

gfx/2d/2D.h

@@ -454,6 +454,7 @@ class SourceSurface : public external::AtomicRefCounted<SourceSurface> {
       case SurfaceType::DATA_RECYCLING_SHARED:
       case SurfaceType::DATA_ALIGNED:
       case SurfaceType::DATA_SHARED_WRAPPER:
+      case SurfaceType::DATA_MAPPED:
         return true;
       default:
         return false;

gfx/2d/Logging.h

@@ -700,6 +700,9 @@ class Log final {
         case SurfaceType::DATA_SHARED_WRAPPER:
           mMessage << "SurfaceType::DATA_SHARED_WRAPPER";
           break;
+        case SurfaceType::DATA_MAPPED:
+          mMessage << "SurfaceType::DATA_MAPPED";
+          break;
         default:
           mMessage << "Invalid SurfaceType (" << (int)aType << ")";
           break;

gfx/2d/SourceSurfaceRawData.h

@@ -27,18 +27,22 @@ class SourceSurfaceMappedData final : public DataSourceSurface {
   uint8_t* GetData() final { return mMap.GetData(); }
   int32_t Stride() final { return mMap.GetStride(); }
 
-  SurfaceType GetType() const final { return SurfaceType::DATA; }
+  SurfaceType GetType() const final { return SurfaceType::DATA_MAPPED; }
   IntSize GetSize() const final { return mSize; }
   SurfaceFormat GetFormat() const final { return mFormat; }
 
   void SizeOfExcludingThis(MallocSizeOf aMallocSizeOf,
                            SizeOfInfo& aInfo) const override {
-    aInfo.AddType(SurfaceType::DATA);
+    aInfo.AddType(SurfaceType::DATA_MAPPED);
     mMap.GetSurface()->SizeOfExcludingThis(aMallocSizeOf, aInfo);
   }
 
   void GuaranteePersistance() final {}
 
+  const DataSourceSurface* GetScopedSurface() const {
+    return mMap.GetSurface();
+  }
+
  private:
   ScopedMap mMap;
   IntSize mSize;

gfx/2d/Types.h

@@ -44,6 +44,7 @@ enum class SurfaceType : int8_t {
   DATA_ALIGNED,           /* Data surface using aligned heap memory */
   DATA_SHARED_WRAPPER,    /* Shared memory mapped in from another process */
   BLOB_IMAGE,             /* Recorded blob image */
+  DATA_MAPPED,            /* Data surface wrapping a ScopedMap */
 };
 
 enum class SurfaceFormat : int8_t {

image/Image.cpp

@@ -12,6 +12,7 @@
 #include "nsContentUtils.h"
 #include "mozilla/gfx/Point.h"
 #include "mozilla/gfx/Rect.h"
+#include "mozilla/gfx/SourceSurfaceRawData.h"
 #include "mozilla/Services.h"
 #include "mozilla/SizeOfState.h"
 #include "mozilla/TimeStamp.h"
@@ -357,6 +358,78 @@ bool ImageResource::UpdateImageContainer(
   return !mImageContainers.IsEmpty();
 }
 
+void ImageResource::CollectSizeOfSurfaces(
+    nsTArray<SurfaceMemoryCounter>& aCounters,
+    MallocSizeOf aMallocSizeOf) const {
+  MOZ_ASSERT(NS_IsMainThread());
+
+  for (const auto& entry : mImageContainers) {
+    RefPtr<layers::ImageContainer> container(entry.mContainer);
+    if (!container) {
+      continue;
+    }
+
+    AutoTArray<layers::ImageContainer::OwningImage, 1> images;
+    container->GetCurrentImages(&images);
+    if (images.IsEmpty()) {
+      continue;
+    }
+
+    RefPtr<gfx::SourceSurface> surface = images[0].mImage->GetAsSourceSurface();
+    if (!surface) {
+      MOZ_ASSERT_UNREACHABLE("No surface in container!");
+      continue;
+    }
+
+    // The surface might be wrapping another.
+    bool isMappedSurface = surface->GetType() == gfx::SurfaceType::DATA_MAPPED;
+    const gfx::SourceSurface* actualSurface =
+        isMappedSurface
+            ? static_cast<gfx::SourceSurfaceMappedData*>(surface.get())
+                  ->GetScopedSurface()
+            : surface.get();
+
+    // Check if the surface is already in the report. Ignore if so.
+    bool found = false;
+    for (const auto& counter : aCounters) {
+      if (counter.Surface() == actualSurface) {
+        found = true;
+        break;
+      }
+    }
+    if (found) {
+      continue;
+    }
+
+    // The surface isn't in the report, so it isn't stored in SurfaceCache. We
+    // need to add our own entry here so that it will be included in the memory
+    // report.
+    gfx::SourceSurface::SizeOfInfo info;
+    surface->SizeOfExcludingThis(aMallocSizeOf, info);
+
+    uint32_t heapBytes = aMallocSizeOf(actualSurface);
+    if (isMappedSurface) {
+      heapBytes += aMallocSizeOf(surface.get());
+    }
+
+    SurfaceKey key = ContainerSurfaceKey(surface->GetSize(), entry.mSVGContext,
+                                         ToSurfaceFlags(entry.mFlags));
+    SurfaceMemoryCounter counter(key, actualSurface, /* aIsLocked */ false,
+                                 /* aCannotSubstitute */ false,
+                                 /* aIsFactor2 */ false, /* aFinished */ true,
+                                 SurfaceMemoryCounterType::CONTAINER);
+
+    counter.Values().SetDecodedHeap(info.mHeapBytes + heapBytes);
+    counter.Values().SetDecodedNonHeap(info.mNonHeapBytes);
+    counter.Values().SetDecodedUnknown(info.mUnknownBytes);
+    counter.Values().SetExternalHandles(info.mExternalHandles);
+    counter.Values().SetExternalId(info.mExternalId);
+    counter.Values().SetSurfaceTypes(info.mTypes);
+
+    aCounters.AppendElement(counter);
+  }
+}
+
 void ImageResource::ReleaseImageContainer() {
   MOZ_ASSERT(NS_IsMainThread());
   mImageContainers.Clear();

image/Image.h

@@ -6,6 +6,7 @@
 #ifndef mozilla_image_Image_h
 #define mozilla_image_Image_h
 
+#include "mozilla/Attributes.h"
 #include "mozilla/Maybe.h"
 #include "mozilla/MemoryReporting.h"
 #include "mozilla/ProfilerMarkers.h"
@@ -84,14 +85,15 @@ struct MemoryCounter {
   uint32_t mSurfaceTypes;
 };
 
-enum class SurfaceMemoryCounterType { NORMAL, COMPOSITING, COMPOSITING_PREV };
+enum class SurfaceMemoryCounterType { NORMAL, CONTAINER };
 
 struct SurfaceMemoryCounter {
   SurfaceMemoryCounter(
-      const SurfaceKey& aKey, bool aIsLocked, bool aCannotSubstitute,
-      bool aIsFactor2, bool aFinished,
+      const SurfaceKey& aKey, const gfx::SourceSurface* aSurface,
+      bool aIsLocked, bool aCannotSubstitute, bool aIsFactor2, bool aFinished,
       SurfaceMemoryCounterType aType = SurfaceMemoryCounterType::NORMAL)
       : mKey(aKey),
+        mSurface(aSurface),
         mType(aType),
         mIsLocked(aIsLocked),
         mCannotSubstitute(aCannotSubstitute),
@@ -99,6 +101,7 @@ struct SurfaceMemoryCounter {
         mFinished(aFinished) {}
 
   const SurfaceKey& Key() const { return mKey; }
+  const gfx::SourceSurface* Surface() const { return mSurface; }
   MemoryCounter& Values() { return mValues; }
   const MemoryCounter& Values() const { return mValues; }
   SurfaceMemoryCounterType Type() const { return mType; }
@@ -109,6 +112,7 @@ struct SurfaceMemoryCounter {
 
  private:
   const SurfaceKey mKey;
+  const gfx::SourceSurface* MOZ_NON_OWNING_REF mSurface;
   MemoryCounter mValues;
   const SurfaceMemoryCounterType mType;
   const bool mIsLocked;
@@ -319,6 +323,14 @@ class ImageResource : public Image {
    */
   nsIURI* GetURI() const override { return mURI; }
 
+  /*
+   * Should be called by its subclasses after they populate @aCounters so that
+   * we can cross reference against any of our ImageContainers that contain
+   * surfaces not in the cache.
+   */
+  void CollectSizeOfSurfaces(nsTArray<SurfaceMemoryCounter>& aCounters,
+                             MallocSizeOf aMallocSizeOf) const override;
+
  protected:
   explicit ImageResource(nsIURI* aURI);
   ~ImageResource();

image/RasterImage.cpp

@@ -709,6 +709,7 @@ void RasterImage::CollectSizeOfSurfaces(
     nsTArray<SurfaceMemoryCounter>& aCounters,
     MallocSizeOf aMallocSizeOf) const {
   SurfaceCache::CollectSizeOfSurfaces(ImageKey(this), aCounters, aMallocSizeOf);
+  ImageResource::CollectSizeOfSurfaces(aCounters, aMallocSizeOf);
 }
 
 bool RasterImage::SetMetadata(const ImageMetadata& aMetadata,

image/SourceSurfaceBlobImage.h

@@ -103,6 +103,7 @@ class SourceSurfaceBlobImage final : public gfx::SourceSurface {
   void SizeOfExcludingThis(MallocSizeOf aMallocSizeOf,
                            SizeOfInfo& aInfo) const override {
     aInfo.AddType(gfx::SurfaceType::BLOB_IMAGE);
+    aInfo.mHeapBytes += mKeys.ShallowSizeOfExcludingThis(aMallocSizeOf);
   }
 
  private:

image/SurfaceCache.cpp

@@ -193,10 +193,10 @@ class CachedSurface {
       // for surfaces with PlaybackType::eAnimated.)
       aCachedSurface->mProvider->AddSizeOfExcludingThis(
           mMallocSizeOf, [&](ISurfaceProvider::AddSizeOfCbData& aMetadata) {
-            SurfaceMemoryCounter counter(aCachedSurface->GetSurfaceKey(),
-                                         aCachedSurface->IsLocked(),
-                                         aCachedSurface->CannotSubstitute(),
-                                         aIsFactor2, aMetadata.mFinished);
+            SurfaceMemoryCounter counter(
+                aCachedSurface->GetSurfaceKey(), aMetadata.mSurface,
+                aCachedSurface->IsLocked(), aCachedSurface->CannotSubstitute(),
+                aIsFactor2, aMetadata.mFinished);
 
             counter.Values().SetDecodedHeap(aMetadata.mHeapBytes);
             counter.Values().SetDecodedNonHeap(aMetadata.mNonHeapBytes);

image/SurfaceCache.h

@@ -88,6 +88,9 @@ class SurfaceKey {
                                      PlaybackType);
   friend SurfaceKey VectorSurfaceKey(const IntSize&,
                                      const Maybe<SVGImageContext>&);
+  friend SurfaceKey ContainerSurfaceKey(
+      const gfx::IntSize& aSize, const Maybe<SVGImageContext>& aSVGContext,
+      SurfaceFlags aFlags);
 
   IntSize mSize;
   Maybe<SVGImageContext> mSVGContext;
@@ -112,6 +115,12 @@ inline SurfaceKey VectorSurfaceKey(const gfx::IntSize& aSize,
                     DefaultSurfaceFlags());
 }
 
+inline SurfaceKey ContainerSurfaceKey(const gfx::IntSize& aSize,
+                                      const Maybe<SVGImageContext>& aSVGContext,
+                                      SurfaceFlags aFlags) {
+  return SurfaceKey(aSize, aSVGContext, PlaybackType::eStatic, aFlags);
+}
+
 /**
  * AvailabilityState is used to track whether an ISurfaceProvider has a surface
  * available or is just a placeholder.

image/VectorImage.cpp

@@ -375,6 +375,7 @@ void VectorImage::CollectSizeOfSurfaces(
     nsTArray<SurfaceMemoryCounter>& aCounters,
     MallocSizeOf aMallocSizeOf) const {
   SurfaceCache::CollectSizeOfSurfaces(ImageKey(this), aCounters, aMallocSizeOf);
+  ImageResource::CollectSizeOfSurfaces(aCounters, aMallocSizeOf);
 }
 
 nsresult VectorImage::OnImageDataComplete(nsIRequest* aRequest,

image/imgFrame.cpp

@@ -933,8 +933,9 @@ void imgFrame::AddSizeOfExcludingThis(MallocSizeOf aMallocSizeOf,
   MonitorAutoLock lock(mMonitor);
 
   AddSizeOfCbData metadata;
-
+  metadata.mSurface = mOptSurface ? mOptSurface.get() : mRawSurface.get();
   metadata.mFinished = mFinished;
+
   if (mLockedSurface) {
     // The locked surface should only be present if we have mRawSurface. Hence
     // we only need to get its allocation size to avoid double counting.

image/imgFrame.h

@@ -187,6 +187,7 @@ class imgFrame {
     AddSizeOfCbData()
         : SourceSurface::SizeOfInfo(), mIndex(0), mFinished(false) {}
 
+    const gfx::SourceSurface* mSurface;
     size_t mIndex;
     bool mFinished;
   };

image/imgLoader.cpp

@@ -339,17 +339,28 @@ class imgMemoryReporter final : public nsIMemoryReporter {
       layers::SharedSurfacesMemoryReport& aSharedSurfaces) {
     for (const SurfaceMemoryCounter& counter : aCounter.Surfaces()) {
       nsAutoCString surfacePathPrefix(aPathPrefix);
-      if (counter.IsLocked()) {
-        surfacePathPrefix.AppendLiteral("locked/");
-      } else {
-        surfacePathPrefix.AppendLiteral("unlocked/");
-      }
-      if (counter.IsFactor2()) {
-        surfacePathPrefix.AppendLiteral("factor2/");
-      }
-      if (counter.CannotSubstitute()) {
-        surfacePathPrefix.AppendLiteral("cannot_substitute/");
+      switch (counter.Type()) {
+        case SurfaceMemoryCounterType::NORMAL:
+          if (counter.IsLocked()) {
+            surfacePathPrefix.AppendLiteral("locked/");
+          } else {
+            surfacePathPrefix.AppendLiteral("unlocked/");
+          }
+          if (counter.IsFactor2()) {
+            surfacePathPrefix.AppendLiteral("factor2/");
+          }
+          if (counter.CannotSubstitute()) {
+            surfacePathPrefix.AppendLiteral("cannot_substitute/");
+          }
+          break;
+        case SurfaceMemoryCounterType::CONTAINER:
+          surfacePathPrefix.AppendLiteral("container/");
+          break;
+        default:
+          MOZ_ASSERT_UNREACHABLE("Unknown counter type");
+          break;
       }
+
       surfacePathPrefix.AppendLiteral("types=");
       surfacePathPrefix.AppendInt(counter.Values().SurfaceTypes(), 16);
       surfacePathPrefix.AppendLiteral("/surface(");
@@ -370,70 +381,62 @@ class imgMemoryReporter final : public nsIMemoryReporter {
       ImageMemoryReporter::AppendSharedSurfacePrefix(surfacePathPrefix, counter,
                                                      aSharedSurfaces);
 
-      if (counter.Type() == SurfaceMemoryCounterType::NORMAL) {
-        PlaybackType playback = counter.Key().Playback();
-        if (playback == PlaybackType::eAnimated) {
-          if (StaticPrefs::image_mem_debug_reporting()) {
-            surfacePathPrefix.AppendPrintf(
-                " (animation %4u)", uint32_t(counter.Values().FrameIndex()));
-          } else {
-            surfacePathPrefix.AppendLiteral(" (animation)");
-          }
+      PlaybackType playback = counter.Key().Playback();
+      if (playback == PlaybackType::eAnimated) {
+        if (StaticPrefs::image_mem_debug_reporting()) {
+          surfacePathPrefix.AppendPrintf(
+              " (animation %4u)", uint32_t(counter.Values().FrameIndex()));
+        } else {
+          surfacePathPrefix.AppendLiteral(" (animation)");
         }
+      }
 
-        if (counter.Key().Flags() != DefaultSurfaceFlags()) {
-          surfacePathPrefix.AppendLiteral(", flags:");
-          surfacePathPrefix.AppendInt(uint32_t(counter.Key().Flags()),
-                                      /* aRadix = */ 16);
-        }
+      if (counter.Key().Flags() != DefaultSurfaceFlags()) {
+        surfacePathPrefix.AppendLiteral(", flags:");
+        surfacePathPrefix.AppendInt(uint32_t(counter.Key().Flags()),
+                                    /* aRadix = */ 16);
+      }
 
-        if (counter.Key().SVGContext()) {
-          const SVGImageContext& context = counter.Key().SVGContext().ref();
-          surfacePathPrefix.AppendLiteral(", svgContext:[ ");
-          if (context.GetViewportSize()) {
-            const CSSIntSize& size = context.GetViewportSize().ref();
-            surfacePathPrefix.AppendLiteral("viewport=(");
-            surfacePathPrefix.AppendInt(size.width);
-            surfacePathPrefix.AppendLiteral("x");
-            surfacePathPrefix.AppendInt(size.height);
-            surfacePathPrefix.AppendLiteral(") ");
-          }
-          if (context.GetPreserveAspectRatio()) {
-            nsAutoString aspect;
-            context.GetPreserveAspectRatio()->ToString(aspect);
-            surfacePathPrefix.AppendLiteral("preserveAspectRatio=(");
-            LossyAppendUTF16toASCII(aspect, surfacePathPrefix);
-            surfacePathPrefix.AppendLiteral(") ");
-          }
-          if (context.GetContextPaint()) {
-            const SVGEmbeddingContextPaint* paint = context.GetContextPaint();
-            surfacePathPrefix.AppendLiteral("contextPaint=(");
-            if (paint->GetFill()) {
-              surfacePathPrefix.AppendLiteral(" fill=");
-              surfacePathPrefix.AppendInt(paint->GetFill()->ToABGR(), 16);
-            }
-            if (paint->GetFillOpacity()) {
-              surfacePathPrefix.AppendLiteral(" fillOpa=");
-              surfacePathPrefix.AppendFloat(paint->GetFillOpacity());
-            }
-            if (paint->GetStroke()) {
-              surfacePathPrefix.AppendLiteral(" stroke=");
-              surfacePathPrefix.AppendInt(paint->GetStroke()->ToABGR(), 16);
-            }
-            if (paint->GetStrokeOpacity()) {
-              surfacePathPrefix.AppendLiteral(" strokeOpa=");
-              surfacePathPrefix.AppendFloat(paint->GetStrokeOpacity());
-            }
-            surfacePathPrefix.AppendLiteral(" ) ");
-          }
-          surfacePathPrefix.AppendLiteral("]");
+      if (counter.Key().SVGContext()) {
+        const SVGImageContext& context = counter.Key().SVGContext().ref();
+        surfacePathPrefix.AppendLiteral(", svgContext:[ ");
+        if (context.GetViewportSize()) {
+          const CSSIntSize& size = context.GetViewportSize().ref();
+          surfacePathPrefix.AppendLiteral("viewport=(");
+          surfacePathPrefix.AppendInt(size.width);
+          surfacePathPrefix.AppendLiteral("x");
+          surfacePathPrefix.AppendInt(size.height);
+          surfacePathPrefix.AppendLiteral(") ");
         }
-      } else if (counter.Type() == SurfaceMemoryCounterType::COMPOSITING) {
-        surfacePathPrefix.AppendLiteral(", compositing frame");
-      } else if (counter.Type() == SurfaceMemoryCounterType::COMPOSITING_PREV) {
-        surfacePathPrefix.AppendLiteral(", compositing prev frame");
-      } else {
-        MOZ_ASSERT_UNREACHABLE("Unknown counter type");
+        if (context.GetPreserveAspectRatio()) {
+          nsAutoString aspect;
+          context.GetPreserveAspectRatio()->ToString(aspect);
+          surfacePathPrefix.AppendLiteral("preserveAspectRatio=(");
+          LossyAppendUTF16toASCII(aspect, surfacePathPrefix);
+          surfacePathPrefix.AppendLiteral(") ");
+        }
+        if (context.GetContextPaint()) {
+          const SVGEmbeddingContextPaint* paint = context.GetContextPaint();
+          surfacePathPrefix.AppendLiteral("contextPaint=(");
+          if (paint->GetFill()) {
+            surfacePathPrefix.AppendLiteral(" fill=");
+            surfacePathPrefix.AppendInt(paint->GetFill()->ToABGR(), 16);
+          }
+          if (paint->GetFillOpacity() != 1.0) {
+            surfacePathPrefix.AppendLiteral(" fillOpa=");
+            surfacePathPrefix.AppendFloat(paint->GetFillOpacity());
+          }
+          if (paint->GetStroke()) {
+            surfacePathPrefix.AppendLiteral(" stroke=");
+            surfacePathPrefix.AppendInt(paint->GetStroke()->ToABGR(), 16);
+          }
+          if (paint->GetStrokeOpacity() != 1.0) {
+            surfacePathPrefix.AppendLiteral(" strokeOpa=");
+            surfacePathPrefix.AppendFloat(paint->GetStrokeOpacity());
+          }
+          surfacePathPrefix.AppendLiteral(" ) ");
+        }
+        surfacePathPrefix.AppendLiteral("]");
       }
 
       surfacePathPrefix.AppendLiteral(")/");