Bug 1515521 - Add memory reporters for interning data. r=kvark

Differential Revision: https://phabricator.services.mozilla.com/D15044

--HG--
extra : moz-landing-system : lando

gfx/thebes/gfxPlatform.cpp

@@ -707,6 +707,8 @@ WebRenderMemoryReporter::CollectReports(nsIHandleReportCallback* aHandleReport,
         helper.Report(aReport.rasterized_blobs,
                       "resource-cache/rasterized-blobs");
         helper.Report(aReport.shader_cache, "shader-cache");
+        helper.Report(aReport.data_stores, "interning/data-stores");
+        helper.Report(aReport.interners, "interning/interners");
 
         // GPU Memory.
         helper.ReportTexture(aReport.gpu_cache_textures, "gpu-cache");

gfx/webrender_bindings/WebRenderAPI.cpp

@@ -30,6 +30,7 @@ namespace wr {
 using layers::Stringify;
 
 MOZ_DEFINE_MALLOC_SIZE_OF(WebRenderMallocSizeOf)
+MOZ_DEFINE_MALLOC_ENCLOSING_SIZE_OF(WebRenderMallocEnclosingSizeOf)
 
 class NewRenderer : public RendererEvent {
  public:
@@ -79,7 +80,8 @@ class NewRenderer : public RendererEvent {
                                          : nullptr,
             aRenderThread.Shaders() ? aRenderThread.Shaders()->RawShaders()
                                     : nullptr,
-            aRenderThread.ThreadPool().Raw(), &WebRenderMallocSizeOf,
+            aRenderThread.ThreadPool().Raw(),
+            &WebRenderMallocSizeOf, &WebRenderMallocEnclosingSizeOf,
             mDocHandle, &wrRenderer, mMaxTextureSize)) {
       // wr_window_new puts a message into gfxCriticalNote if it returns false
       return;

gfx/webrender_bindings/src/bindings.rs

@@ -996,6 +996,7 @@ pub extern "C" fn wr_window_new(window_id: WrWindowId,
                                 shaders: Option<&mut WrShaders>,
                                 thread_pool: *mut WrThreadPool,
                                 size_of_op: VoidPtrToSizeFn,
+                                enclosing_size_of_op: VoidPtrToSizeFn,
                                 out_handle: &mut *mut DocumentHandle,
                                 out_renderer: &mut *mut Renderer,
                                 out_max_texture_size: *mut i32)
@@ -1051,6 +1052,7 @@ pub extern "C" fn wr_window_new(window_id: WrWindowId,
         workers: Some(workers.clone()),
         thread_listener: Some(Box::new(GeckoProfilerThreadListener::new())),
         size_of_op: Some(size_of_op),
+        enclosing_size_of_op: Some(enclosing_size_of_op),
         cached_programs,
         resource_override_path: unsafe {
             let override_charptr = gfx_wr_resource_path_override();

gfx/webrender_bindings/webrender_ffi_generated.h

@@ -540,6 +540,8 @@ struct MemoryReport {
   uintptr_t images;
   uintptr_t rasterized_blobs;
   uintptr_t shader_cache;
+  uintptr_t data_stores;
+  uintptr_t interners;
   uintptr_t gpu_cache_textures;
   uintptr_t vertex_data_textures;
   uintptr_t render_target_textures;
@@ -558,6 +560,8 @@ struct MemoryReport {
            images == aOther.images &&
            rasterized_blobs == aOther.rasterized_blobs &&
            shader_cache == aOther.shader_cache &&
+           data_stores == aOther.data_stores &&
+           interners == aOther.interners &&
            gpu_cache_textures == aOther.gpu_cache_textures &&
            vertex_data_textures == aOther.vertex_data_textures &&
            render_target_textures == aOther.render_target_textures &&
@@ -1992,6 +1996,7 @@ bool wr_window_new(WrWindowId aWindowId,
                    WrShaders *aShaders,
                    WrThreadPool *aThreadPool,
                    VoidPtrToSizeFn aSizeOfOp,
+                   VoidPtrToSizeFn aEnclosingSizeOfOp,
                    DocumentHandle **aOutHandle,
                    Renderer **aOutRenderer,
                    int32_t *aOutMaxTextureSize)

gfx/wr/webrender/src/intern.rs

@@ -3,12 +3,14 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 use api::{LayoutPrimitiveInfo, LayoutRect};
+use api::VoidPtrToSizeFn;
 use internal_types::FastHashMap;
 use profiler::ResourceProfileCounter;
 use std::fmt::Debug;
 use std::hash::Hash;
 use std::marker::PhantomData;
 use std::{mem, ops, u64};
+use std::os::raw::c_void;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use util::VecHelper;
 
@@ -183,6 +185,11 @@ impl<S, T, M> DataStore<S, T, M> where S: Debug, T: From<S>, M: Debug
 
         debug_assert!(data_iter.next().is_none());
     }
+
+    /// Reports CPU heap usage.
+    pub fn malloc_size_of(&self, op: VoidPtrToSizeFn) -> usize {
+        unsafe { op(self.items.as_ptr() as *const c_void) }
+    }
 }
 
 /// Retrieve an item from the store via handle
@@ -375,6 +382,17 @@ where
 
         updates
     }
+
+    /// Reports CPU heap usage.
+    pub fn malloc_size_of(&self, op: VoidPtrToSizeFn, eop: VoidPtrToSizeFn) -> usize {
+        let mut bytes = 0;
+        unsafe {
+            bytes += op(self.local_data.as_ptr() as *const c_void);
+            bytes += self.map.values().next()
+                .map_or(0, |v| eop(v as *const _ as *const c_void));
+        }
+        bytes
+    }
 }
 
 /// Retrieve the local data for an item from the interner via handle

gfx/wr/webrender/src/render_backend.rs

@@ -269,6 +269,15 @@ impl FrameResources {
             }
         }
     }
+
+    /// Reports CPU heap usage.
+    fn report_memory(&self, op: VoidPtrToSizeFn, r: &mut MemoryReport) {
+        r.data_stores += self.clip_data_store.malloc_size_of(op);
+        r.data_stores += self.prim_data_store.malloc_size_of(op);
+        r.data_stores += self.linear_grad_data_store.malloc_size_of(op);
+        r.data_stores += self.radial_grad_data_store.malloc_size_of(op);
+        r.data_stores += self.text_run_data_store.malloc_size_of(op);
+    }
 }
 
 struct Document {
@@ -1005,7 +1014,7 @@ impl RenderBackend {
                 self.notifier.wake_up();
             }
             ApiMsg::ReportMemory(tx) => {
-                tx.send(self.report_memory()).unwrap();
+                self.report_memory(tx);
             }
             ApiMsg::DebugCommand(option) => {
                 let msg = match option {
@@ -1493,7 +1502,7 @@ impl RenderBackend {
         serde_json::to_string(&debug_root).unwrap()
     }
 
-    fn report_memory(&self) -> MemoryReport {
+    fn report_memory(&self, tx: ::api::channel::MsgSender<MemoryReport>) {
         let mut report = MemoryReport::default();
         let op = self.size_of_op.unwrap();
         report.gpu_cache_metadata = self.gpu_cache.malloc_size_of(op);
@@ -1503,11 +1512,16 @@ impl RenderBackend {
             }
             report.hit_testers +=
                 doc.hit_tester.as_ref().map_or(0, |ht| ht.malloc_size_of(op));
+
+            doc.resources.report_memory(op, &mut report)
         }
 
         report += self.resource_cache.report_memory(op);
 
-        report
+        // Send a message to report memory on the scene-builder thread, which
+        // will add its report to this one and send the result back to the original
+        // thread waiting on the request.
+        self.scene_tx.send(SceneBuilderRequest::ReportMemory(report, tx)).unwrap();
     }
 }
 

gfx/wr/webrender/src/renderer.rs

@@ -1577,9 +1577,18 @@ pub struct Renderer {
     /// copy the WR output to.
     output_image_handler: Option<Box<OutputImageHandler>>,
 
-    /// Optional function pointer for memory reporting.
+    /// Optional function pointer for measuring memory used by a given
+    /// heap-allocated pointer.
     size_of_op: Option<VoidPtrToSizeFn>,
 
+    /// Optional function pointer for measuring memory used by a given
+    /// heap-allocated region of memory. Unlike the above, pointers passed
+    /// to this function do not need to point to the start of the allocation,
+    /// and can be anywhere in the allocated region. This is useful for measuring
+    /// structures like hashmaps that don't expose pointers to the start of the
+    /// allocation, but do expose pointers to elements within the allocation.
+    _enclosing_size_of_op: Option<VoidPtrToSizeFn>,
+
     // Currently allocated FBOs for output frames.
     output_targets: FastHashMap<u32, FrameOutput>,
 
@@ -1870,6 +1879,7 @@ impl Renderer {
             });
         let sampler = options.sampler;
         let size_of_op = options.size_of_op;
+        let enclosing_size_of_op = options.enclosing_size_of_op;
         let namespace_alloc_by_client = options.namespace_alloc_by_client;
 
         let blob_image_handler = options.blob_image_handler.take();
@@ -1885,7 +1895,10 @@ impl Renderer {
         let (scene_builder, scene_tx, scene_rx) = SceneBuilder::new(
             config,
             api_tx.clone(),
-            scene_builder_hooks);
+            scene_builder_hooks,
+            size_of_op,
+            enclosing_size_of_op,
+        );
         thread::Builder::new().name(scene_thread_name.clone()).spawn(move || {
             register_thread_with_profiler(scene_thread_name.clone());
             if let Some(ref thread_listener) = *thread_listener_for_scene_builder {
@@ -2018,6 +2031,7 @@ impl Renderer {
             external_image_handler: None,
             output_image_handler: None,
             size_of_op: options.size_of_op,
+            _enclosing_size_of_op: options.enclosing_size_of_op,
             output_targets: FastHashMap::default(),
             cpu_profiles: VecDeque::new(),
             gpu_profiles: VecDeque::new(),
@@ -4773,6 +4787,7 @@ pub struct RendererOptions {
     pub recorder: Option<Box<ApiRecordingReceiver>>,
     pub thread_listener: Option<Box<ThreadListener + Send + Sync>>,
     pub size_of_op: Option<VoidPtrToSizeFn>,
+    pub enclosing_size_of_op: Option<VoidPtrToSizeFn>,
     pub cached_programs: Option<Rc<ProgramCache>>,
     pub debug_flags: DebugFlags,
     pub renderer_id: Option<u64>,
@@ -4810,6 +4825,7 @@ impl Default for RendererOptions {
             recorder: None,
             thread_listener: None,
             size_of_op: None,
+            enclosing_size_of_op: None,
             renderer_id: None,
             cached_programs: None,
             disable_dual_source_blending: false,

gfx/wr/webrender/src/scene_builder.rs

@@ -5,6 +5,7 @@
 use api::{AsyncBlobImageRasterizer, BlobImageRequest, BlobImageParams, BlobImageResult};
 use api::{DocumentId, PipelineId, ApiMsg, FrameMsg, ResourceUpdate, ExternalEvent, Epoch};
 use api::{BuiltDisplayList, ColorF, LayoutSize, NotificationRequest, Checkpoint, IdNamespace};
+use api::{MemoryReport, VoidPtrToSizeFn};
 use api::channel::MsgSender;
 #[cfg(feature = "capture")]
 use capture::CaptureConfig;
@@ -168,6 +169,7 @@ pub enum SceneBuilderRequest {
     SimulateLongSceneBuild(u32),
     SimulateLongLowPrioritySceneBuild(u32),
     Stop,
+    ReportMemory(MemoryReport, ::api::channel::MsgSender<MemoryReport>),
     #[cfg(feature = "capture")]
     SaveScene(CaptureConfig),
     #[cfg(feature = "replay")]
@@ -214,6 +216,22 @@ pub struct DocumentResources {
     pub yuv_image_interner: YuvImageDataInterner,
 }
 
+impl DocumentResources {
+    /// Reports CPU heap memory used by the interners.
+    fn report_memory(
+        &self,
+        op: VoidPtrToSizeFn,
+        eop: VoidPtrToSizeFn,
+        r: &mut MemoryReport,
+    ) {
+        r.interners += self.clip_interner.malloc_size_of(op, eop);
+        r.interners += self.prim_interner.malloc_size_of(op, eop);
+        r.interners += self.linear_grad_interner.malloc_size_of(op, eop);
+        r.interners += self.radial_grad_interner.malloc_size_of(op, eop);
+        r.interners += self.text_run_interner.malloc_size_of(op, eop);
+    }
+}
+
 // Access to `DocumentResources` interners by `Internable`
 pub trait InternerMut<I: Internable>
 {
@@ -275,6 +293,8 @@ pub struct SceneBuilder {
     config: FrameBuilderConfig,
     hooks: Option<Box<SceneBuilderHooks + Send>>,
     simulate_slow_ms: u32,
+    size_of_op: Option<VoidPtrToSizeFn>,
+    enclosing_size_of_op: Option<VoidPtrToSizeFn>,
 }
 
 impl SceneBuilder {
@@ -282,6 +302,8 @@ impl SceneBuilder {
         config: FrameBuilderConfig,
         api_tx: MsgSender<ApiMsg>,
         hooks: Option<Box<SceneBuilderHooks + Send>>,
+        size_of_op: Option<VoidPtrToSizeFn>,
+        enclosing_size_of_op: Option<VoidPtrToSizeFn>,
     ) -> (Self, Sender<SceneBuilderRequest>, Receiver<SceneBuilderResult>) {
         let (in_tx, in_rx) = channel();
         let (out_tx, out_rx) = channel();
@@ -293,6 +315,8 @@ impl SceneBuilder {
                 api_tx,
                 config,
                 hooks,
+                size_of_op,
+                enclosing_size_of_op,
                 simulate_slow_ms: 0,
             },
             in_tx,
@@ -352,6 +376,10 @@ impl SceneBuilder {
                     // get the Stop when the RenderBackend loop is exiting.
                     break;
                 }
+                Ok(SceneBuilderRequest::ReportMemory(mut report, tx)) => {
+                    report += self.report_memory();
+                    tx.send(report).unwrap();
+                }
                 Ok(SceneBuilderRequest::SimulateLongSceneBuild(time_ms)) => {
                     self.simulate_slow_ms = time_ms
                 }
@@ -733,6 +761,18 @@ impl SceneBuilder {
             }
         }
     }
+
+    /// Reports CPU heap memory used by the SceneBuilder.
+    fn report_memory(&self) -> MemoryReport {
+        let op = self.size_of_op.unwrap();
+        let eop = self.enclosing_size_of_op.unwrap();
+        let mut report = MemoryReport::default();
+        for doc in self.documents.values() {
+            doc.resources.report_memory(op, eop, &mut report);
+        }
+
+        report
+    }
 }
 
 /// A scene builder thread which executes expensive operations such as blob rasterization

gfx/wr/webrender_api/src/api.rs

@@ -848,6 +848,9 @@ pub struct MemoryReport {
     pub images: usize,
     pub rasterized_blobs: usize,
     pub shader_cache: usize,
+    pub data_stores: usize,
+    pub interners: usize,
+
     //
     // GPU memory.
     //
@@ -871,6 +874,8 @@ impl ::std::ops::AddAssign for MemoryReport {
         self.images += other.images;
         self.rasterized_blobs += other.rasterized_blobs;
         self.shader_cache += other.shader_cache;
+        self.data_stores += other.data_stores;
+        self.interners += other.interners;
         self.gpu_cache_textures += other.gpu_cache_textures;
         self.vertex_data_textures += other.vertex_data_textures;
         self.render_target_textures += other.render_target_textures;