Bug 1714382 - Include the Mac memory pressure state in crash reports r=spohl,KrisWright

Subscribe to memory pressure events on macOS and add crash report annotations to parent and content process crash reports that can be used to determine if the system was under memory pressure at the time of the crash.

Include the memory pressure level reported via the DISPATCH_SOURCE_TYPE_MEMORYPRESSURE dispatch with timestamps of transitions, the memory pressure level as read from the kern.memorystatus_vm_pressure_level sysctl, and a measurement of the percentage of available memory in the system read from the kern.memorystatus_level sysctl.

Differential Revision: https://phabricator.services.mozilla.com/D116725

dom/ipc/ContentParent.cpp

@@ -257,6 +257,7 @@
 
 #if defined(XP_MACOSX)
 #  include "nsMacUtilsImpl.h"
+#  include "mozilla/AvailableMemoryWatcher.h"
 #endif
 
 #if defined(ANDROID) || defined(LINUX)
@@ -2060,6 +2061,12 @@ void ContentParent::ActorDestroy(ActorDestroyReason why) {
         // if mCreatedPairedMinidumps is true, we've already generated
         // parent/child dumps for desktop crashes.
         if (!mCreatedPairedMinidumps) {
+#if defined(XP_MACOSX)
+          RefPtr<nsAvailableMemoryWatcherBase> memWatcher;
+          memWatcher = nsAvailableMemoryWatcherBase::GetSingleton();
+          memWatcher->AddChildAnnotations(mCrashReporter);
+#endif
+
           if (mCrashReporter->GenerateCrashReport(OtherPid())) {
             // Propagate `isLikelyOOM`.
             Unused << props->SetPropertyAsBool(u"isLikelyOOM"_ns,

toolkit/crashreporter/CrashAnnotations.yaml

@@ -600,6 +600,47 @@ JSOutOfMemory:
     to satisfy the allocation.
   type: string
 
+MacMemoryPressure:
+  description: >
+    The current memory pressure state as provided by the macOS memory pressure
+    dispatch source. The annotation value is one of "Normal" for no memory
+    pressure, "Unset" indicating a memory pressure event has not been received,
+    "Warning" or "Critical" mapping to the system memory pressure levels,
+    or "Unexpected" for an unexpected level. This is a Mac-specific annotation.
+  type: string
+
+MacMemoryPressureNormalTime:
+  description: >
+    The time when the memory pressure state last transitioned to 'Normal'
+    expressed as seconds since the Epoch.
+  type: string
+
+MacMemoryPressureWarningTime:
+  description: >
+    The time when the memory pressure state last transitioned to 'Warning'
+    expressed as seconds since the Epoch.
+  type: string
+
+MacMemoryPressureCriticalTime:
+  description: >
+    The time when the memory pressure state last transitioned to 'Critical'
+    expressed as seconds since the Epoch.
+  type: string
+
+MacMemoryPressureSysctl:
+  description: >
+    The value of the memory pressure sysctl
+    'kern.memorystatus_vm_pressure_level'. Indicates which memory
+    pressure level the system is in at the time of the crash. The expected
+    values are one of 4 (Critical), 2 (Warning), or 0 (Normal).
+  type: integer
+
+MacAvailableMemorySysctl:
+  description: >
+    The value of the available memory sysctl 'kern.memorystatus_level'.
+    Expected to be a percentage integer value.
+  type: integer
+
 LauncherProcessState:
   description: >
     Launcher process enabled state. The integer value of this annotation must

widget/cocoa/nsAppShell.h

@@ -18,6 +18,7 @@
 
 namespace mozilla {
 class ProfilingStackOwner;
+class nsAvailableMemoryWatcherBase;
 }
 
 // GeckoNSApplication
@@ -47,12 +48,16 @@ class nsAppShell : public nsBaseAppShell {
   // public only to be visible to Objective-C code that must call it
   void WillTerminate();
 
+  static void OnMemoryPressureChanged(dispatch_source_memorypressure_flags_t aPressureLevel);
+
  protected:
   virtual ~nsAppShell();
 
   virtual void ScheduleNativeEventCallback() override;
   virtual bool ProcessNextNativeEvent(bool aMayWait) override;
 
+  void InitMemoryPressureObserver();
+
   static void ProcessGeckoEvents(void* aInfo);
 
  protected:
@@ -69,6 +74,9 @@ class nsAppShell : public nsBaseAppShell {
   // Non-null while the native event loop is in the waiting state.
   mozilla::ProfilingStackOwner* mProfilingStackOwnerWhileWaiting = nullptr;
 
+  dispatch_source_t mMemoryPressureSource = nullptr;
+  RefPtr<mozilla::nsAvailableMemoryWatcherBase> mAvailableMemoryWatcher;
+
   bool mRunningEventLoop;
   bool mStarted;
   bool mTerminated;

widget/cocoa/nsAppShell.mm

@@ -10,6 +10,7 @@
 
 #import <Cocoa/Cocoa.h>
 
+#include "mozilla/AvailableMemoryWatcher.h"
 #include "CustomCocoaEvents.h"
 #include "mozilla/WidgetTraceEvent.h"
 #include "nsAppShell.h"
@@ -20,6 +21,7 @@
 #include "nsString.h"
 #include "nsIRollupListener.h"
 #include "nsIWidget.h"
+#include "nsMemoryPressure.h"
 #include "nsThreadUtils.h"
 #include "nsServiceManagerUtils.h"
 #include "nsObjCExceptions.h"
@@ -227,6 +229,11 @@ nsAppShell::~nsAppShell() {
 
   hal::Shutdown();
 
+  if (mMemoryPressureSource) {
+    dispatch_release(mMemoryPressureSource);
+    mMemoryPressureSource = nullptr;
+  }
+
   if (mCFRunLoop) {
     if (mCFRunLoopSource) {
       ::CFRunLoopRemoveSource(mCFRunLoop, mCFRunLoopSource, kCFRunLoopCommonModes);
@@ -391,6 +398,8 @@ nsresult nsAppShell::Init() {
     } else {
       screenManager.SetHelper(mozilla::MakeUnique<ScreenHelperCocoa>());
     }
+
+    InitMemoryPressureObserver();
   }
 
   nsresult rv = nsBaseAppShell::Init();
@@ -849,6 +858,54 @@ nsAppShell::AfterProcessNextEvent(nsIThreadInternal* aThread, bool aEventWasProc
   NS_OBJC_END_TRY_BLOCK_RETURN(NS_ERROR_FAILURE);
 }
 
+void nsAppShell::InitMemoryPressureObserver() {
+  mAvailableMemoryWatcher = nsAvailableMemoryWatcherBase::GetSingleton();
+
+  // Testing shows that sometimes the memory pressure event is not fired for
+  // over a minute after the memory pressure change is reflected in sysctl
+  // values. Hence this may need to be augmented with polling of the memory
+  // pressure sysctls for lower latency reactions to OS memory pressure. This
+  // was also observed when using DISPATCH_QUEUE_PRIORITY_HIGH.
+  mMemoryPressureSource =
+      dispatch_source_create(DISPATCH_SOURCE_TYPE_MEMORYPRESSURE, 0,
+                             DISPATCH_MEMORYPRESSURE_NORMAL | DISPATCH_MEMORYPRESSURE_WARN |
+                                 DISPATCH_MEMORYPRESSURE_CRITICAL,
+                             dispatch_get_main_queue());
+
+  dispatch_source_set_event_handler(mMemoryPressureSource, ^{
+    dispatch_source_memorypressure_flags_t pressureLevel =
+        dispatch_source_get_data(mMemoryPressureSource);
+    nsAppShell::OnMemoryPressureChanged(pressureLevel);
+  });
+
+  dispatch_resume(mMemoryPressureSource);
+}
+
+void nsAppShell::OnMemoryPressureChanged(dispatch_source_memorypressure_flags_t aPressureLevel) {
+  // The memory pressure dispatch source is created (above) with
+  // dispatch_get_main_queue() which always fires on the main thread.
+  MOZ_ASSERT(NS_IsMainThread());
+
+  MacMemoryPressureLevel geckoPressureLevel;
+  switch (aPressureLevel) {
+    case DISPATCH_MEMORYPRESSURE_NORMAL:
+      geckoPressureLevel = MacMemoryPressureLevel::Normal;
+      break;
+    case DISPATCH_MEMORYPRESSURE_WARN:
+      geckoPressureLevel = MacMemoryPressureLevel::Warning;
+      break;
+    case DISPATCH_MEMORYPRESSURE_CRITICAL:
+      geckoPressureLevel = MacMemoryPressureLevel::Critical;
+      break;
+    default:
+      geckoPressureLevel = MacMemoryPressureLevel::Unexpected;
+  }
+
+  RefPtr<mozilla::nsAvailableMemoryWatcherBase> watcher(
+      nsAvailableMemoryWatcherBase::GetSingleton());
+  watcher->OnMemoryPressureChanged(geckoPressureLevel);
+}
+
 // AppShellDelegate implementation
 
 @implementation AppShellDelegate

xpcom/base/AvailableMemoryWatcher.cpp

@@ -112,7 +112,7 @@ void nsAvailableMemoryWatcherBase::RecordTelemetryEventOnHighMemory() {
 
 // Define the fallback method for a platform for which a platform-specific
 // CreateAvailableMemoryWatcher() is not defined.
-#if !defined(XP_WIN)
+#if !defined(XP_WIN) && !defined(XP_MACOSX)
 already_AddRefed<nsAvailableMemoryWatcherBase> CreateAvailableMemoryWatcher() {
   RefPtr instance(new nsAvailableMemoryWatcherBase);
   return do_AddRef(instance);

xpcom/base/AvailableMemoryWatcher.h

@@ -8,11 +8,24 @@
 #define mozilla_AvailableMemoryWatcher_h
 
 #include "mozilla/TimeStamp.h"
+#include "mozilla/ipc/CrashReporterHost.h"
+#include "mozilla/UniquePtr.h"
 #include "nsCOMPtr.h"
 #include "nsIAvailableMemoryWatcherBase.h"
 
 namespace mozilla {
 
+#if defined(XP_MACOSX)
+// An internal representation of the Mac memory-pressure level constants.
+enum class MacMemoryPressureLevel {
+  Unset,
+  Unexpected,
+  Normal,
+  Warning,
+  Critical,
+};
+#endif
+
 // This class implements a platform-independent part to watch the system's
 // memory situation and invoke the registered callbacks when we detect
 // a low-memory situation or a high-memory situation.
@@ -37,6 +50,12 @@ class nsAvailableMemoryWatcherBase : public nsIAvailableMemoryWatcherBase {
 
   nsAvailableMemoryWatcherBase();
 
+#if defined(XP_MACOSX)
+  virtual void OnMemoryPressureChanged(MacMemoryPressureLevel aLevel){};
+  virtual void AddChildAnnotations(
+      const UniquePtr<ipc::CrashReporterHost>& aCrashReporter){};
+#endif
+
   NS_DECL_THREADSAFE_ISUPPORTS
   NS_DECL_NSIAVAILABLEMEMORYWATCHERBASE
 };

xpcom/base/AvailableMemoryWatcherMac.cpp

@@ -0,0 +1,253 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include <sys/sysctl.h>
+#include <sys/types.h>
+#include <time.h>
+
+#include "AvailableMemoryWatcher.h"
+#include "Logging.h"
+#include "nsExceptionHandler.h"
+#include "nsICrashReporter.h"
+#include "nsISupports.h"
+#include "nsMemoryPressure.h"
+
+#define MP_LOG(...) MOZ_LOG(gMPLog, mozilla::LogLevel::Debug, (__VA_ARGS__))
+static mozilla::LazyLogModule gMPLog("MemoryPressure");
+
+namespace mozilla {
+
+class nsAvailableMemoryWatcher final : public nsAvailableMemoryWatcherBase {
+ public:
+  NS_DECL_ISUPPORTS_INHERITED
+
+  nsAvailableMemoryWatcher();
+  nsresult Init();
+
+  void OnMemoryPressureChanged(MacMemoryPressureLevel aLevel) override;
+  void AddChildAnnotations(
+      const UniquePtr<ipc::CrashReporterHost>& aCrashReporter) override;
+
+ private:
+  ~nsAvailableMemoryWatcher(){};
+
+  void InitParentAnnotations();
+  void UpdateParentAnnotations();
+
+  void AddParentAnnotation(CrashReporter::Annotation aAnnotation,
+                           nsAutoCString aString) {
+    CrashReporter::AnnotateCrashReport(aAnnotation, aString);
+  }
+  void AddParentAnnotation(CrashReporter::Annotation aAnnotation,
+                           uint32_t aData) {
+    CrashReporter::AnnotateCrashReport(aAnnotation, aData);
+  }
+
+  void ReadSysctls();
+
+  // Init has been called.
+  bool mInitialized;
+
+  // The memory pressure reported to the application by macOS.
+  MacMemoryPressureLevel mLevel;
+
+  // The value of the vm.memory_pressure sysctl. The OS notifies the
+  // application when the memory pressure level changes, but the sysctl
+  // value can be read at any time. Unofficially, the sysctl value
+  // corresponds to the OS memory pressure level with 4=>critical,
+  // 2=>warning, and 1=>normal.
+  uint32_t mLevelSysctl;
+
+  // The value of the kern.memorystatus_level sysctl. Unofficially,
+  // this is the percentage of available memory. (Also readable
+  // via the undocumented memorystatus_get_level syscall.)
+  int mAvailMemSysctl;
+
+  // The string representation of `mLevel`. i.e., normal, warning, or critical.
+  // Set to "unset" until a memory pressure change is reported to the process
+  // by the OS.
+  nsAutoCString mLevelStr;
+
+  // Timestamps for memory pressure level changes. Specifically, the Unix
+  // time in string form. Saved as Unix time to allow comparisons with
+  // the crash time.
+  nsAutoCString mNormalTimeStr;
+  nsAutoCString mWarningTimeStr;
+  nsAutoCString mCriticalTimeStr;
+};
+
+NS_IMPL_ISUPPORTS(nsAvailableMemoryWatcher, nsIAvailableMemoryWatcherBase);
+
+nsAvailableMemoryWatcher::nsAvailableMemoryWatcher()
+    : mInitialized(false),
+      mLevel(MacMemoryPressureLevel::Unset),
+      mLevelSysctl(0xFFFFFFFF),
+      mAvailMemSysctl(-1),
+      mLevelStr("Unset"),
+      mNormalTimeStr("Unset"),
+      mWarningTimeStr("Unset"),
+      mCriticalTimeStr("Unset") {}
+
+nsresult nsAvailableMemoryWatcher::Init() {
+  // Users of nsAvailableMemoryWatcher should use
+  // nsAvailableMemoryWatcherBase::GetSingleton() and not call Init directly.
+  MOZ_ASSERT(!mInitialized);
+  if (mInitialized) {
+    return NS_ERROR_ALREADY_INITIALIZED;
+  }
+
+  ReadSysctls();
+  MP_LOG("Initial memory pressure sysctl: %d", mLevelSysctl);
+  MP_LOG("Initial available memory sysctl: %d", mAvailMemSysctl);
+
+  // Set the initial state of all annotations for parent crash reports.
+  // Content process crash reports are set when a crash occurs and
+  // AddChildAnnotations() is called.
+  CrashReporter::AnnotateCrashReport(
+      CrashReporter::Annotation::MacMemoryPressure, mLevelStr);
+  CrashReporter::AnnotateCrashReport(
+      CrashReporter::Annotation::MacMemoryPressureNormalTime, mNormalTimeStr);
+  CrashReporter::AnnotateCrashReport(
+      CrashReporter::Annotation::MacMemoryPressureWarningTime, mWarningTimeStr);
+  CrashReporter::AnnotateCrashReport(
+      CrashReporter::Annotation::MacMemoryPressureCriticalTime,
+      mCriticalTimeStr);
+  CrashReporter::AnnotateCrashReport(
+      CrashReporter::Annotation::MacMemoryPressureSysctl, mLevelSysctl);
+  CrashReporter::AnnotateCrashReport(
+      CrashReporter::Annotation::MacAvailableMemorySysctl, mAvailMemSysctl);
+
+  mInitialized = true;
+  return NS_OK;
+}
+
+already_AddRefed<nsAvailableMemoryWatcherBase> CreateAvailableMemoryWatcher() {
+  // Users of nsAvailableMemoryWatcher should use
+  // nsAvailableMemoryWatcherBase::GetSingleton().
+  RefPtr watcher(new nsAvailableMemoryWatcher());
+  watcher->Init();
+  return watcher.forget();
+}
+
+const char* LevelToString(MacMemoryPressureLevel aLevel) {
+  switch (aLevel) {
+    case MacMemoryPressureLevel::Unset:
+      return "Unset";
+    case MacMemoryPressureLevel::Unexpected:
+      return "Unexpected";
+    case MacMemoryPressureLevel::Normal:
+      return "Normal";
+    case MacMemoryPressureLevel::Warning:
+      return "Warning";
+    case MacMemoryPressureLevel::Critical:
+      return "Critical";
+  }
+}
+
+// Update the memory pressure level, level change timestamps, and sysctl
+// level crash report annotations.
+void nsAvailableMemoryWatcher::UpdateParentAnnotations() {
+  // Generate a string representation of the current Unix time.
+  time_t timeChanged = time(NULL);
+  nsAutoCString timeChangedString;
+  timeChangedString =
+      nsPrintfCString("%" PRIu64, static_cast<uint64_t>(timeChanged));
+
+  nsAutoCString pressureLevelString;
+  Maybe<CrashReporter::Annotation> pressureLevelKey;
+
+  switch (mLevel) {
+    case MacMemoryPressureLevel::Normal:
+      mNormalTimeStr = timeChangedString;
+      pressureLevelString = "Normal";
+      pressureLevelKey.emplace(
+          CrashReporter::Annotation::MacMemoryPressureNormalTime);
+      break;
+    case MacMemoryPressureLevel::Warning:
+      mWarningTimeStr = timeChangedString;
+      pressureLevelString = "Warning";
+      pressureLevelKey.emplace(
+          CrashReporter::Annotation::MacMemoryPressureWarningTime);
+      break;
+    case MacMemoryPressureLevel::Critical:
+      mCriticalTimeStr = timeChangedString;
+      pressureLevelString = "Critical";
+      pressureLevelKey.emplace(
+          CrashReporter::Annotation::MacMemoryPressureCriticalTime);
+      break;
+    default:
+      pressureLevelString = "Unexpected";
+      break;
+  }
+
+  MP_LOG("Transitioning to %s at time %s", pressureLevelString.get(),
+         timeChangedString.get());
+
+  // Save the current memory pressure level.
+  AddParentAnnotation(CrashReporter::Annotation::MacMemoryPressure,
+                      pressureLevelString);
+
+  // Save the time we transitioned to the current memory pressure level.
+  if (pressureLevelKey.isSome()) {
+    AddParentAnnotation(pressureLevelKey.value(), timeChangedString);
+  }
+
+  AddParentAnnotation(CrashReporter::Annotation::MacMemoryPressureSysctl,
+                      mLevelSysctl);
+  AddParentAnnotation(CrashReporter::Annotation::MacAvailableMemorySysctl,
+                      mAvailMemSysctl);
+}
+
+void nsAvailableMemoryWatcher::ReadSysctls() {
+  // Pressure level
+  uint32_t level;
+  size_t size = sizeof(level);
+  if (sysctlbyname("kern.memorystatus_vm_pressure_level", &level, &size, NULL,
+                   0) == -1) {
+    MP_LOG("Failure reading memory pressure sysctl");
+  }
+  mLevelSysctl = level;
+
+  // Available memory percent
+  int availPercent;
+  size = sizeof(availPercent);
+  if (sysctlbyname("kern.memorystatus_level", &availPercent, &size, NULL,
+                   0) == -1) {
+    MP_LOG("Failure reading available memory level");
+  }
+  mAvailMemSysctl = availPercent;
+}
+
+/* virtual */
+void nsAvailableMemoryWatcher::OnMemoryPressureChanged(
+    MacMemoryPressureLevel aLevel) {
+  MOZ_ASSERT(mInitialized);
+  mLevel = aLevel;
+  ReadSysctls();
+  MP_LOG("level: %s, level sysctl: %d, available memory: %d percent",
+      LevelToString(aLevel), mLevelSysctl, mAvailMemSysctl);
+  UpdateParentAnnotations();
+}
+
+/* virtual */
+// Add all annotations to the provided crash reporter instance.
+void nsAvailableMemoryWatcher::AddChildAnnotations(
+    const UniquePtr<ipc::CrashReporterHost>& aCrashReporter) {
+  aCrashReporter->AddAnnotation(CrashReporter::Annotation::MacMemoryPressure,
+                                mLevelStr);
+  aCrashReporter->AddAnnotation(
+      CrashReporter::Annotation::MacMemoryPressureNormalTime, mNormalTimeStr);
+  aCrashReporter->AddAnnotation(
+      CrashReporter::Annotation::MacMemoryPressureWarningTime, mWarningTimeStr);
+  aCrashReporter->AddAnnotation(
+      CrashReporter::Annotation::MacMemoryPressureCriticalTime,
+      mCriticalTimeStr);
+  aCrashReporter->AddAnnotation(
+      CrashReporter::Annotation::MacMemoryPressureSysctl, mLevelSysctl);
+  aCrashReporter->AddAnnotation(
+      CrashReporter::Annotation::MacAvailableMemorySysctl, mAvailMemSysctl);
+}
+}  // namespace mozilla

xpcom/base/moz.build

@@ -107,6 +107,7 @@ EXPORTS.mozilla += [
     "AppShutdown.h",
     "AutoRestore.h",
     "AvailableMemoryTracker.h",
+    "AvailableMemoryWatcher.h",
     "ClearOnShutdown.h",
     "CountingAllocatorBase.h",
     "CycleCollectedJSContext.h",
@@ -207,6 +208,11 @@ if CONFIG["OS_TARGET"] == "WINNT":
         "MemoryInfo.cpp",
     ]
 
+if CONFIG["OS_TARGET"] == "Darwin":
+    UNIFIED_SOURCES += [
+        "AvailableMemoryWatcherMac.cpp",
+    ]
+
 GeneratedFile("ErrorList.h", script="ErrorList.py", entry_point="error_list_h")
 GeneratedFile(
     "ErrorNamesInternal.h", script="ErrorList.py", entry_point="error_names_internal_h"