Bug 1863731 - IPC Fuzzing: Add MOZ_FUZZ_PROTOID_FILTER. r=truber

Differential Revision: https://phabricator.services.mozilla.com/D193064

tools/fuzzing/ipc/IPCFuzzController.cpp

@@ -158,6 +158,26 @@ void IPCFuzzController::OnActorConnected(IProtocol* protocol) {
     return;
   }
 
+  MOZ_FUZZING_NYX_DEBUG(
+      "DEBUG: IPCFuzzController::OnActorConnected() Mutex try\n");
+
+  // Called on background threads and modifies `actorIds`.
+  MutexAutoLock lock(mMutex);
+
+  MOZ_FUZZING_NYX_DEBUG(
+      "DEBUG: IPCFuzzController::OnActorConnected() Mutex locked\n");
+
+  static bool protoIdFilterInitialized = false;
+  static bool allowNewActors = false;
+  static std::string protoIdFilter;
+  if (!protoIdFilterInitialized) {
+    const char* protoIdFilterStr = getenv("MOZ_FUZZ_PROTOID_FILTER");
+    if (protoIdFilterStr) {
+      protoIdFilter = std::string(protoIdFilterStr);
+    }
+    protoIdFilterInitialized = true;
+  }
+
 #ifdef FUZZ_DEBUG
   MOZ_FUZZING_NYX_PRINTF("INFO: [OnActorConnected] ActorID %d Protocol: %s\n",
                          protocol->Id(), protocol->GetProtocolName());
@@ -167,20 +187,37 @@ void IPCFuzzController::OnActorConnected(IProtocol* protocol) {
 
   Maybe<PortName> portName = channel->GetPortName();
   if (portName) {
-    MOZ_FUZZING_NYX_DEBUG(
-        "DEBUG: IPCFuzzController::OnActorConnected() Mutex try\n");
-    // Called on background threads and modifies `actorIds`.
-    MutexAutoLock lock(mMutex);
-    MOZ_FUZZING_NYX_DEBUG(
-        "DEBUG: IPCFuzzController::OnActorConnected() Mutex locked\n");
+    if (!protoIdFilter.empty() &&
+        (!Nyx::instance().started() || !allowNewActors) &&
+        strcmp(protocol->GetProtocolName(), protoIdFilter.c_str()) &&
+        !actorIds[*portName].empty()) {
+      MOZ_FUZZING_NYX_PRINTF(
+          "INFO: [OnActorConnected] ActorID %d Protocol: %s ignored due to "
+          "filter.\n",
+          protocol->Id(), protocol->GetProtocolName());
+      return;
+    } else if (!protoIdFilter.empty() &&
+               !strcmp(protocol->GetProtocolName(), protoIdFilter.c_str())) {
+      MOZ_FUZZING_NYX_PRINTF(
+          "INFO: [OnActorConnected] ActorID %d Protocol: %s matches target.\n",
+          protocol->Id(), protocol->GetProtocolName());
+    } else if (!protoIdFilter.empty() && actorIds[*portName].empty()) {
+      MOZ_FUZZING_NYX_PRINTF(
+          "INFO: [OnActorConnected] ActorID %d Protocol: %s is toplevel "
+          "actor.\n",
+          protocol->Id(), protocol->GetProtocolName());
+    }
+
     actorIds[*portName].emplace_back(protocol->Id(), protocol->GetProtocolId());
 
-    // Fix the port we will be using for at least the next 5 messages
-    useLastPortName = true;
-    lastActorPortName = *portName;
+    if (Nyx::instance().started() && protoIdFilter.empty()) {
+      // Fix the port we will be using for at least the next 5 messages
+      useLastPortName = true;
+      lastActorPortName = *portName;
 
-    // Use this actor for the next 5 messages
-    useLastActor = 5;
+      // Use this actor for the next 5 messages
+      useLastActor = 5;
+    }
   } else {
     MOZ_FUZZING_NYX_DEBUG("WARNING: No port name on actor?!\n");
   }
@@ -261,6 +298,31 @@ bool IPCFuzzController::ObserveIPCMessage(mozilla::ipc::NodeChannel* channel,
   } else if (aMessage.type() == mIPCTriggerMsg) {
     MOZ_FUZZING_NYX_PRINT("DEBUG: Ready message detected.\n");
 
+    if (!haveTargetNodeName && !!getenv("MOZ_FUZZ_PROTOID_FILTER")) {
+      // With a protocol filter set, we want to pin to the actor that
+      // received the ready message and stay there. We should do this here
+      // because OnActorConnected can be called even after the ready message
+      // has been received and potentially override the correct actor.
+
+      // Get the port name associated with this message
+      Vector<char, 256, InfallibleAllocPolicy> footer;
+      if (!footer.initLengthUninitialized(aMessage.event_footer_size()) ||
+          !aMessage.ReadFooter(footer.begin(), footer.length(), false)) {
+        MOZ_FUZZING_NYX_ABORT("ERROR: Failed to read message footer.\n");
+      }
+
+      UniquePtr<Event> event =
+          Event::Deserialize(footer.begin(), footer.length());
+
+      if (!event || event->type() != Event::kUserMessage) {
+        MOZ_FUZZING_NYX_ABORT("ERROR: Trigger message is not kUserMessage?!\n");
+      }
+
+      lastActorPortName = event->port_name();
+      useLastPortName = true;
+      useLastActor = 1024;
+    }
+
     // TODO: This is specific to PContent fuzzing. If we later want to fuzz
     // a different process pair, we need additional signals here.
     OnChildReady();