diff --git a/kernel/exit.c b/kernel/exit.c
index 80efdfda6662..890e5cb6799b 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -432,6 +432,8 @@ retry:
 	 * Search through everything else, we should not get here often.
 	 */
 	for_each_process(g) {
+		if (atomic_read(&mm->mm_users) <= 1)
+			break;
 		if (g->flags & PF_KTHREAD)
 			continue;
 		for_each_thread(g, c) {