From b7237e3bbd36e7c520c4cbaf1f866b6dcc265a99 Mon Sep 17 00:00:00 2001
From: Peter Zhu <peter@peterzhu.ca>
Date: Mon, 21 Aug 2023 14:13:24 -0400
Subject: [PATCH] Free all empty heap pages in Process.warmup

This commit adds `free_empty_pages` which frees all empty heap pages and
moves the number of pages freed to the allocatable pages counter. This
is used in Process.warmup to improve performance because page
invalidation from copy-on-write is slower than allocating a new page.
---
 gc.c                      | 44 +++++++++++++++++++++++++++++++++++++++
 process.c                 |  6 ++++--
 test/ruby/test_process.rb | 22 ++++++++++++++++++++
 3 files changed, 70 insertions(+), 2 deletions(-)

diff --git a/gc.c b/gc.c
index 6c4bffa95d..34334f3278 100644
--- a/gc.c
+++ b/gc.c
@@ -9647,11 +9647,55 @@ gc_start_internal(rb_execution_context_t *ec, VALUE self, VALUE full_mark, VALUE
     return Qnil;
 }
 
+static void
+free_empty_pages(void)
+{
+    rb_objspace_t *objspace = &rb_objspace;
+
+    for (int i = 0; i < SIZE_POOL_COUNT; i++) {
+        /* Move all empty pages to the tomb heap for freeing. */
+        rb_size_pool_t *size_pool = &size_pools[i];
+        rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool);
+        rb_heap_t *tomb_heap = SIZE_POOL_TOMB_HEAP(size_pool);
+
+        size_t freed_pages = 0;
+
+        struct heap_page **next_page_ptr = &heap->free_pages;
+        struct heap_page *page = heap->free_pages;
+        while (page) {
+            /* All finalizers should have been ran in gc_start_internal, so there
+            * should be no objects that require finalization. */
+            GC_ASSERT(page->final_slots == 0);
+
+            struct heap_page *next_page = page->free_next;
+
+            if (page->free_slots == page->total_slots) {
+                heap_unlink_page(objspace, heap, page);
+                heap_add_page(objspace, size_pool, tomb_heap, page);
+                freed_pages++;
+            }
+            else {
+                *next_page_ptr = page;
+                next_page_ptr = &page->free_next;
+            }
+
+            page = next_page;
+        }
+
+        *next_page_ptr = NULL;
+
+        size_pool_allocatable_pages_set(objspace, size_pool, size_pool->allocatable_pages + freed_pages);
+    }
+
+    heap_pages_free_unused_pages(objspace);
+}
+
 void
 rb_gc_prepare_heap(void)
 {
     rb_objspace_each_objects(gc_set_candidate_object_i, NULL);
     gc_start_internal(NULL, Qtrue, Qtrue, Qtrue, Qtrue, Qtrue);
+    free_empty_pages();
 }
 
 static int
diff --git a/process.c b/process.c
index fa2ae7344a..37dc524415 100644
--- a/process.c
+++ b/process.c
@@ -8671,10 +8671,12 @@ static VALUE rb_mProcID_Syscall;
  *
  *  On CRuby, +Process.warmup+:
  *
- *  * Perform a major GC.
+ *  * Performs a major GC.
  *  * Compacts the heap.
  *  * Promotes all surviving objects to the old generation.
- *  * Precompute the coderange of all strings.
+ *  * Precomputes the coderange of all strings.
+ *  * Frees all empty heap pages and increments the allocatable pages counter
+ *    by the number of pages freed.
  */
 
 static VALUE
diff --git a/test/ruby/test_process.rb b/test/ruby/test_process.rb
index 38dcb8054f..095ab27f5d 100644
--- a/test/ruby/test_process.rb
+++ b/test/ruby/test_process.rb
@@ -2725,4 +2725,26 @@ EOS
       assert_include(ObjectSpace.dump(obj), '"coderange":"7bit"')
     end;
   end
+
+  def test_warmup_frees_pages
+    assert_separately([{"RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO" => "1.0"}, "-W0"], "#{<<~"begin;"}\n#{<<~'end;'}")
+    begin;
+      TIMES = 10_000
+      ary = Array.new(TIMES)
+      TIMES.times do |i|
+        ary[i] = Object.new
+      end
+      ary.clear
+      ary = nil
+
+      total_pages_before = GC.stat(:heap_eden_pages) + GC.stat(:heap_allocatable_pages)
+
+      Process.warmup
+
+      # Number of pages freed should cause equal increase in number of allocatable pages.
+      assert_equal(total_pages_before, GC.stat(:heap_eden_pages) + GC.stat(:heap_allocatable_pages))
+      assert_equal(0, GC.stat(:heap_tomb_pages))
+      assert_operator(GC.stat(:total_freed_pages), :>, 0)
+    end;
+  end
 end