Ensure that vm_stack is cleared in `thread_cleanup_func_before_exec`.

If `vm_stack` is left dangling in a forked process, the gc attempts to scan
it, but it is invalid and will cause a segfault. Therefore, we clear it
before forking.

In order to simplify this, `rb_ec_clear_vm_stack` was introduced.

cont.c

@@ -268,13 +268,6 @@ fiber_status_set(rb_fiber_t *fib, enum fiber_status s)
     fib->status = s;
 }
 
-void
-rb_ec_set_vm_stack(rb_execution_context_t *ec, VALUE *stack, size_t size)
-{
-    ec->vm_stack = stack;
-    ec->vm_stack_size = size;
-}
-
 static inline void
 ec_switch(rb_thread_t *th, rb_fiber_t *fib)
 {
@@ -705,7 +698,7 @@ cont_capture(volatile int *volatile stat)
     cont->saved_vm_stack.ptr = ALLOC_N(VALUE, ec->vm_stack_size);
     MEMCPY(cont->saved_vm_stack.ptr, ec->vm_stack, VALUE, ec->vm_stack_size);
 #endif
-    rb_ec_set_vm_stack(&cont->saved_ec, NULL, 0);
+    rb_ec_clear_vm_stack(&cont->saved_ec);
     cont_save_machine_stack(th, cont);
 
     /* backup ensure_list to array for search in another context */
@@ -1793,7 +1786,7 @@ rb_fiber_close(rb_fiber_t *fib)
         }
     }
 
-    rb_ec_set_vm_stack(ec, NULL, 0);
+    rb_ec_clear_vm_stack(ec);
 
 #if !FIBER_USE_NATIVE
     /* should not mark machine stack any more */

thread.c

@@ -596,7 +596,11 @@ thread_cleanup_func_before_exec(void *th_ptr)
 {
     rb_thread_t *th = th_ptr;
     th->status = THREAD_KILLED;
+    // The thread stack doesn't exist in the forked process:
     th->ec->machine.stack_start = th->ec->machine.stack_end = NULL;
+
+    // The vm_stack is `alloca`ed on the thread stack, so it's gone too:
+    rb_ec_clear_vm_stack(th->ec);
 }
 
 static void
@@ -713,19 +717,19 @@ thread_start_func_2(rb_thread_t *th, VALUE *stack_start)
         rb_bug("thread_start_func_2 must not be used for main thread");
     }
 
+    thread_debug("thread start: %p\n", (void *)th);
+    VM_ASSERT((size * sizeof(VALUE)) <= th->ec->machine.stack_maxsize);
+
     vm_stack = alloca(size * sizeof(VALUE));
     VM_ASSERT(vm_stack);
 
     gvl_acquire(th->vm, th);
 
     rb_ec_initialize_vm_stack(th->ec, vm_stack, size);
-
-    ruby_thread_set_native(th);
-
     th->ec->machine.stack_start = STACK_DIR_UPPER(vm_stack + size, vm_stack);
     th->ec->machine.stack_maxsize -= size * sizeof(VALUE);
 
-    thread_debug("thread start: %p\n", (void *)th);
+    ruby_thread_set_native(th);
 
     {
 	thread_debug("thread start (get lock): %p\n", (void *)th);
@@ -806,7 +810,10 @@ thread_start_func_2(rb_thread_t *th, VALUE *stack_start)
 
         rb_fiber_close(th->ec->fiber_ptr);
     }
+
     thread_cleanup_func(th, FALSE);
+    VM_ASSERT(th->ec->vm_stack == NULL);
+
     gvl_release(th->vm);
 
     return 0;
@@ -2253,6 +2260,7 @@ rb_threadptr_execute_interrupts(rb_thread_t *th, int blocking_timing)
 	    if (th->status == THREAD_RUNNABLE)
 		th->running_time_us += TIME_QUANTUM_USEC;
 
+            VM_ASSERT(th->ec->cfp);
 	    EXEC_EVENT_HOOK(th->ec, RUBY_INTERNAL_EVENT_SWITCH, th->ec->cfp->self,
 			    0, 0, 0, Qundef);
 

vm.c

@@ -2685,20 +2685,36 @@ thread_alloc(VALUE klass)
     return obj;
 }
 
+inline void
+rb_ec_set_vm_stack(rb_execution_context_t *ec, VALUE *stack, size_t size)
+{
+    ec->vm_stack = stack;
+    ec->vm_stack_size = size;
+}
+
 void
 rb_ec_initialize_vm_stack(rb_execution_context_t *ec, VALUE *stack, size_t size)
 {
-  rb_ec_set_vm_stack(ec, stack, size);
+    rb_ec_set_vm_stack(ec, stack, size);
 
-  ec->cfp = (void *)(ec->vm_stack + ec->vm_stack_size);
+    ec->cfp = (void *)(ec->vm_stack + ec->vm_stack_size);
 
-  rb_vm_push_frame(ec,
-      NULL /* dummy iseq */,
-      VM_FRAME_MAGIC_DUMMY | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH | VM_FRAME_FLAG_CFRAME /* dummy frame */,
-      Qnil /* dummy self */, VM_BLOCK_HANDLER_NONE /* dummy block ptr */,
-      0 /* dummy cref/me */,
-      0 /* dummy pc */, ec->vm_stack, 0, 0
-  );
+    rb_vm_push_frame(ec,
+        NULL /* dummy iseq */,
+        VM_FRAME_MAGIC_DUMMY | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH | VM_FRAME_FLAG_CFRAME /* dummy frame */,
+        Qnil /* dummy self */, VM_BLOCK_HANDLER_NONE /* dummy block ptr */,
+        0 /* dummy cref/me */,
+        0 /* dummy pc */, ec->vm_stack, 0, 0
+    );
+}
+
+void
+rb_ec_clear_vm_stack(rb_execution_context_t *ec)
+{
+    rb_ec_set_vm_stack(ec, NULL, 0);
+
+    // Avoid dangling pointers:
+    // ec->cfp = NULL;
 }
 
 static void

vm_core.h

@@ -907,6 +907,10 @@ void rb_ec_set_vm_stack(rb_execution_context_t *ec, VALUE *stack, size_t size);
 // @param size the size of the stack, as in `VALUE stack[size]`.
 void rb_ec_initialize_vm_stack(rb_execution_context_t *ec, VALUE *stack, size_t size);
 
+// Clear (set to `NULL`) the vm_stack pointer.
+// @param ec the execution context to update.
+void rb_ec_clear_vm_stack(rb_execution_context_t *ec);
+
 typedef struct rb_thread_struct {
     struct list_node vmlt_node;
     VALUE self;