diff --git a/lib/drb/drb.rb b/lib/drb/drb.rb
index a20ee1f143..6da8c5f6a7 100644
--- a/lib/drb/drb.rb
+++ b/lib/drb/drb.rb
@@ -1866,6 +1866,11 @@ module DRb
   # Removes +server+ from the list of registered servers.
   def remove_server(server)
     @server.delete(server.uri)
+    mutex.synchronize do
+      if @primary_server == server
+        @primary_server = nil
+      end
+    end
   end
   module_function :remove_server