Check whether object is valid in allocation_info_tracer_compact

peterzhu2118 · peterzhu2118 · commit 516a6cd1ad62 · 2024-12-16T12:24:24.000-05:00
When reference updating ObjectSpace.trace_object_allocations, we need to
check whether the object is valid or not because it does not mark the
object so the object may be dead. This can cause a segmentation fault
if the object is on a free heap page.

For example, the following script crashes:

    require "objspace"

    objs = []
    ObjectSpace.trace_object_allocations do
      1_000_000.times do
        objs &lt;&lt; Object.new
      end
    end

    objs = nil

    # Free pages that the objs were on
    GC.start

    # Run compaction and check that it doesn't crash
    GC.compact
diff --git a/ext/objspace/object_tracing.c b/ext/objspace/object_tracing.c
@@ -194,6 +194,10 @@ allocation_info_tracer_compact_update_object_table_i(st_data_t key, st_data_t va
 {
     st_table *table = (st_table *)data;
 
+    if (!rb_gc_pointer_to_heap_p(key)) {
+        return ST_DELETE;
+    }
+
     if (key != rb_gc_location(key)) {
         DURING_GC_COULD_MALLOC_REGION_START();
         {
diff --git a/gc.c b/gc.c
@@ -1724,6 +1724,12 @@ rb_objspace_garbage_object_p(VALUE obj)
     return rb_gc_impl_garbage_object_p(rb_gc_get_objspace(), obj);
 }
 
+bool
+rb_gc_pointer_to_heap_p(VALUE obj)
+{
+    return rb_gc_impl_pointer_to_heap_p(rb_gc_get_objspace(), (void *)obj);
+}
+
 /*
  *  call-seq:
  *     ObjectSpace._id2ref(object_id) -> an_object
diff --git a/internal/gc.h b/internal/gc.h
@@ -227,6 +227,7 @@ void rb_objspace_reachable_objects_from(VALUE obj, void (func)(VALUE, void *), v
 void rb_objspace_reachable_objects_from_root(void (func)(const char *category, VALUE, void *), void *data);
 int rb_objspace_internal_object_p(VALUE obj);
 int rb_objspace_garbage_object_p(VALUE obj);
+bool rb_gc_pointer_to_heap_p(VALUE obj);
 
 void rb_objspace_each_objects(
     int (*callback)(void *start, void *end, size_t stride, void *data),
diff --git a/test/objspace/test_objspace.rb b/test/objspace/test_objspace.rb
@@ -305,6 +305,29 @@ def test_trace_object_allocations_compaction
     RUBY
   end
 
+  def test_trace_object_allocations_compaction_freed_pages
+    omit "compaction is not supported on this platform" unless GC.respond_to?(:compact)
+
+    assert_normal_exit(<<~RUBY)
+      require "objspace"
+
+      objs = []
+      ObjectSpace.trace_object_allocations do
+        1_000_000.times do
+          objs << Object.new
+        end
+      end
+
+      objs = nil
+
+      # Free pages that the objs were on
+      GC.start
+
+      # Run compaction and check that it doesn't crash
+      GC.compact
+    RUBY
+  end
+
   def test_dump_flags
     # Ensure that the fstring is promoted to old generation
     4.times { GC.start }

Original file line number	Diff line number	Diff line change
`@@ -194,6 +194,10 @@ allocation_info_tracer_compact_update_object_table_i(st_data_t key, st_data_t va`
`194`	`194`	`{`
`195`	`195`	`st_table table = (st_table )data;`
`196`	`196`
	`197`	`+ if (!rb_gc_pointer_to_heap_p(key)) {`
	`198`	`+ return ST_DELETE;`
	`199`	`+ }`
	`200`	`+`
`197`	`201`	`if (key != rb_gc_location(key)) {`
`198`	`202`	`DURING_GC_COULD_MALLOC_REGION_START();`
`199`	`203`	`{`