diff options
author | Peter Zhu <[email protected]> | 2023-06-13 16:19:33 -0400 |
---|---|---|
committer | Peter Zhu <[email protected]> | 2023-07-13 09:27:33 -0400 |
commit | 1e7b67f73370056c7521306b9f037b6cbccf640d (patch) | |
tree | 288d284f69cea8935adf323da99f84eff2bb7707 | |
parent | fff4773085bd70bf14ace146388dd0b9d80f244a (diff) |
[Feature #19730] Remove transient heap
Notes
Notes:
Merged: https://github.com/ruby/ruby/pull/7942
-rw-r--r-- | .github/workflows/compilers.yml | 1 | ||||
-rw-r--r-- | array.c | 136 | ||||
-rw-r--r-- | common.mk | 212 | ||||
-rw-r--r-- | debug_counter.h | 9 | ||||
-rw-r--r-- | gc.c | 45 | ||||
-rw-r--r-- | include/ruby/internal/core/rarray.h | 88 | ||||
-rw-r--r-- | inits.c | 3 | ||||
-rw-r--r-- | internal/array.h | 18 | ||||
-rw-r--r-- | internal/struct.h | 29 | ||||
-rw-r--r-- | internal/variable.h | 29 | ||||
-rw-r--r-- | ractor.c | 12 | ||||
-rw-r--r-- | struct.c | 34 | ||||
-rw-r--r-- | transient_heap.c | 976 | ||||
-rw-r--r-- | transient_heap.h | 63 | ||||
-rw-r--r-- | variable.c | 61 | ||||
-rw-r--r-- | yjit/src/cruby_bindings.inc.rs | 1 |
16 files changed, 16 insertions, 1701 deletions
diff --git a/.github/workflows/compilers.yml b/.github/workflows/compilers.yml index 56cdaf2bb0..e3d68b018f 100644 --- a/.github/workflows/compilers.yml +++ b/.github/workflows/compilers.yml @@ -178,7 +178,6 @@ jobs: # - { name: USE_LAZY_LOAD, env: { cppflags: '-DUSE_LAZY_LOAD' } } # - { name: USE_SYMBOL_GC=0, env: { cppflags: '-DUSE_SYMBOL_GC=0' } } # - { name: USE_THREAD_CACHE=0, env: { cppflags: '-DUSE_THREAD_CACHE=0' } } -# - { name: USE_TRANSIENT_HEAP=0, env: { cppflags: '-DUSE_TRANSIENT_HEAP=0' } } - { name: USE_RUBY_DEBUG_LOG=1, env: { cppflags: '-DUSE_RUBY_DEBUG_LOG=1' } } # - { name: USE_DEBUG_COUNTER, env: { cppflags: '-DUSE_DEBUG_COUNTER=1', RUBY_DEBUG_COUNTER_DISABLE: '1' } } @@ -28,7 +28,6 @@ #include "ruby/encoding.h" #include "ruby/st.h" #include "ruby/util.h" -#include "transient_heap.h" #include "builtin.h" #if !ARRAY_DEBUG @@ -97,7 +96,6 @@ should_be_T_ARRAY(VALUE ary) #define FL_SET_EMBED(a) do { \ assert(!ARY_SHARED_P(a)); \ FL_SET((a), RARRAY_EMBED_FLAG); \ - RARY_TRANSIENT_UNSET(a); \ ary_verify(a); \ } while (0) @@ -166,7 +164,6 @@ should_be_T_ARRAY(VALUE ary) #define FL_SET_SHARED_ROOT(ary) do { \ assert(!OBJ_FROZEN(ary)); \ assert(!ARY_EMBED_P(ary)); \ - assert(!RARRAY_TRANSIENT_P(ary)); \ FL_SET((ary), RARRAY_SHARED_ROOT_FLAG); \ } while (0) @@ -250,7 +247,6 @@ ary_verify_(VALUE ary, const char *file, int line) ary_verify(root); } else if (ARY_EMBED_P(ary)) { - assert(!RARRAY_TRANSIENT_P(ary)); assert(!ARY_SHARED_P(ary)); assert(RARRAY_LEN(ary) <= ary_embed_capa(ary)); } @@ -267,14 +263,6 @@ ary_verify_(VALUE ary, const char *file, int line) #endif } -#if USE_TRANSIENT_HEAP - if (RARRAY_TRANSIENT_P(ary)) { - assert(rb_transient_heap_managed_ptr_p(RARRAY_CONST_PTR_TRANSIENT(ary))); - } -#endif - - rb_transient_heap_verify(); - return ary; } @@ -367,131 +355,37 @@ ary_memcpy(VALUE ary, long beg, long argc, const VALUE *argv) static VALUE * ary_heap_alloc(VALUE ary, size_t capa) { - VALUE *ptr = rb_transient_heap_alloc(ary, sizeof(VALUE) * capa); - - if (ptr != NULL) { - RARY_TRANSIENT_SET(ary); - } - else { - RARY_TRANSIENT_UNSET(ary); - ptr = ALLOC_N(VALUE, capa); - } - - return ptr; + return ALLOC_N(VALUE, capa); } static void ary_heap_free_ptr(VALUE ary, const VALUE *ptr, long size) { - if (RARRAY_TRANSIENT_P(ary)) { - /* ignore it */ - } - else { - ruby_sized_xfree((void *)ptr, size); - } + ruby_sized_xfree((void *)ptr, size); } static void ary_heap_free(VALUE ary) { - if (RARRAY_TRANSIENT_P(ary)) { - RARY_TRANSIENT_UNSET(ary); - } - else { - ary_heap_free_ptr(ary, ARY_HEAP_PTR(ary), ARY_HEAP_SIZE(ary)); - } + ary_heap_free_ptr(ary, ARY_HEAP_PTR(ary), ARY_HEAP_SIZE(ary)); } static size_t ary_heap_realloc(VALUE ary, size_t new_capa) { - size_t alloc_capa = new_capa; - size_t old_capa = ARY_HEAP_CAPA(ary); - - if (RARRAY_TRANSIENT_P(ary)) { - if (new_capa <= old_capa) { - /* do nothing */ - alloc_capa = old_capa; - } - else { - VALUE *new_ptr = rb_transient_heap_alloc(ary, sizeof(VALUE) * new_capa); - - if (new_ptr == NULL) { - new_ptr = ALLOC_N(VALUE, new_capa); - RARY_TRANSIENT_UNSET(ary); - } - - MEMCPY(new_ptr, ARY_HEAP_PTR(ary), VALUE, old_capa); - ARY_SET_PTR(ary, new_ptr); - } - } - else { - SIZED_REALLOC_N(RARRAY(ary)->as.heap.ptr, VALUE, new_capa, old_capa); - } - ary_verify(ary); - - return alloc_capa; -} - -#if USE_TRANSIENT_HEAP -static inline void -rb_ary_transient_heap_evacuate_(VALUE ary, int transient, int promote) -{ - if (transient) { - assert(!ARY_SHARED_ROOT_P(ary)); - - VALUE *new_ptr; - const VALUE *old_ptr = ARY_HEAP_PTR(ary); - long capa = ARY_HEAP_CAPA(ary); - - assert(ARY_OWNS_HEAP_P(ary)); - assert(RARRAY_TRANSIENT_P(ary)); - assert(!ARY_PTR_USING_P(ary)); - - if (promote) { - new_ptr = ALLOC_N(VALUE, capa); - RARY_TRANSIENT_UNSET(ary); - } - else { - new_ptr = ary_heap_alloc(ary, capa); - } - - MEMCPY(new_ptr, old_ptr, VALUE, capa); - /* do not use ARY_SET_PTR() because they assert !frozen */ - RARRAY(ary)->as.heap.ptr = new_ptr; - } - + SIZED_REALLOC_N(RARRAY(ary)->as.heap.ptr, VALUE, new_capa, ARY_HEAP_CAPA(ary)); ary_verify(ary); -} -void -rb_ary_transient_heap_evacuate(VALUE ary, int promote) -{ - rb_ary_transient_heap_evacuate_(ary, RARRAY_TRANSIENT_P(ary), promote); + return new_capa; } void -rb_ary_detransient(VALUE ary) -{ - assert(RARRAY_TRANSIENT_P(ary)); - rb_ary_transient_heap_evacuate_(ary, TRUE, TRUE); -} -#else -void -rb_ary_detransient(VALUE ary) -{ - /* do nothing */ -} -#endif - -void rb_ary_make_embedded(VALUE ary) { assert(rb_ary_embeddable_p(ary)); if (!ARY_EMBED_P(ary)) { const VALUE *buf = ARY_HEAP_PTR(ary); long len = ARY_HEAP_LEN(ary); - bool was_transient = RARRAY_TRANSIENT_P(ary); // FL_SET_EMBED also unsets the transient flag FL_SET_EMBED(ary); @@ -499,9 +393,7 @@ rb_ary_make_embedded(VALUE ary) MEMCPY((void *)ARY_EMBED_PTR(ary), (void *)buf, VALUE, len); - if (!was_transient) { - ary_heap_free_ptr(ary, buf, len * sizeof(VALUE)); - } + ary_heap_free_ptr(ary, buf, len * sizeof(VALUE)); } } @@ -957,7 +849,6 @@ VALUE rb_ary_hidden_new(long capa) { VALUE ary = ary_new(0, capa); - rb_ary_transient_heap_evacuate(ary, TRUE); return ary; } @@ -980,13 +871,8 @@ rb_ary_free(VALUE ary) RB_DEBUG_COUNTER_INC(obj_ary_extracapa); } - if (RARRAY_TRANSIENT_P(ary)) { - RB_DEBUG_COUNTER_INC(obj_ary_transient); - } - else { - RB_DEBUG_COUNTER_INC(obj_ary_ptr); - ary_heap_free(ary); - } + RB_DEBUG_COUNTER_INC(obj_ary_ptr); + ary_heap_free(ary); } else { RB_DEBUG_COUNTER_INC(obj_ary_embed); @@ -1024,14 +910,11 @@ ary_make_shared(VALUE ary) } else if (OBJ_FROZEN(ary)) { if (!ARY_EMBED_P(ary)) { - rb_ary_transient_heap_evacuate(ary, TRUE); ary_shrink_capa(ary); } return ary; } else { - rb_ary_transient_heap_evacuate(ary, TRUE); - long capa = ARY_CAPA(ary); long len = RARRAY_LEN(ary); @@ -2353,9 +2236,8 @@ rb_ary_resize(VALUE ary, long len) else if (len <= ary_embed_capa(ary)) { const VALUE *ptr = ARY_HEAP_PTR(ary); long ptr_capa = ARY_HEAP_SIZE(ary); - bool is_malloc_ptr = !ARY_SHARED_P(ary) && !RARRAY_TRANSIENT_P(ary); + bool is_malloc_ptr = !ARY_SHARED_P(ary); - FL_UNSET(ary, RARRAY_TRANSIENT_FLAG); FL_SET_EMBED(ary); MEMCPY((VALUE *)ARY_EMBED_PTR(ary), ptr, VALUE, len); /* WB: no new reference */ @@ -198,7 +198,6 @@ COMMONOBJS = array.$(OBJEXT) \ thread.$(OBJEXT) \ time.$(OBJEXT) \ transcode.$(OBJEXT) \ - transient_heap.$(OBJEXT) \ util.$(OBJEXT) \ variable.$(OBJEXT) \ version.$(OBJEXT) \ @@ -2125,7 +2124,6 @@ array.$(OBJEXT): {$(VPATH)}st.h array.$(OBJEXT): {$(VPATH)}subst.h array.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h array.$(OBJEXT): {$(VPATH)}thread_native.h -array.$(OBJEXT): {$(VPATH)}transient_heap.h array.$(OBJEXT): {$(VPATH)}util.h array.$(OBJEXT): {$(VPATH)}vm_core.h array.$(OBJEXT): {$(VPATH)}vm_opts.h @@ -7148,7 +7146,6 @@ gc.$(OBJEXT): {$(VPATH)}symbol.h gc.$(OBJEXT): {$(VPATH)}thread.h gc.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h gc.$(OBJEXT): {$(VPATH)}thread_native.h -gc.$(OBJEXT): {$(VPATH)}transient_heap.h gc.$(OBJEXT): {$(VPATH)}util.h gc.$(OBJEXT): {$(VPATH)}vm_callinfo.h gc.$(OBJEXT): {$(VPATH)}vm_core.h @@ -7574,7 +7571,6 @@ hash.$(OBJEXT): {$(VPATH)}subst.h hash.$(OBJEXT): {$(VPATH)}symbol.h hash.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h hash.$(OBJEXT): {$(VPATH)}thread_native.h -hash.$(OBJEXT): {$(VPATH)}transient_heap.h hash.$(OBJEXT): {$(VPATH)}util.h hash.$(OBJEXT): {$(VPATH)}vm_core.h hash.$(OBJEXT): {$(VPATH)}vm_debug.h @@ -11869,7 +11865,6 @@ ractor.$(OBJEXT): {$(VPATH)}subst.h ractor.$(OBJEXT): {$(VPATH)}thread.h ractor.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h ractor.$(OBJEXT): {$(VPATH)}thread_native.h -ractor.$(OBJEXT): {$(VPATH)}transient_heap.h ractor.$(OBJEXT): {$(VPATH)}variable.h ractor.$(OBJEXT): {$(VPATH)}vm_core.h ractor.$(OBJEXT): {$(VPATH)}vm_debug.h @@ -16127,7 +16122,6 @@ struct.$(OBJEXT): {$(VPATH)}struct.c struct.$(OBJEXT): {$(VPATH)}subst.h struct.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h struct.$(OBJEXT): {$(VPATH)}thread_native.h -struct.$(OBJEXT): {$(VPATH)}transient_heap.h struct.$(OBJEXT): {$(VPATH)}vm_core.h struct.$(OBJEXT): {$(VPATH)}vm_opts.h symbol.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h @@ -16971,211 +16965,6 @@ transcode.$(OBJEXT): {$(VPATH)}st.h transcode.$(OBJEXT): {$(VPATH)}subst.h transcode.$(OBJEXT): {$(VPATH)}transcode.c transcode.$(OBJEXT): {$(VPATH)}transcode_data.h -transient_heap.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h -transient_heap.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h -transient_heap.$(OBJEXT): $(CCAN_DIR)/list/list.h -transient_heap.$(OBJEXT): $(CCAN_DIR)/str/str.h -transient_heap.$(OBJEXT): $(hdrdir)/ruby/ruby.h -transient_heap.$(OBJEXT): $(top_srcdir)/internal/array.h -transient_heap.$(OBJEXT): $(top_srcdir)/internal/basic_operators.h -transient_heap.$(OBJEXT): $(top_srcdir)/internal/compilers.h -transient_heap.$(OBJEXT): $(top_srcdir)/internal/gc.h -transient_heap.$(OBJEXT): $(top_srcdir)/internal/imemo.h -transient_heap.$(OBJEXT): $(top_srcdir)/internal/sanitizers.h -transient_heap.$(OBJEXT): $(top_srcdir)/internal/serial.h -transient_heap.$(OBJEXT): $(top_srcdir)/internal/static_assert.h -transient_heap.$(OBJEXT): $(top_srcdir)/internal/struct.h -transient_heap.$(OBJEXT): $(top_srcdir)/internal/variable.h -transient_heap.$(OBJEXT): $(top_srcdir)/internal/vm.h -transient_heap.$(OBJEXT): $(top_srcdir)/internal/warnings.h -transient_heap.$(OBJEXT): {$(VPATH)}assert.h -transient_heap.$(OBJEXT): {$(VPATH)}atomic.h -transient_heap.$(OBJEXT): {$(VPATH)}backward/2/assume.h -transient_heap.$(OBJEXT): {$(VPATH)}backward/2/attributes.h -transient_heap.$(OBJEXT): {$(VPATH)}backward/2/bool.h -transient_heap.$(OBJEXT): {$(VPATH)}backward/2/gcc_version_since.h -transient_heap.$(OBJEXT): {$(VPATH)}backward/2/inttypes.h -transient_heap.$(OBJEXT): {$(VPATH)}backward/2/limits.h -transient_heap.$(OBJEXT): {$(VPATH)}backward/2/long_long.h -transient_heap.$(OBJEXT): {$(VPATH)}backward/2/stdalign.h -transient_heap.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h -transient_heap.$(OBJEXT): {$(VPATH)}config.h -transient_heap.$(OBJEXT): {$(VPATH)}constant.h -transient_heap.$(OBJEXT): {$(VPATH)}debug.h -transient_heap.$(OBJEXT): {$(VPATH)}debug_counter.h -transient_heap.$(OBJEXT): {$(VPATH)}defines.h -transient_heap.$(OBJEXT): {$(VPATH)}encoding.h -transient_heap.$(OBJEXT): {$(VPATH)}id.h -transient_heap.$(OBJEXT): {$(VPATH)}id_table.h -transient_heap.$(OBJEXT): {$(VPATH)}intern.h -transient_heap.$(OBJEXT): {$(VPATH)}internal.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/abi.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/anyargs.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/arithmetic.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/arithmetic/char.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/arithmetic/double.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/arithmetic/fixnum.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/arithmetic/gid_t.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/arithmetic/int.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/arithmetic/intptr_t.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/arithmetic/long.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/arithmetic/long_long.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/arithmetic/mode_t.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/arithmetic/off_t.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/arithmetic/pid_t.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/arithmetic/short.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/arithmetic/size_t.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/arithmetic/st_data_t.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/arithmetic/uid_t.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/assume.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/attr/alloc_size.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/attr/artificial.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/attr/cold.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/attr/const.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/attr/constexpr.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/attr/deprecated.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/attr/diagnose_if.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/attr/enum_extensibility.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/attr/error.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/attr/flag_enum.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/attr/forceinline.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/attr/format.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/attr/maybe_unused.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/attr/noalias.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/attr/nodiscard.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/attr/noexcept.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/attr/noinline.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/attr/nonnull.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/attr/noreturn.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/attr/packed_struct.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/attr/pure.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/attr/restrict.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/attr/returns_nonnull.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/attr/warning.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/attr/weakref.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/cast.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/compiler_is.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/compiler_is/apple.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/compiler_is/clang.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/compiler_is/gcc.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/compiler_is/intel.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/compiler_is/msvc.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/compiler_is/sunpro.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/compiler_since.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/config.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/constant_p.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/core.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/core/rarray.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/core/rbasic.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/core/rbignum.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/core/rclass.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/core/rdata.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/core/rfile.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/core/rhash.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/core/robject.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/core/rregexp.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/core/rstring.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/core/rstruct.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/core/rtypeddata.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/ctype.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/dllexport.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/dosish.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/encoding/coderange.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/encoding/ctype.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/encoding/encoding.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/encoding/pathname.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/encoding/re.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/encoding/sprintf.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/encoding/string.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/encoding/symbol.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/encoding/transcode.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/error.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/eval.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/event.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/fl_type.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/gc.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/glob.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/globals.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/has/attribute.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/has/builtin.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/has/c_attribute.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/has/cpp_attribute.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/has/declspec_attribute.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/has/extension.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/has/feature.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/has/warning.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/array.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/bignum.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/class.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/compar.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/complex.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/cont.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/dir.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/enum.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/enumerator.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/error.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/eval.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/file.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/hash.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/io.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/load.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/marshal.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/numeric.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/object.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/parse.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/proc.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/process.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/random.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/range.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/rational.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/re.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/ruby.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/select.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/select/largesize.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/signal.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/sprintf.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/string.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/struct.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/thread.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/time.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/variable.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/intern/vm.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/interpreter.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/iterator.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/memory.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/method.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/module.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/newobj.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/scan_args.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/special_consts.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/static_assert.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/stdalign.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/stdbool.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/symbol.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/value.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/value_type.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/variable.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/warning_push.h -transient_heap.$(OBJEXT): {$(VPATH)}internal/xmalloc.h -transient_heap.$(OBJEXT): {$(VPATH)}method.h -transient_heap.$(OBJEXT): {$(VPATH)}missing.h -transient_heap.$(OBJEXT): {$(VPATH)}node.h -transient_heap.$(OBJEXT): {$(VPATH)}onigmo.h -transient_heap.$(OBJEXT): {$(VPATH)}oniguruma.h -transient_heap.$(OBJEXT): {$(VPATH)}ruby_assert.h -transient_heap.$(OBJEXT): {$(VPATH)}ruby_atomic.h -transient_heap.$(OBJEXT): {$(VPATH)}rubyparser.h -transient_heap.$(OBJEXT): {$(VPATH)}shape.h -transient_heap.$(OBJEXT): {$(VPATH)}st.h -transient_heap.$(OBJEXT): {$(VPATH)}subst.h -transient_heap.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h -transient_heap.$(OBJEXT): {$(VPATH)}thread_native.h -transient_heap.$(OBJEXT): {$(VPATH)}transient_heap.c -transient_heap.$(OBJEXT): {$(VPATH)}transient_heap.h -transient_heap.$(OBJEXT): {$(VPATH)}vm_core.h -transient_heap.$(OBJEXT): {$(VPATH)}vm_debug.h -transient_heap.$(OBJEXT): {$(VPATH)}vm_opts.h -transient_heap.$(OBJEXT): {$(VPATH)}vm_sync.h util.$(OBJEXT): $(hdrdir)/ruby/ruby.h util.$(OBJEXT): $(top_srcdir)/internal/compilers.h util.$(OBJEXT): $(top_srcdir)/internal/sanitizers.h @@ -17550,7 +17339,6 @@ variable.$(OBJEXT): {$(VPATH)}subst.h variable.$(OBJEXT): {$(VPATH)}symbol.h variable.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h variable.$(OBJEXT): {$(VPATH)}thread_native.h -variable.$(OBJEXT): {$(VPATH)}transient_heap.h variable.$(OBJEXT): {$(VPATH)}util.h variable.$(OBJEXT): {$(VPATH)}variable.c variable.$(OBJEXT): {$(VPATH)}variable.h diff --git a/debug_counter.h b/debug_counter.h index f1208d83a8..910f6ebf41 100644 --- a/debug_counter.h +++ b/debug_counter.h @@ -233,7 +233,6 @@ RB_DEBUG_COUNTER(obj_promote) RB_DEBUG_COUNTER(obj_wb_unprotect) RB_DEBUG_COUNTER(obj_obj_embed) -RB_DEBUG_COUNTER(obj_obj_transient) RB_DEBUG_COUNTER(obj_obj_ptr) RB_DEBUG_COUNTER(obj_obj_too_complex) @@ -244,7 +243,6 @@ RB_DEBUG_COUNTER(obj_str_nofree) RB_DEBUG_COUNTER(obj_str_fstr) RB_DEBUG_COUNTER(obj_ary_embed) -RB_DEBUG_COUNTER(obj_ary_transient) RB_DEBUG_COUNTER(obj_ary_ptr) RB_DEBUG_COUNTER(obj_ary_extracapa) /* @@ -268,11 +266,9 @@ RB_DEBUG_COUNTER(obj_hash_g8) RB_DEBUG_COUNTER(obj_hash_null) RB_DEBUG_COUNTER(obj_hash_ar) RB_DEBUG_COUNTER(obj_hash_st) -RB_DEBUG_COUNTER(obj_hash_transient) RB_DEBUG_COUNTER(obj_hash_force_convert) RB_DEBUG_COUNTER(obj_struct_embed) -RB_DEBUG_COUNTER(obj_struct_transient) RB_DEBUG_COUNTER(obj_struct_ptr) RB_DEBUG_COUNTER(obj_data_empty) @@ -327,11 +323,6 @@ RB_DEBUG_COUNTER(heap_xmalloc) RB_DEBUG_COUNTER(heap_xrealloc) RB_DEBUG_COUNTER(heap_xfree) -/* transient_heap */ -RB_DEBUG_COUNTER(theap_alloc) -RB_DEBUG_COUNTER(theap_alloc_fail) -RB_DEBUG_COUNTER(theap_evacuate) - // VM sync RB_DEBUG_COUNTER(vm_sync_lock) RB_DEBUG_COUNTER(vm_sync_lock_enter) @@ -131,7 +131,6 @@ #include "ruby_assert.h" #include "ruby_atomic.h" #include "symbol.h" -#include "transient_heap.h" #include "vm_core.h" #include "vm_sync.h" #include "vm_callinfo.h" @@ -1722,7 +1721,6 @@ RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, struct heap_page *pag { MARK_IN_BITMAP(&page->uncollectible_bits[0], obj); objspace->rgengc.old_objects++; - rb_transient_heap_promote(obj); #if RGENGC_PROFILE >= 2 objspace->profile.total_promoted_count++; @@ -3468,9 +3466,6 @@ obj_free(rb_objspace_t *objspace, VALUE obj) else if (RANY(obj)->as.basic.flags & ROBJECT_EMBED) { RB_DEBUG_COUNTER_INC(obj_obj_embed); } - else if (ROBJ_TRANSIENT_P(obj)) { - RB_DEBUG_COUNTER_INC(obj_obj_transient); - } else { xfree(RANY(obj)->as.object.as.heap.ivptr); RB_DEBUG_COUNTER_INC(obj_obj_ptr); @@ -3641,9 +3636,6 @@ obj_free(rb_objspace_t *objspace, VALUE obj) RANY(obj)->as.rstruct.as.heap.ptr == NULL) { RB_DEBUG_COUNTER_INC(obj_struct_embed); } - else if (RSTRUCT_TRANSIENT_P(obj)) { - RB_DEBUG_COUNTER_INC(obj_struct_transient); - } else { xfree((void *)RANY(obj)->as.rstruct.as.heap.ptr); RB_DEBUG_COUNTER_INC(obj_struct_ptr); @@ -7219,12 +7211,6 @@ gc_mark_children(rb_objspace_t *objspace, VALUE obj) for (i=0; i < len; i++) { gc_mark(objspace, ptr[i]); } - - if (LIKELY(during_gc)) { - if (!ARY_EMBED_P(obj) && RARRAY_TRANSIENT_P(obj)) { - rb_transient_heap_mark(obj, ptr); - } - } } break; @@ -7268,11 +7254,6 @@ gc_mark_children(rb_objspace_t *objspace, VALUE obj) for (i = 0; i < len; i++) { gc_mark(objspace, ptr[i]); } - - if (LIKELY(during_gc) && - ROBJ_TRANSIENT_P(obj)) { - rb_transient_heap_mark(obj, ptr); - } } if (shape) { VALUE klass = RBASIC_CLASS(obj); @@ -7329,11 +7310,6 @@ gc_mark_children(rb_objspace_t *objspace, VALUE obj) for (i=0; i<len; i++) { gc_mark(objspace, ptr[i]); } - - if (LIKELY(during_gc) && - RSTRUCT_TRANSIENT_P(obj)) { - rb_transient_heap_mark(obj, ptr); - } } break; @@ -8098,13 +8074,6 @@ rb_gc_verify_internal_consistency(void) gc_verify_internal_consistency(&rb_objspace); } -static VALUE -gc_verify_transient_heap_internal_consistency(VALUE dmy) -{ - rb_transient_heap_verify(); - return Qnil; -} - static void heap_move_pooled_pages_to_free_pages(rb_heap_t *heap) { @@ -8325,7 +8294,6 @@ gc_marks_finish(rb_objspace_t *objspace) objspace->rgengc.need_major_gc ? "major" : "minor"); } - rb_transient_heap_finish_marking(); rb_ractor_finish_marking(); gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_END_MARK, 0); @@ -9403,7 +9371,6 @@ gc_start(rb_objspace_t *objspace, unsigned int reason) objspace->profile.heap_used_at_gc_start = heap_allocated_pages; gc_prof_setup_new_record(objspace, reason); gc_reset_malloc_info(objspace, do_full_mark); - rb_transient_heap_start_marking(do_full_mark); gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_START, 0 /* TODO: pass minor/immediate flag? */); GC_ASSERT(during_gc); @@ -9988,12 +9955,7 @@ gc_ref_update_object(rb_objspace_t *objspace, VALUE v) // Object can be re-embedded memcpy(ROBJECT(v)->as.ary, ptr, sizeof(VALUE) * ROBJECT_IV_COUNT(v)); RB_FL_SET_RAW(v, ROBJECT_EMBED); - if (ROBJ_TRANSIENT_P(v)) { - ROBJ_TRANSIENT_UNSET(v); - } - else { - xfree(ptr); - } + xfree(ptr); ptr = ROBJECT(v)->as.ary; } @@ -10675,7 +10637,6 @@ gc_update_references(rb_objspace_t *objspace) } } rb_vm_update_references(vm); - rb_transient_heap_update_references(); rb_gc_update_global_tbl(); global_symbols.ids = rb_gc_location(global_symbols.ids); global_symbols.dsymbol_fstr_hash = rb_gc_location(global_symbols.dsymbol_fstr_hash); @@ -13505,10 +13466,9 @@ rb_raw_obj_info_buitin_type(char *const buff, const size_t buff_size, const VALU RARRAY_LEN(obj)); } else { - APPEND_F("[%s%s%s] len: %ld, capa:%ld ptr:%p", + APPEND_F("[%s%s] len: %ld, capa:%ld ptr:%p", C(ARY_EMBED_P(obj), "E"), C(ARY_SHARED_P(obj), "S"), - C(RARRAY_TRANSIENT_P(obj), "T"), RARRAY_LEN(obj), ARY_EMBED_P(obj) ? -1L : RARRAY(obj)->as.heap.aux.capa, (void *)RARRAY_CONST_PTR_TRANSIENT(obj)); @@ -13954,7 +13914,6 @@ Init_GC(void) /* internal methods */ rb_define_singleton_method(rb_mGC, "verify_internal_consistency", gc_verify_internal_consistency_m, 0); - rb_define_singleton_method(rb_mGC, "verify_transient_heap_internal_consistency", gc_verify_transient_heap_internal_consistency, 0); #if MALLOC_ALLOCATED_SIZE rb_define_singleton_method(rb_mGC, "malloc_allocated_size", gc_malloc_allocated_size, 0); rb_define_singleton_method(rb_mGC, "malloc_allocations", gc_malloc_allocations, 0); diff --git a/include/ruby/internal/core/rarray.h b/include/ruby/internal/core/rarray.h index 9670f93d0c..ae437425af 100644 --- a/include/ruby/internal/core/rarray.h +++ b/include/ruby/internal/core/rarray.h @@ -36,18 +36,6 @@ #include "ruby/assert.h" /** - * @private - * @warning Do not touch this macro. - * @warning It is an implementation detail. - * @warning The value of this macro must match for ruby itself and all - * extension libraries, otherwise serious memory corruption shall - * occur. - */ -#ifndef USE_TRANSIENT_HEAP -# define USE_TRANSIENT_HEAP 1 -#endif - -/** * Convenient casting macro. * * @param obj An object, which is in fact an ::RArray. @@ -59,11 +47,6 @@ #define RARRAY_EMBED_LEN_MASK RARRAY_EMBED_LEN_MASK #define RARRAY_EMBED_LEN_MAX RARRAY_EMBED_LEN_MAX #define RARRAY_EMBED_LEN_SHIFT RARRAY_EMBED_LEN_SHIFT -#if USE_TRANSIENT_HEAP -# define RARRAY_TRANSIENT_FLAG RARRAY_TRANSIENT_FLAG -#else -# define RARRAY_TRANSIENT_FLAG 0 -#endif /** @endcond */ #define RARRAY_LEN rb_array_len /**< @alias{rb_array_len} */ #define RARRAY_CONST_PTR rb_array_const_ptr /**< @alias{rb_array_const_ptr} */ @@ -80,7 +63,6 @@ #define RARRAY_EMBED_LEN RARRAY_EMBED_LEN #define RARRAY_LENINT RARRAY_LENINT -#define RARRAY_TRANSIENT_P RARRAY_TRANSIENT_P #define RARRAY_ASET RARRAY_ASET #define RARRAY_PTR RARRAY_PTR /** @endcond */ @@ -132,24 +114,6 @@ enum ruby_rarray_flags { */ RARRAY_EMBED_LEN_MASK = RUBY_FL_USER9 | RUBY_FL_USER8 | RUBY_FL_USER7 | RUBY_FL_USER6 | RUBY_FL_USER5 | RUBY_FL_USER4 | RUBY_FL_USER3 - -#if USE_TRANSIENT_HEAP - , - - /** - * This flag has something to do with an array's "transiency". A transient - * array is an array of young generation (of generational GC), who stores - * its elements inside of dedicated memory pages called a transient heap. - * Not every young generation share that storage scheme, but elder - * generations must no join. - * - * @internal - * - * 3rd parties must not be aware that there even is more than one way to - * store array elements. It was a bad idea to expose this to them. - */ - RARRAY_TRANSIENT_FLAG = RUBY_FL_USER13 -#endif }; /** @@ -249,16 +213,6 @@ VALUE *rb_ary_ptr_use_start(VALUE ary); */ void rb_ary_ptr_use_end(VALUE a); -#if USE_TRANSIENT_HEAP -/** - * Destructively converts an array of transient backend into ordinal one. - * - * @param[out] a An object of ::RArray. - * @pre `a` must be a transient array. - * @post `a` gets out of transient heap, destructively. - */ -void rb_ary_detransient(VALUE a); -#endif RBIMPL_SYMBOL_EXPORT_END() RBIMPL_ATTR_PURE_UNLESS_DEBUG() @@ -331,33 +285,6 @@ RARRAY_LENINT(VALUE ary) } RBIMPL_ATTR_PURE_UNLESS_DEBUG() -RBIMPL_ATTR_ARTIFICIAL() -/** - * Queries if the array is a transient array. - * - * @param[in] ary Array in question. - * @retval true Yes it is. - * @retval false No it isn't. - * @pre `ary` must be an instance of ::RArray. - * - * @internal - * - * @shyouhei doesn't understand the benefit of this function called from - * extension libraries. - */ -static inline bool -RARRAY_TRANSIENT_P(VALUE ary) -{ - RBIMPL_ASSERT_TYPE(ary, RUBY_T_ARRAY); - -#if USE_TRANSIENT_HEAP - return RB_FL_ANY_RAW(ary, RARRAY_TRANSIENT_FLAG); -#else - return false; -#endif -} - -RBIMPL_ATTR_PURE_UNLESS_DEBUG() /** * @private * @@ -380,9 +307,7 @@ rb_array_const_ptr_transient(VALUE a) } } -#if ! USE_TRANSIENT_HEAP RBIMPL_ATTR_PURE_UNLESS_DEBUG() -#endif /** * @private * @@ -398,11 +323,6 @@ rb_array_const_ptr(VALUE a) { RBIMPL_ASSERT_TYPE(a, RUBY_T_ARRAY); -#if USE_TRANSIENT_HEAP - if (RARRAY_TRANSIENT_P(a)) { - rb_ary_detransient(a); - } -#endif return rb_array_const_ptr_transient(a); } @@ -424,14 +344,6 @@ rb_array_ptr_use_start(VALUE a, { RBIMPL_ASSERT_TYPE(a, RUBY_T_ARRAY); -#if USE_TRANSIENT_HEAP - if (!allow_transient) { - if (RARRAY_TRANSIENT_P(a)) { - rb_ary_detransient(a); - } - } -#endif - return rb_ary_ptr_use_start(a); } @@ -22,9 +22,6 @@ rb_call_inits(void) { CALL(default_shapes); CALL(Thread_Mutex); -#if USE_TRANSIENT_HEAP - CALL(TransientHeap); -#endif CALL(vm_postponed_job); CALL(Method); CALL(RandomSeedCore); diff --git a/internal/array.h b/internal/array.h index 7e55b66a87..5697d39315 100644 --- a/internal/array.h +++ b/internal/array.h @@ -40,8 +40,6 @@ VALUE rb_ary_diff(VALUE ary1, VALUE ary2); static inline VALUE rb_ary_entry_internal(VALUE ary, long offset); static inline bool ARY_PTR_USING_P(VALUE ary); -static inline void RARY_TRANSIENT_SET(VALUE ary); -static inline void RARY_TRANSIENT_UNSET(VALUE ary); VALUE rb_ary_tmp_new_from_values(VALUE, long, const VALUE *); VALUE rb_check_to_array(VALUE ary); @@ -118,22 +116,6 @@ ARY_SHARED_ROOT_REFCNT(VALUE ary) return RARRAY(ary)->as.heap.aux.capa; } -static inline void -RARY_TRANSIENT_SET(VALUE ary) -{ -#if USE_TRANSIENT_HEAP - FL_SET_RAW(ary, RARRAY_TRANSIENT_FLAG); -#endif -} - -static inline void -RARY_TRANSIENT_UNSET(VALUE ary) -{ -#if USE_TRANSIENT_HEAP - FL_UNSET_RAW(ary, RARRAY_TRANSIENT_FLAG); -#endif -} - #undef rb_ary_new_from_args #if RBIMPL_HAS_WARNING("-Wgnu-zero-variadic-macro-arguments") # /* Skip it; clang -pedantic doesn't like the following */ diff --git a/internal/struct.h b/internal/struct.h index 9b56254541..00158c87d9 100644 --- a/internal/struct.h +++ b/internal/struct.h @@ -61,9 +61,6 @@ VALUE rb_struct_init_copy(VALUE copy, VALUE s); VALUE rb_struct_lookup(VALUE s, VALUE idx); VALUE rb_struct_s_keyword_init(VALUE klass); static inline const VALUE *rb_struct_const_heap_ptr(VALUE st); -static inline bool RSTRUCT_TRANSIENT_P(VALUE st); -static inline void RSTRUCT_TRANSIENT_SET(VALUE st); -static inline void RSTRUCT_TRANSIENT_UNSET(VALUE st); static inline long RSTRUCT_EMBED_LEN(VALUE st); static inline long RSTRUCT_LEN(VALUE st); static inline int RSTRUCT_LENINT(VALUE st); @@ -71,32 +68,6 @@ static inline const VALUE *RSTRUCT_CONST_PTR(VALUE st); static inline void RSTRUCT_SET(VALUE st, long k, VALUE v); static inline VALUE RSTRUCT_GET(VALUE st, long k); -static inline bool -RSTRUCT_TRANSIENT_P(VALUE st) -{ -#if USE_TRANSIENT_HEAP - return FL_TEST_RAW(st, RSTRUCT_TRANSIENT_FLAG); -#else - return false; -#endif -} - -static inline void -RSTRUCT_TRANSIENT_SET(VALUE st) -{ -#if USE_TRANSIENT_HEAP - FL_SET_RAW(st, RSTRUCT_TRANSIENT_FLAG); -#endif -} - -static inline void -RSTRUCT_TRANSIENT_UNSET(VALUE st) -{ -#if USE_TRANSIENT_HEAP - FL_UNSET_RAW(st, RSTRUCT_TRANSIENT_FLAG); -#endif -} - static inline long RSTRUCT_EMBED_LEN(VALUE st) { diff --git a/internal/variable.h b/internal/variable.h index 4fb7b23cf5..ed622cd905 100644 --- a/internal/variable.h +++ b/internal/variable.h @@ -32,9 +32,6 @@ rb_gvar_getter_t *rb_gvar_getter_function_of(ID); rb_gvar_setter_t *rb_gvar_setter_function_of(ID); void rb_gvar_readonly_setter(VALUE v, ID id, VALUE *_); void rb_gvar_ractor_local(const char *name); -static inline bool ROBJ_TRANSIENT_P(VALUE obj); -static inline void ROBJ_TRANSIENT_SET(VALUE obj); -static inline void ROBJ_TRANSIENT_UNSET(VALUE obj); /** * Sets the name of a module. @@ -75,30 +72,4 @@ void rb_ensure_iv_list_size(VALUE obj, uint32_t len, uint32_t newsize); struct gen_ivtbl *rb_ensure_generic_iv_list_size(VALUE obj, rb_shape_t *shape, uint32_t newsize); attr_index_t rb_obj_ivar_set(VALUE obj, ID id, VALUE val); -static inline bool -ROBJ_TRANSIENT_P(VALUE obj) -{ -#if USE_TRANSIENT_HEAP - return FL_TEST_RAW(obj, ROBJECT_TRANSIENT_FLAG); -#else - return false; -#endif -} - -static inline void -ROBJ_TRANSIENT_SET(VALUE obj) -{ -#if USE_TRANSIENT_HEAP - FL_SET_RAW(obj, ROBJECT_TRANSIENT_FLAG); -#endif -} - -static inline void -ROBJ_TRANSIENT_UNSET(VALUE obj) -{ -#if USE_TRANSIENT_HEAP - FL_UNSET_RAW(obj, ROBJECT_TRANSIENT_FLAG); -#endif -} - #endif /* INTERNAL_VARIABLE_H */ @@ -16,7 +16,6 @@ #include "internal/struct.h" #include "internal/thread.h" #include "variable.h" -#include "transient_heap.h" #include "yjit.h" #include "rjit.h" @@ -1833,7 +1832,6 @@ cancel_single_ractor_mode(void) VALUE was_disabled = rb_gc_enable(); rb_gc_start(); - rb_transient_heap_evacuate(); if (was_disabled) { rb_gc_disable(); @@ -3203,10 +3201,6 @@ obj_traverse_replace_i(VALUE obj, struct obj_traverse_replace_data *data) (st_data_t)&d); } else { -#if USE_TRANSIENT_HEAP - if (data->move) rb_obj_transient_heap_evacuate(obj, TRUE); -#endif - uint32_t len = ROBJECT_IV_COUNT(obj); VALUE *ptr = ROBJECT_IVPTR(obj); @@ -3222,9 +3216,6 @@ obj_traverse_replace_i(VALUE obj, struct obj_traverse_replace_data *data) case T_ARRAY: { rb_ary_cancel_sharing(obj); -#if USE_TRANSIENT_HEAP - if (data->move) rb_ary_transient_heap_evacuate(obj, TRUE); -#endif for (int i = 0; i < RARRAY_LENINT(obj); i++) { VALUE e = rb_ary_entry(obj, i); @@ -3265,9 +3256,6 @@ obj_traverse_replace_i(VALUE obj, struct obj_traverse_replace_data *data) case T_STRUCT: { -#if USE_TRANSIENT_HEAP - if (data->move) rb_struct_transient_heap_evacuate(obj, TRUE); -#endif long len = RSTRUCT_LEN(obj); const VALUE *ptr = RSTRUCT_CONST_PTR(obj); @@ -18,7 +18,6 @@ #include "internal/proc.h" #include "internal/struct.h" #include "internal/symbol.h" -#include "transient_heap.h" #include "vm_core.h" #include "builtin.h" @@ -798,39 +797,8 @@ rb_struct_initialize(VALUE self, VALUE values) static VALUE * struct_heap_alloc(VALUE st, size_t len) { - VALUE *ptr = rb_transient_heap_alloc((VALUE)st, sizeof(VALUE) * len); - - if (ptr) { - RSTRUCT_TRANSIENT_SET(st); - return ptr; - } - else { - RSTRUCT_TRANSIENT_UNSET(st); - return ALLOC_N(VALUE, len); - } -} - -#if USE_TRANSIENT_HEAP -void -rb_struct_transient_heap_evacuate(VALUE obj, int promote) -{ - if (RSTRUCT_TRANSIENT_P(obj)) { - const VALUE *old_ptr = rb_struct_const_heap_ptr(obj); - VALUE *new_ptr; - long len = RSTRUCT_LEN(obj); - - if (promote) { - new_ptr = ALLOC_N(VALUE, len); - FL_UNSET_RAW(obj, RSTRUCT_TRANSIENT_FLAG); - } - else { - new_ptr = struct_heap_alloc(obj, len); - } - MEMCPY(new_ptr, old_ptr, VALUE, len); - RSTRUCT(obj)->as.heap.ptr = new_ptr; - } + return ALLOC_N(VALUE, len); } -#endif static VALUE struct_alloc(VALUE klass) diff --git a/transient_heap.c b/transient_heap.c deleted file mode 100644 index 6859926f93..0000000000 --- a/transient_heap.c +++ /dev/null @@ -1,976 +0,0 @@ -/********************************************************************** - - transient_heap.c - implement transient_heap. - - Copyright (C) 2018 Koichi Sasada - -**********************************************************************/ - -#include "debug_counter.h" -#include "internal.h" -#include "internal/array.h" -#include "internal/gc.h" -#include "internal/sanitizers.h" -#include "internal/static_assert.h" -#include "internal/struct.h" -#include "internal/variable.h" -#include "ruby/debug.h" -#include "ruby/ruby.h" -#include "ruby_assert.h" -#include "transient_heap.h" -#include "vm_debug.h" -#include "vm_sync.h" - -#if USE_TRANSIENT_HEAP /* USE_TRANSIENT_HEAP */ -/* - * 1: enable assertions - * 2: enable verify all transient heaps - */ -#ifndef TRANSIENT_HEAP_CHECK_MODE -#define TRANSIENT_HEAP_CHECK_MODE 0 -#endif -#define TH_ASSERT(expr) RUBY_ASSERT_MESG_WHEN(TRANSIENT_HEAP_CHECK_MODE > 0, expr, #expr) - -/* - * 1: show events - * 2: show dump at events - * 3: show all operations - */ -#define TRANSIENT_HEAP_DEBUG 0 - -/* For Debug: Provide blocks infinitely. - * This mode generates blocks unlimitedly - * and prohibit access free'ed blocks to check invalid access. - */ -#define TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK 0 - -#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK -#include <sys/mman.h> -#include <errno.h> -#endif - -/* For Debug: Prohibit promoting to malloc space. - */ -#define TRANSIENT_HEAP_DEBUG_DONT_PROMOTE 0 - -/* size configuration */ -#define TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE 1024 - - /* K M */ -#define TRANSIENT_HEAP_BLOCK_SIZE (1024 * 32 ) /* 32KB int16_t */ -#ifndef TRANSIENT_HEAP_TOTAL_SIZE -#define TRANSIENT_HEAP_TOTAL_SIZE (1024 * 1024 * 32) /* 32 MB */ -#endif -#define TRANSIENT_HEAP_ALLOC_MAX (1024 * 2 ) /* 2 KB */ -#define TRANSIENT_HEAP_BLOCK_NUM (TRANSIENT_HEAP_TOTAL_SIZE / TRANSIENT_HEAP_BLOCK_SIZE) -#define TRANSIENT_HEAP_USABLE_SIZE (TRANSIENT_HEAP_BLOCK_SIZE - sizeof(struct transient_heap_block_header)) - -#define TRANSIENT_HEAP_ALLOC_MAGIC 0xfeab -#define TRANSIENT_HEAP_ALLOC_ALIGN RUBY_ALIGNOF(void *) - -#define TRANSIENT_HEAP_ALLOC_MARKING_LAST -1 -#define TRANSIENT_HEAP_ALLOC_MARKING_FREE -2 - -enum transient_heap_status { - transient_heap_none, - transient_heap_marking, - transient_heap_escaping -}; - -struct transient_heap_block { - struct transient_heap_block_header { - int16_t index; - int16_t last_marked_index; - int16_t objects; - struct transient_heap_block *next_block; - } info; - char buff[TRANSIENT_HEAP_USABLE_SIZE]; -}; - -struct transient_heap { - struct transient_heap_block *using_blocks; - struct transient_heap_block *marked_blocks; - struct transient_heap_block *free_blocks; - int total_objects; - int total_marked_objects; - int total_blocks; - enum transient_heap_status status; - - VALUE *promoted_objects; - int promoted_objects_size; - int promoted_objects_index; - - struct transient_heap_block *arena; - int arena_index; /* increment only */ -}; - -struct transient_alloc_header { - uint16_t magic; - uint16_t size; - int16_t next_marked_index; - int16_t dummy; - VALUE obj; -}; - -static struct transient_heap global_transient_heap; - -static void transient_heap_promote_add(struct transient_heap* theap, VALUE obj); -static const void *transient_heap_ptr(VALUE obj, int error); -static int transient_header_managed_ptr_p(struct transient_heap* theap, const void *ptr); - -#define ROUND_UP(v, a) (((size_t)(v) + (a) - 1) & ~((a) - 1)) - -static void -transient_heap_block_dump(struct transient_heap* theap, struct transient_heap_block *block) -{ - int i=0, n=0; - - while (i<block->info.index) { - void *ptr = &block->buff[i]; - struct transient_alloc_header *header = ptr; - fprintf(stderr, "%4d %8d %p size:%4d next:%4d %s\n", n, i, ptr, header->size, header->next_marked_index, rb_obj_info(header->obj)); - i += header->size; - n++; - } -} - -static void -transient_heap_blocks_dump(struct transient_heap* theap, struct transient_heap_block *block, const char *type_str) -{ - while (block) { - fprintf(stderr, "- transient_heap_dump: %s:%p index:%d objects:%d last_marked_index:%d next:%p\n", - type_str, (void *)block, block->info.index, block->info.objects, block->info.last_marked_index, (void *)block->info.next_block); - - transient_heap_block_dump(theap, block); - block = block->info.next_block; - } -} - -static void -transient_heap_dump(struct transient_heap* theap) -{ - fprintf(stderr, "transient_heap_dump objects:%d marked_objects:%d blocks:%d\n", theap->total_objects, theap->total_marked_objects, theap->total_blocks); - transient_heap_blocks_dump(theap, theap->using_blocks, "using_blocks"); - transient_heap_blocks_dump(theap, theap->marked_blocks, "marked_blocks"); - transient_heap_blocks_dump(theap, theap->free_blocks, "free_blocks"); -} - -/* Debug: dump all transient_heap blocks */ -void -rb_transient_heap_dump(void) -{ - transient_heap_dump(&global_transient_heap); -} - -#if TRANSIENT_HEAP_CHECK_MODE >= 2 -ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static void transient_heap_ptr_check(struct transient_heap *theap, VALUE obj)); -static void -transient_heap_ptr_check(struct transient_heap *theap, VALUE obj) -{ - if (!UNDEF_P(obj)) { - const void *ptr = transient_heap_ptr(obj, FALSE); - TH_ASSERT(ptr == NULL || transient_header_managed_ptr_p(theap, ptr)); - } -} - -ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static int transient_heap_block_verify(struct transient_heap *theap, struct transient_heap_block *block)); -static int -transient_heap_block_verify(struct transient_heap *theap, struct transient_heap_block *block) -{ - int i=0, n=0; - struct transient_alloc_header *header; - - while (i<block->info.index) { - header = (void *)&block->buff[i]; - TH_ASSERT(header->magic == TRANSIENT_HEAP_ALLOC_MAGIC); - transient_heap_ptr_check(theap, header->obj); - n ++; - i += header->size; - } - TH_ASSERT(block->info.objects == n); - - return n; -} - -static int -transient_heap_blocks_verify(struct transient_heap *theap, struct transient_heap_block *blocks, int *block_num_ptr) -{ - int n = 0; - struct transient_heap_block *block = blocks; - while (block) { - n += transient_heap_block_verify(theap, block); - *block_num_ptr += 1; - block = block->info.next_block; - } - - return n; -} -#endif - -static void -transient_heap_verify(struct transient_heap *theap) -{ -#if TRANSIENT_HEAP_CHECK_MODE >= 2 - int n=0, block_num=0; - - n += transient_heap_blocks_verify(theap, theap->using_blocks, &block_num); - n += transient_heap_blocks_verify(theap, theap->marked_blocks, &block_num); - - TH_ASSERT(n == theap->total_objects); - TH_ASSERT(n >= theap->total_marked_objects); - TH_ASSERT(block_num == theap->total_blocks); -#endif -} - -/* Debug: check assertions for all transient_heap blocks */ -void -rb_transient_heap_verify(void) -{ - transient_heap_verify(&global_transient_heap); -} - -static struct transient_heap* -transient_heap_get(void) -{ - struct transient_heap* theap = &global_transient_heap; - transient_heap_verify(theap); - return theap; -} - -static void -reset_block(struct transient_heap_block *block) -{ - __msan_allocated_memory(block, sizeof block); - block->info.index = 0; - block->info.objects = 0; - block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST; - block->info.next_block = NULL; - __asan_poison_memory_region(&block->buff, sizeof block->buff); -} - -static void -connect_to_free_blocks(struct transient_heap *theap, struct transient_heap_block *block) -{ - block->info.next_block = theap->free_blocks; - theap->free_blocks = block; -} - -static void -connect_to_using_blocks(struct transient_heap *theap, struct transient_heap_block *block) -{ - block->info.next_block = theap->using_blocks; - theap->using_blocks = block; -} - -#if 0 -static void -connect_to_marked_blocks(struct transient_heap *theap, struct transient_heap_block *block) -{ - block->info.next_block = theap->marked_blocks; - theap->marked_blocks = block; -} -#endif - -static void -append_to_marked_blocks(struct transient_heap *theap, struct transient_heap_block *append_blocks) -{ - if (theap->marked_blocks) { - struct transient_heap_block *block = theap->marked_blocks, *last_block = NULL; - while (block) { - last_block = block; - block = block->info.next_block; - } - - TH_ASSERT(last_block->info.next_block == NULL); - last_block->info.next_block = append_blocks; - } - else { - theap->marked_blocks = append_blocks; - } -} - -static struct transient_heap_block * -transient_heap_block_alloc(struct transient_heap* theap) -{ - struct transient_heap_block *block; -#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK - block = mmap(NULL, TRANSIENT_HEAP_BLOCK_SIZE, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - -1, 0); - if (block == MAP_FAILED) rb_bug("transient_heap_block_alloc: err:%d", errno); -#else - if (theap->arena == NULL) { - theap->arena = rb_aligned_malloc(TRANSIENT_HEAP_BLOCK_SIZE, TRANSIENT_HEAP_TOTAL_SIZE); - if (theap->arena == NULL) { - rb_bug("transient_heap_block_alloc: failed"); - } - } - - TH_ASSERT(theap->arena_index < TRANSIENT_HEAP_BLOCK_NUM); - block = &theap->arena[theap->arena_index++]; - TH_ASSERT(((intptr_t)block & (TRANSIENT_HEAP_BLOCK_SIZE - 1)) == 0); -#endif - reset_block(block); - - TH_ASSERT(((intptr_t)block->buff & (TRANSIENT_HEAP_ALLOC_ALIGN-1)) == 0); - if (0) fprintf(stderr, "transient_heap_block_alloc: %4d %p\n", theap->total_blocks, (void *)block); - return block; -} - - -static struct transient_heap_block * -transient_heap_allocatable_block(struct transient_heap* theap) -{ - struct transient_heap_block *block; - -#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK - block = transient_heap_block_alloc(theap); - theap->total_blocks++; -#else - /* get one block from free_blocks */ - block = theap->free_blocks; - if (block) { - theap->free_blocks = block->info.next_block; - block->info.next_block = NULL; - theap->total_blocks++; - } -#endif - - return block; -} - -static struct transient_alloc_header * -transient_heap_allocatable_header(struct transient_heap* theap, size_t size) -{ - struct transient_heap_block *block = theap->using_blocks; - - while (block) { - TH_ASSERT(block->info.index <= (int16_t)TRANSIENT_HEAP_USABLE_SIZE); - - if (TRANSIENT_HEAP_USABLE_SIZE - block->info.index >= size) { - struct transient_alloc_header *header = (void *)&block->buff[block->info.index]; - block->info.index += size; - block->info.objects++; - return header; - } - else { - block = transient_heap_allocatable_block(theap); - if (block) connect_to_using_blocks(theap, block); - } - } - - return NULL; -} - -void * -rb_transient_heap_alloc(VALUE obj, size_t req_size) -{ - // only on single main ractor - if (ruby_single_main_ractor == NULL) return NULL; - - void *ret; - struct transient_heap* theap = transient_heap_get(); - size_t size = ROUND_UP(req_size + sizeof(struct transient_alloc_header), TRANSIENT_HEAP_ALLOC_ALIGN); - - TH_ASSERT(RB_TYPE_P(obj, T_ARRAY) || - RB_TYPE_P(obj, T_OBJECT) || - RB_TYPE_P(obj, T_STRUCT)); /* supported types */ - - if (size > TRANSIENT_HEAP_ALLOC_MAX) { - if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [too big: %ld] %s\n", (long)size, rb_obj_info(obj)); - ret = NULL; - } -#if TRANSIENT_HEAP_DEBUG_DONT_PROMOTE == 0 - else if (RB_OBJ_PROMOTED_RAW(obj)) { - if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [promoted object] %s\n", rb_obj_info(obj)); - ret = NULL; - } -#else - else if (RBASIC_CLASS(obj) == 0) { - if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [hidden object] %s\n", rb_obj_info(obj)); - ret = NULL; - } -#endif - else { - struct transient_alloc_header *header = transient_heap_allocatable_header(theap, size); - if (header) { - void *ptr; - - /* header is poisoned to prevent buffer overflow, should - * unpoison first... */ - asan_unpoison_memory_region(header, sizeof *header, true); - - header->size = size; - header->magic = TRANSIENT_HEAP_ALLOC_MAGIC; - header->next_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_FREE; - header->obj = obj; /* TODO: can we eliminate it? */ - - /* header is fixed; shall poison again */ - asan_poison_memory_region(header, sizeof *header); - ptr = header + 1; - - theap->total_objects++; /* statistics */ - -#if TRANSIENT_HEAP_DEBUG_DONT_PROMOTE - if (RB_OBJ_PROMOTED_RAW(obj)) { - transient_heap_promote_add(theap, obj); - } -#endif - if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: header:%p ptr:%p size:%d obj:%s\n", (void *)header, ptr, (int)size, rb_obj_info(obj)); - - RB_DEBUG_COUNTER_INC(theap_alloc); - - /* ptr is set up; OK to unpoison. */ - asan_unpoison_memory_region(ptr, size - sizeof *header, true); - ret = ptr; - } - else { - if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [no enough space: %ld] %s\n", (long)size, rb_obj_info(obj)); - RB_DEBUG_COUNTER_INC(theap_alloc_fail); - ret = NULL; - } - } - - return ret; -} - -void -Init_TransientHeap(void) -{ - int i, block_num; - struct transient_heap* theap = transient_heap_get(); - -#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK - block_num = 0; -#else - TH_ASSERT(TRANSIENT_HEAP_BLOCK_SIZE * TRANSIENT_HEAP_BLOCK_NUM == TRANSIENT_HEAP_TOTAL_SIZE); - block_num = TRANSIENT_HEAP_BLOCK_NUM; -#endif - for (i=0; i<block_num; i++) { - connect_to_free_blocks(theap, transient_heap_block_alloc(theap)); - } - theap->using_blocks = transient_heap_allocatable_block(theap); - - theap->promoted_objects_size = TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE; - theap->promoted_objects_index = 0; - /* should not use ALLOC_N to be free from GC */ - theap->promoted_objects = malloc(sizeof(VALUE) * theap->promoted_objects_size); - STATIC_ASSERT( - integer_overflow, - sizeof(VALUE) <= SIZE_MAX / TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE); - if (theap->promoted_objects == NULL) rb_bug("Init_TransientHeap: malloc failed."); -} - -static struct transient_heap_block * -blocks_alloc_header_to_block(struct transient_heap *theap, struct transient_heap_block *blocks, struct transient_alloc_header *header) -{ - struct transient_heap_block *block = blocks; - - while (block) { - if (block->buff <= (char *)header && (char *)header < block->buff + TRANSIENT_HEAP_USABLE_SIZE) { - return block; - } - block = block->info.next_block; - } - - return NULL; -} - -static struct transient_heap_block * -alloc_header_to_block_verbose(struct transient_heap *theap, struct transient_alloc_header *header) -{ - struct transient_heap_block *block; - - if ((block = blocks_alloc_header_to_block(theap, theap->marked_blocks, header)) != NULL) { - if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "alloc_header_to_block: found in marked_blocks\n"); - return block; - } - else if ((block = blocks_alloc_header_to_block(theap, theap->using_blocks, header)) != NULL) { - if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "alloc_header_to_block: found in using_blocks\n"); - return block; - } - else { - return NULL; - } -} - -static struct transient_alloc_header * -ptr_to_alloc_header(const void *ptr) -{ - struct transient_alloc_header *header = (void *)ptr; - header -= 1; - return header; -} - -static int -transient_header_managed_ptr_p(struct transient_heap* theap, const void *ptr) -{ - if (alloc_header_to_block_verbose(theap, ptr_to_alloc_header(ptr))) { - return TRUE; - } - else { - return FALSE; - } -} - - -int -rb_transient_heap_managed_ptr_p(const void *ptr) -{ - return transient_header_managed_ptr_p(transient_heap_get(), ptr); -} - -static struct transient_heap_block * -alloc_header_to_block(struct transient_heap *theap, struct transient_alloc_header *header) -{ - struct transient_heap_block *block; -#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK - block = alloc_header_to_block_verbose(theap, header); - if (block == NULL) { - transient_heap_dump(theap); - rb_bug("alloc_header_to_block: not found in mark_blocks (%p)", header); - } -#else - block = (void *)((intptr_t)header & ~(TRANSIENT_HEAP_BLOCK_SIZE-1)); - TH_ASSERT(block == alloc_header_to_block_verbose(theap, header)); -#endif - return block; -} - -void -rb_transient_heap_mark(VALUE obj, const void *ptr) -{ - ASSERT_vm_locking(); - - struct transient_alloc_header *header = ptr_to_alloc_header(ptr); - asan_unpoison_memory_region(header, sizeof *header, false); - if (header->magic != TRANSIENT_HEAP_ALLOC_MAGIC) rb_bug("rb_transient_heap_mark: wrong header, %s (%p)", rb_obj_info(obj), ptr); - if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_mark: %s (%p)\n", rb_obj_info(obj), ptr); - -#if TRANSIENT_HEAP_CHECK_MODE > 0 - { - struct transient_heap* theap = transient_heap_get(); - TH_ASSERT(theap->status == transient_heap_marking); - TH_ASSERT(transient_header_managed_ptr_p(theap, ptr)); - - if (header->magic != TRANSIENT_HEAP_ALLOC_MAGIC) { - transient_heap_dump(theap); - rb_bug("rb_transient_heap_mark: magic is broken"); - } - else if (header->obj != obj) { - // transient_heap_dump(theap); - rb_bug("rb_transient_heap_mark: unmatch (%s is stored, but %s is given)", - rb_obj_info(header->obj), rb_obj_info(obj)); - } - } -#endif - - if (header->next_marked_index != TRANSIENT_HEAP_ALLOC_MARKING_FREE) { - /* already marked */ - return; - } - else { - struct transient_heap* theap = transient_heap_get(); - struct transient_heap_block *block = alloc_header_to_block(theap, header); - __asan_unpoison_memory_region(&block->info, sizeof block->info); - header->next_marked_index = block->info.last_marked_index; - block->info.last_marked_index = (int)((char *)header - block->buff); - theap->total_marked_objects++; - - transient_heap_verify(theap); - } -} - -ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static const void *transient_heap_ptr(VALUE obj, int error)); -static const void * -transient_heap_ptr(VALUE obj, int error) -{ - const void *ptr = NULL; - - switch (BUILTIN_TYPE(obj)) { - case T_ARRAY: - if (RARRAY_TRANSIENT_P(obj)) { - TH_ASSERT(!ARY_EMBED_P(obj)); - ptr = RARRAY(obj)->as.heap.ptr; - } - break; - case T_OBJECT: - if (ROBJ_TRANSIENT_P(obj)) { - RUBY_ASSERT(!rb_shape_obj_too_complex(obj)); - ptr = ROBJECT_IVPTR(obj); - } - break; - case T_STRUCT: - if (RSTRUCT_TRANSIENT_P(obj)) { - ptr = rb_struct_const_heap_ptr(obj); - } - break; - default: - if (error) { - rb_bug("transient_heap_ptr: unknown obj %s", rb_obj_info(obj)); - } - } - - return ptr; -} - -static void -transient_heap_promote_add(struct transient_heap* theap, VALUE obj) -{ - if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_promote: %s\n", rb_obj_info(obj)); - - if (TRANSIENT_HEAP_DEBUG_DONT_PROMOTE) { - /* duplicate check */ - int i; - for (i=0; i<theap->promoted_objects_index; i++) { - if (theap->promoted_objects[i] == obj) return; - } - } - - if (theap->promoted_objects_size <= theap->promoted_objects_index) { - theap->promoted_objects_size *= 2; - if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "rb_transient_heap_promote: expand table to %d\n", theap->promoted_objects_size); - if (UNLIKELY((size_t)theap->promoted_objects_size > SIZE_MAX / sizeof(VALUE))) { - /* realloc failure due to integer overflow */ - theap->promoted_objects = NULL; - } - else { - theap->promoted_objects = realloc(theap->promoted_objects, theap->promoted_objects_size * sizeof(VALUE)); - } - if (theap->promoted_objects == NULL) rb_bug("rb_transient_heap_promote: realloc failed"); - } - theap->promoted_objects[theap->promoted_objects_index++] = obj; -} - -void -rb_transient_heap_promote(VALUE obj) -{ - ASSERT_vm_locking(); - - if (transient_heap_ptr(obj, FALSE)) { - struct transient_heap* theap = transient_heap_get(); - transient_heap_promote_add(theap, obj); - } - else { - /* ignore */ - } -} - -static struct transient_alloc_header * -alloc_header(struct transient_heap_block* block, int index) -{ - return (void *)&block->buff[index]; -} - -static void -transient_heap_reset(void) -{ - ASSERT_vm_locking(); - - struct transient_heap* theap = transient_heap_get(); - struct transient_heap_block* block; - - if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_reset\n"); - - block = theap->marked_blocks; - while (block) { - struct transient_heap_block *next_block = block->info.next_block; - theap->total_objects -= block->info.objects; -#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK - if (madvise(block, TRANSIENT_HEAP_BLOCK_SIZE, MADV_DONTNEED) != 0) { - rb_bug("madvise err:%d", errno); - } - if (mprotect(block, TRANSIENT_HEAP_BLOCK_SIZE, PROT_NONE) != 0) { - rb_bug("mprotect err:%d", errno); - } -#else - reset_block(block); - connect_to_free_blocks(theap, block); -#endif - theap->total_blocks--; - block = next_block; - } - - if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_reset block_num:%d\n", theap->total_blocks); - - theap->marked_blocks = NULL; - theap->total_marked_objects = 0; -} - -static void -transient_heap_block_evacuate(struct transient_heap* theap, struct transient_heap_block* block) -{ - int marked_index = block->info.last_marked_index; - block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST; - - while (marked_index >= 0) { - struct transient_alloc_header *header = alloc_header(block, marked_index); - asan_unpoison_memory_region(header, sizeof *header, true); - VALUE obj = header->obj; - TH_ASSERT(header->magic == TRANSIENT_HEAP_ALLOC_MAGIC); - if (header->magic != TRANSIENT_HEAP_ALLOC_MAGIC) rb_bug("transient_heap_block_evacuate: wrong header %p %s", (void *)header, rb_obj_info(obj)); - - if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, " * transient_heap_block_evacuate %p %s\n", (void *)header, rb_obj_info(obj)); - - if (obj != Qnil) { - RB_DEBUG_COUNTER_INC(theap_evacuate); - - switch (BUILTIN_TYPE(obj)) { - case T_ARRAY: - rb_ary_transient_heap_evacuate(obj, !TRANSIENT_HEAP_DEBUG_DONT_PROMOTE); - break; - case T_OBJECT: - rb_obj_transient_heap_evacuate(obj, !TRANSIENT_HEAP_DEBUG_DONT_PROMOTE); - break; - case T_STRUCT: - rb_struct_transient_heap_evacuate(obj, !TRANSIENT_HEAP_DEBUG_DONT_PROMOTE); - break; - default: - rb_bug("unsupported: %s", rb_obj_info(obj)); - } - header->obj = Qundef; /* for debug */ - } - marked_index = header->next_marked_index; - asan_poison_memory_region(header, sizeof *header); - } -} - -#if USE_RUBY_DEBUG_LOG -static const char * -transient_heap_status_cstr(enum transient_heap_status status) -{ - switch (status) { - case transient_heap_none: return "none"; - case transient_heap_marking: return "marking"; - case transient_heap_escaping: return "escaping"; - } - UNREACHABLE_RETURN(NULL); -} -#endif - -static void -transient_heap_update_status(struct transient_heap* theap, enum transient_heap_status status) -{ - RUBY_DEBUG_LOG("%s -> %s", - transient_heap_status_cstr(theap->status), - transient_heap_status_cstr(status)); - - TH_ASSERT(theap->status != status); - theap->status = status; -} - -static void -transient_heap_evacuate(void *dmy) -{ - struct transient_heap* theap = transient_heap_get(); - - if (theap->total_marked_objects == 0) return; - if (ruby_single_main_ractor == NULL) rb_bug("not single ractor mode"); - if (theap->status == transient_heap_marking) { - if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_evacuate: skip while transient_heap_marking\n"); - } - else { - VALUE gc_disabled = rb_gc_disable_no_rest(); - { - struct transient_heap_block* block; - - RUBY_DEBUG_LOG("start gc_disabled:%d", RTEST(gc_disabled)); - - if (TRANSIENT_HEAP_DEBUG >= 1) { - int i; - fprintf(stderr, "!! transient_heap_evacuate start total_blocks:%d\n", theap->total_blocks); - if (TRANSIENT_HEAP_DEBUG >= 4) { - for (i=0; i<theap->promoted_objects_index; i++) fprintf(stderr, "%4d %s\n", i, rb_obj_info(theap->promoted_objects[i])); - } - } - if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap); - - TH_ASSERT(theap->status == transient_heap_none); - transient_heap_update_status(theap, transient_heap_escaping); - - /* evacuate from marked blocks */ - block = theap->marked_blocks; - while (block) { - transient_heap_block_evacuate(theap, block); - block = block->info.next_block; - } - - /* evacuate from using blocks - only affect incremental marking */ - block = theap->using_blocks; - while (block) { - transient_heap_block_evacuate(theap, block); - block = block->info.next_block; - } - - /* all objects in marked_objects are escaped. */ - transient_heap_reset(); - - if (TRANSIENT_HEAP_DEBUG > 0) { - fprintf(stderr, "!! transient_heap_evacuate end total_blocks:%d\n", theap->total_blocks); - } - - transient_heap_verify(theap); - transient_heap_update_status(theap, transient_heap_none); - } - if (gc_disabled != Qtrue) rb_gc_enable(); - RUBY_DEBUG_LOG("finish"); - } -} - -void -rb_transient_heap_evacuate(void) -{ - transient_heap_evacuate(NULL); -} - -static void -clear_marked_index(struct transient_heap_block* block) -{ - int marked_index = block->info.last_marked_index; - - while (marked_index != TRANSIENT_HEAP_ALLOC_MARKING_LAST) { - struct transient_alloc_header *header = alloc_header(block, marked_index); - /* header is poisoned to prevent buffer overflow, should - * unpoison first... */ - asan_unpoison_memory_region(header, sizeof *header, false); - TH_ASSERT(marked_index != TRANSIENT_HEAP_ALLOC_MARKING_FREE); - if (0) fprintf(stderr, "clear_marked_index - block:%p mark_index:%d\n", (void *)block, marked_index); - - marked_index = header->next_marked_index; - header->next_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_FREE; - } - - block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST; -} - -static void -blocks_clear_marked_index(struct transient_heap_block* block) -{ - while (block) { - clear_marked_index(block); - block = block->info.next_block; - } -} - -static void -transient_heap_block_update_refs(struct transient_heap* theap, struct transient_heap_block* block) -{ - int marked_index = block->info.last_marked_index; - - while (marked_index >= 0) { - struct transient_alloc_header *header = alloc_header(block, marked_index); - - asan_unpoison_memory_region(header, sizeof *header, false); - - header->obj = rb_gc_location(header->obj); - - marked_index = header->next_marked_index; - asan_poison_memory_region(header, sizeof *header); - } -} - -static void -transient_heap_blocks_update_refs(struct transient_heap* theap, struct transient_heap_block *block, const char *type_str) -{ - while (block) { - transient_heap_block_update_refs(theap, block); - block = block->info.next_block; - } -} - -void -rb_transient_heap_update_references(void) -{ - ASSERT_vm_locking(); - - struct transient_heap* theap = transient_heap_get(); - int i; - - transient_heap_blocks_update_refs(theap, theap->using_blocks, "using_blocks"); - transient_heap_blocks_update_refs(theap, theap->marked_blocks, "marked_blocks"); - - for (i=0; i<theap->promoted_objects_index; i++) { - VALUE obj = theap->promoted_objects[i]; - theap->promoted_objects[i] = rb_gc_location(obj); - } -} - -void -rb_transient_heap_start_marking(int full_marking) -{ - ASSERT_vm_locking(); - RUBY_DEBUG_LOG("full?:%d", full_marking); - - struct transient_heap* theap = transient_heap_get(); - - if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! rb_transient_heap_start_marking objects:%d blocks:%d promoted:%d full_marking:%d\n", - theap->total_objects, theap->total_blocks, theap->promoted_objects_index, full_marking); - if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap); - - blocks_clear_marked_index(theap->marked_blocks); - blocks_clear_marked_index(theap->using_blocks); - - if (theap->using_blocks) { - if (theap->using_blocks->info.objects > 0) { - append_to_marked_blocks(theap, theap->using_blocks); - theap->using_blocks = NULL; - } - else { - append_to_marked_blocks(theap, theap->using_blocks->info.next_block); - theap->using_blocks->info.next_block = NULL; - } - } - - if (theap->using_blocks == NULL) { - theap->using_blocks = transient_heap_allocatable_block(theap); - } - - TH_ASSERT(theap->status == transient_heap_none); - transient_heap_update_status(theap, transient_heap_marking); - theap->total_marked_objects = 0; - - if (full_marking) { - theap->promoted_objects_index = 0; - } - else { /* mark promoted objects */ - int i; - for (i=0; i<theap->promoted_objects_index; i++) { - VALUE obj = theap->promoted_objects[i]; - const void *ptr = transient_heap_ptr(obj, TRUE); - if (ptr) { - rb_transient_heap_mark(obj, ptr); - } - } - } - - transient_heap_verify(theap); -} - -void -rb_transient_heap_finish_marking(void) -{ - ASSERT_vm_locking(); - struct transient_heap* theap = transient_heap_get(); - - RUBY_DEBUG_LOG("objects:%d, marked:%d", - theap->total_objects, - theap->total_marked_objects); - if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap); - - TH_ASSERT(theap->total_objects >= theap->total_marked_objects); - - TH_ASSERT(theap->status == transient_heap_marking); - transient_heap_update_status(theap, transient_heap_none); - - if (theap->total_marked_objects > 0) { - if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "-> rb_transient_heap_finish_marking register escape func.\n"); - rb_postponed_job_register_one(0, transient_heap_evacuate, NULL); - } - else { - transient_heap_reset(); - } - - transient_heap_verify(theap); -} -#endif /* USE_TRANSIENT_HEAP */ diff --git a/transient_heap.h b/transient_heap.h deleted file mode 100644 index 6c6141f71a..0000000000 --- a/transient_heap.h +++ /dev/null @@ -1,63 +0,0 @@ -#ifndef RUBY_TRANSIENT_HEAP_H -#define RUBY_TRANSIENT_HEAP_H -/********************************************************************** - - transient_heap.h - declarations of transient_heap related APIs. - - Copyright (C) 2018 Koichi Sasada - -**********************************************************************/ - -#include "internal.h" - -#if USE_TRANSIENT_HEAP - -/* public API */ - -/* Allocate req_size bytes from transient_heap. - Allocated memories are free-ed when next GC - if this memory is not marked by `rb_transient_heap_mark()`. - */ -void *rb_transient_heap_alloc(VALUE obj, size_t req_size); - -/* If `obj` uses a memory pointed by `ptr` from transient_heap, - you need to call `rb_transient_heap_mark(obj, ptr)` - to assert liveness of `obj` (and ptr). */ -void rb_transient_heap_mark(VALUE obj, const void *ptr); - -/* used by gc.c */ -void rb_transient_heap_promote(VALUE obj); -void rb_transient_heap_start_marking(int full_marking); -void rb_transient_heap_finish_marking(void); -void rb_transient_heap_update_references(void); - -/* used by ractor.c */ -void rb_transient_heap_evacuate(void); - -/* for debug API */ -void rb_transient_heap_dump(void); -void rb_transient_heap_verify(void); -int rb_transient_heap_managed_ptr_p(const void *ptr); - -/* evacuate functions for each type */ -void rb_ary_transient_heap_evacuate(VALUE ary, int promote); -void rb_obj_transient_heap_evacuate(VALUE obj, int promote); -void rb_struct_transient_heap_evacuate(VALUE st, int promote); - -#else /* USE_TRANSIENT_HEAP */ - -#define rb_transient_heap_alloc(o, s) NULL -#define rb_transient_heap_verify() ((void)0) -#define rb_transient_heap_promote(obj) ((void)0) -#define rb_transient_heap_start_marking(full_marking) ((void)0) -#define rb_transient_heap_update_references() ((void)0) -#define rb_transient_heap_evacuate() ((void)0) -#define rb_transient_heap_finish_marking() ((void)0) -#define rb_transient_heap_mark(obj, ptr) ((void)0) - -#define rb_ary_transient_heap_evacuate(x, y) ((void)0) -#define rb_obj_transient_heap_evacuate(x, y) ((void)0) -#define rb_struct_transient_heap_evacuate(x, y) ((void)0) - -#endif /* USE_TRANSIENT_HEAP */ -#endif diff --git a/variable.c b/variable.c index 750ff04e1d..5a7b2f4611 100644 --- a/variable.c +++ b/variable.c @@ -33,7 +33,6 @@ #include "ruby/encoding.h" #include "ruby/st.h" #include "ruby/util.h" -#include "transient_heap.h" #include "shape.h" #include "symbol.h" #include "variable.h" @@ -1399,67 +1398,18 @@ generic_ivar_set(VALUE obj, ID id, VALUE val) static VALUE * obj_ivar_heap_alloc(VALUE obj, size_t newsize) { - VALUE *newptr = rb_transient_heap_alloc(obj, sizeof(VALUE) * newsize); - - if (newptr != NULL) { - ROBJ_TRANSIENT_SET(obj); - } - else { - ROBJ_TRANSIENT_UNSET(obj); - newptr = ALLOC_N(VALUE, newsize); - } - return newptr; + return ALLOC_N(VALUE, newsize); } static VALUE * obj_ivar_heap_realloc(VALUE obj, int32_t len, size_t newsize) { - VALUE *newptr; - int i; - - if (ROBJ_TRANSIENT_P(obj)) { - const VALUE *orig_ptr = ROBJECT(obj)->as.heap.ivptr; - newptr = obj_ivar_heap_alloc(obj, newsize); - - assert(newptr); - ROBJECT(obj)->as.heap.ivptr = newptr; - for (i=0; i<(int)len; i++) { - newptr[i] = orig_ptr[i]; - } - } - else { - REALLOC_N(ROBJECT(obj)->as.heap.ivptr, VALUE, newsize); - newptr = ROBJECT(obj)->as.heap.ivptr; - } + REALLOC_N(ROBJECT(obj)->as.heap.ivptr, VALUE, newsize); + VALUE *newptr = ROBJECT(obj)->as.heap.ivptr; return newptr; } -#if USE_TRANSIENT_HEAP -void -rb_obj_transient_heap_evacuate(VALUE obj, int promote) -{ - if (ROBJ_TRANSIENT_P(obj)) { - assert(!RB_FL_TEST_RAW(obj, ROBJECT_EMBED)); - - uint32_t len = ROBJECT_IV_CAPACITY(obj); - RUBY_ASSERT(!rb_shape_obj_too_complex(obj)); - const VALUE *old_ptr = ROBJECT_IVPTR(obj); - VALUE *new_ptr; - - if (promote) { - new_ptr = ALLOC_N(VALUE, len); - ROBJ_TRANSIENT_UNSET(obj); - } - else { - new_ptr = obj_ivar_heap_alloc(obj, len); - } - MEMCPY(new_ptr, old_ptr, VALUE, len); - ROBJECT(obj)->as.heap.ivptr = new_ptr; - } -} -#endif - void rb_ensure_iv_list_size(VALUE obj, uint32_t current_capacity, uint32_t new_capacity) { @@ -1575,10 +1525,7 @@ rb_obj_ivar_set(VALUE obj, ID id, VALUE val) rb_shape_set_too_complex(obj); RUBY_ASSERT(rb_shape_obj_too_complex(obj)); - if (ROBJ_TRANSIENT_P(obj)) { - ROBJ_TRANSIENT_UNSET(obj); - } - else if (!(RBASIC(obj)->flags & ROBJECT_EMBED)) { + if (!(RBASIC(obj)->flags & ROBJECT_EMBED)) { xfree(ROBJECT(obj)->as.heap.ivptr); } diff --git a/yjit/src/cruby_bindings.inc.rs b/yjit/src/cruby_bindings.inc.rs index bbf1578a45..3a901b105b 100644 --- a/yjit/src/cruby_bindings.inc.rs +++ b/yjit/src/cruby_bindings.inc.rs @@ -244,7 +244,6 @@ pub type st_foreach_callback_func = ::std::option::Option< >; pub const RARRAY_EMBED_FLAG: ruby_rarray_flags = 8192; pub const RARRAY_EMBED_LEN_MASK: ruby_rarray_flags = 4161536; -pub const RARRAY_TRANSIENT_FLAG: ruby_rarray_flags = 33554432; pub type ruby_rarray_flags = u32; pub const RARRAY_EMBED_LEN_SHIFT: ruby_rarray_consts = 15; pub type ruby_rarray_consts = u32; |