diff options
101 files changed, 1416 insertions, 956 deletions
diff --git a/.github/workflows/annocheck.yml b/.github/workflows/annocheck.yml index 2a7c169b3b..dcff2d699a 100644 --- a/.github/workflows/annocheck.yml +++ b/.github/workflows/annocheck.yml @@ -76,7 +76,7 @@ jobs: - uses: ruby/setup-ruby@d8d83c3960843afb664e821fed6be52f37da5267 # v1.231.0 with: - ruby-version: '3.0' + ruby-version: '3.1' bundler: none # Minimal flags to pass the check. diff --git a/.github/workflows/baseruby.yml b/.github/workflows/baseruby.yml index e87641a857..6b3974bc5b 100644 --- a/.github/workflows/baseruby.yml +++ b/.github/workflows/baseruby.yml @@ -45,7 +45,6 @@ jobs: strategy: matrix: ruby: - - ruby-3.0 - ruby-3.1 - ruby-3.2 - ruby-3.3 diff --git a/.github/workflows/check_dependencies.yml b/.github/workflows/check_dependencies.yml index 39b3f2880c..aa3882c165 100644 --- a/.github/workflows/check_dependencies.yml +++ b/.github/workflows/check_dependencies.yml @@ -42,7 +42,7 @@ jobs: - uses: ruby/setup-ruby@d8d83c3960843afb664e821fed6be52f37da5267 # v1.231.0 with: - ruby-version: '3.0' + ruby-version: '3.1' bundler: none - name: Run configure diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 1d9d57090a..3aeed41b0a 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -50,7 +50,6 @@ jobs: env: enable_install_doc: no - CODEQL_ACTION_CLEANUP_TRAP_CACHES: true strategy: fail-fast: false @@ -121,3 +120,9 @@ jobs: with: sarif_file: sarif-results/${{ matrix.language }}.sarif continue-on-error: true + + - name: Purge the oldest TRAP cache + if: ${{ github.repository == 'ruby/ruby' && matrix.language == 'cpp'}} + run: gh cache list --key codeql --order asc --limit 1 --json key --jq '.[].key' | xargs -I{} gh cache delete {} + env: + GH_TOKEN: ${{ secrets.MATZBOT_GITHUB_WORKFLOW_TOKEN }} diff --git a/.github/workflows/compilers.yml b/.github/workflows/compilers.yml index e53c1ec653..55009b9604 100644 --- a/.github/workflows/compilers.yml +++ b/.github/workflows/compilers.yml @@ -320,6 +320,8 @@ jobs: - 'compileB' - 'compileC' steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github } - uses: ./.github/actions/slack with: label: 'omnibus' diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml index 8743785a3b..49cdd8e879 100644 --- a/.github/workflows/mingw.yml +++ b/.github/workflows/mingw.yml @@ -50,7 +50,6 @@ jobs: # To mitigate flakiness of MinGW CI, we test only one runtime that newer MSYS2 uses. # Ruby 3.2 is the first Windows Ruby to use OpenSSL 3.x - msystem: 'UCRT64' - baseruby: '3.2' test_task: 'check' test-all-opts: '--name=!/TestObjSpace#test_reachable_objects_during_iteration/' fail-fast: false @@ -69,7 +68,7 @@ jobs: - name: Set up Ruby & MSYS2 uses: ruby/setup-ruby@d8d83c3960843afb664e821fed6be52f37da5267 # v1.231.0 with: - ruby-version: ${{ matrix.baseruby }} + ruby-version: '3.2' - name: Misc system & package info working-directory: diff --git a/.github/workflows/modgc.yml b/.github/workflows/modgc.yml index d465028339..5354d9bb97 100644 --- a/.github/workflows/modgc.yml +++ b/.github/workflows/modgc.yml @@ -65,7 +65,7 @@ jobs: - uses: ruby/setup-ruby@d8d83c3960843afb664e821fed6be52f37da5267 # v1.231.0 with: - ruby-version: '3.0' + ruby-version: '3.1' bundler: none if: ${{ contains(matrix.os, 'ubuntu') }} diff --git a/.github/workflows/parsey.yml b/.github/workflows/parse_y.yml index 474ec722ff..824dea5d32 100644 --- a/.github/workflows/parsey.yml +++ b/.github/workflows/parse_y.yml @@ -60,6 +60,11 @@ jobs: - uses: ./.github/actions/setup/ubuntu + - uses: ruby/setup-ruby@d8d83c3960843afb664e821fed6be52f37da5267 # v1.231.0 + with: + ruby-version: '3.1' + bundler: none + - uses: ./.github/actions/setup/directories with: srcdir: src diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 6eb7765856..018b7a86f0 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -70,7 +70,7 @@ jobs: - uses: ruby/setup-ruby@d8d83c3960843afb664e821fed6be52f37da5267 # v1.231.0 with: - ruby-version: '3.0' + ruby-version: '3.1' bundler: none if: ${{ !endsWith(matrix.os, 'arm') }} diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index 8861ec7fd6..88ea7bd76c 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -102,7 +102,7 @@ jobs: - uses: ruby/setup-ruby@d8d83c3960843afb664e821fed6be52f37da5267 # v1.231.0 with: - ruby-version: '3.0' + ruby-version: '3.1' bundler: none - name: Build baseruby @@ -152,6 +152,8 @@ jobs: - name: Run basictest run: wasmtime run ./../build/miniruby --mapdir /::./ -- basictest/test.rb + env: + WASMTIME_BACKTRACE_DETAILS: '1' working-directory: src - name: Run bootstraptest (no thread) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 03c0f7a4f5..50b3284a33 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -65,12 +65,12 @@ jobs: - run: md build working-directory: - - uses: ruby/setup-ruby@d8d83c3960843afb664e821fed6be52f37da5267 # v1.231.0 + - uses: ruby/setup-ruby@e34163cd15f4bb403dcd72d98e295997e6a55798 # v1.238.0 with: - ruby-version: '3.0' + # windows-11-arm has only 3.4.1, 3.4.2, 3.4.3, head + ruby-version: ${{ matrix.os != '11-arm' && '3.1' || '3.4' }} bundler: none windows-toolchain: none - if: ${{ matrix.os != '11-arm' }} - name: Install libraries with scoop run: | @@ -155,17 +155,27 @@ jobs: - run: nmake extract-gems # windows-11-arm runner cannot run `ruby tool/file2lastrev.rb --revision.h --output=revision.h` - - run: | - Set-Content -Path "revision.h" -Value @" - #define RUBY_REVISION "8aedb979da" - #define RUBY_FULL_REVISION "8aedb979da4090116f4fc5a6497f139fd0038881" - #define RUBY_BRANCH_NAME "win-arm" - #define RUBY_RELEASE_DATETIME "2025-04-16T23:18:54Z" - #define RUBY_RELEASE_YEAR 2025 - #define RUBY_RELEASE_MONTH 4 - #define RUBY_RELEASE_DAY 17 - "@ - shell: pwsh + - name: make revision.h + run: | + for /f "tokens=1-3" %%I in ('git log -1 "--date=format-local:%%F %%T" "--format=%%H %%cd" @') do ( + set rev=%%I + set dt=%%J + set tm=%%K + ) + set yy=%dt:~0,4% + set /a mm=100%dt:~5,2% %% 100 + set /a dd=100%dt:~8,2% %% 100 + ( + echo #define RUBY_REVISION "%rev:~,10%" + echo #define RUBY_FULL_REVISION "%rev%" + echo #define RUBY_BRANCH_NAME "%GITHUB_REF%" + echo #define RUBY_RELEASE_DATETIME "%dt%T%tm%" + echo #define RUBY_RELEASE_YEAR %yy% + echo #define RUBY_RELEASE_MONTH %mm% + echo #define RUBY_RELEASE_DAY %dd% + ) > revision.h + env: + TZ: UTC if: ${{ matrix.os == '11-arm' }} - run: nmake diff --git a/.github/workflows/yjit-ubuntu.yml b/.github/workflows/yjit-ubuntu.yml index dd1f745944..c148956c36 100644 --- a/.github/workflows/yjit-ubuntu.yml +++ b/.github/workflows/yjit-ubuntu.yml @@ -137,7 +137,7 @@ jobs: - uses: ruby/setup-ruby@d8d83c3960843afb664e821fed6be52f37da5267 # v1.231.0 with: - ruby-version: '3.0' + ruby-version: '3.1' bundler: none - uses: ./.github/actions/setup/directories diff --git a/.github/workflows/zjit-ubuntu.yml b/.github/workflows/zjit-ubuntu.yml index 2923fcf7bd..46a794f095 100644 --- a/.github/workflows/zjit-ubuntu.yml +++ b/.github/workflows/zjit-ubuntu.yml @@ -75,7 +75,7 @@ jobs: - uses: ruby/setup-ruby@a6e6f86333f0a2523ece813039b8b4be04560854 # v1.190.0 with: - ruby-version: '3.0' + ruby-version: '3.1' bundler: none - uses: taiki-e/install-action@v2 @@ -62,10 +62,11 @@ The following default gems are updated. * RubyGems 3.7.0.dev * bundler 2.7.0.dev -* json 2.11.3 +* erb 5.0.0 +* json 2.12.0 * optparse 0.7.0.dev.2 * prism 1.4.0 -* psych 5.2.5 +* psych 5.2.6 * stringio 3.1.8.dev * strscan 3.1.5.dev * uri 1.0.3 @@ -102,11 +103,24 @@ The following bundled gems are updated. ## C API updates +* IO + + * `rb_thread_fd_close` is deprecated and now a no-op. If you need to expose + file descriptors from C extensions to Ruby code, create an `IO` instance + using `RUBY_IO_MODE_EXTERNAL` and use `rb_io_close(io)` to close it (this + also interrupts and waits for all pending operations on the `IO` + instance). Directly closing file descriptors does not interrupt pending + operations, and may lead to undefined beahviour. In other words, if two + `IO` objects share the same file descriptor, closing one does not affect + the other. [[Feature #18455]] + ## Implementation improvements ## JIT +[Feature #18455]: https://bugs.ruby-lang.org/issues/18455 [Feature #19908]: https://bugs.ruby-lang.org/issues/19908 +[Feature #20610]: https://bugs.ruby-lang.org/issues/20610 [Feature #20724]: https://bugs.ruby-lang.org/issues/20724 [Feature #21047]: https://bugs.ruby-lang.org/issues/21047 [Bug #21049]: https://bugs.ruby-lang.org/issues/21049 diff --git a/benchmark/module_eqq.yml b/benchmark/module_eqq.yml index a561fb86dc..2f9c490d92 100644 --- a/benchmark/module_eqq.yml +++ b/benchmark/module_eqq.yml @@ -1,4 +1,5 @@ prelude: | + module SomeModule; end class SimpleClass; end class MediumClass 10.times { include Module.new } @@ -24,4 +25,8 @@ benchmark: SimpleClass === LargeObj simple_class_eqq_huge_obj: | SimpleClass === HugeObj -loop_count: 20000000 + simple_class_eqq_module: | + SimpleClass === HugeObj + module_eqq_module: | + SomeModule === HugeObj +loop_count: 10000000 diff --git a/benchmark/object_allocate.yml b/benchmark/object_allocate.yml index bdbd4536db..c6269923f0 100644 --- a/benchmark/object_allocate.yml +++ b/benchmark/object_allocate.yml @@ -45,4 +45,5 @@ benchmark: allocate_kwarg_params: "KWArg.new(a: 1, b: 2, c: 3, d: 4)" allocate_mixed_params: "Mixed.new(1, 2, c: 3, d: 4)" allocate_no_params: "Object.new" + allocate_allocate: "Object.allocate" loop_count: 100000 diff --git a/bootstraptest/test_ractor.rb b/bootstraptest/test_ractor.rb index 5bf697a5d0..914807246c 100644 --- a/bootstraptest/test_ractor.rb +++ b/bootstraptest/test_ractor.rb @@ -544,7 +544,7 @@ assert_equal '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]', %q{ }.sort } -# an exception in a Ractor will be re-raised at Ractor#receive +# an exception in a Ractor main thread will be re-raised at Ractor#receive assert_equal '[RuntimeError, "ok", true]', %q{ r = Ractor.new do raise 'ok' # exception will be transferred receiver @@ -558,6 +558,18 @@ assert_equal '[RuntimeError, "ok", true]', %q{ end } +# an exception in a Ractor non-main thread will not be re-raised at Ractor#receive +assert_equal 'ok', %q{ + r = Ractor.new do + Thread.new do + raise 'ng' + end + sleep 0.1 + 'ok' + end + r.take +} + # threads in a ractor will killed assert_equal '{ok: 3}', %q{ Ractor.new Ractor.current do |main| @@ -1564,6 +1576,24 @@ assert_equal '1', %q{ }.take } +# Ractor-local storage +assert_equal '2', %q{ + Ractor.new { + fails = 0 + begin + Ractor.main[:key] # cannot get ractor local storage from non-main ractor + rescue => e + fails += 1 if e.message =~ /Cannot get ractor local/ + end + begin + Ractor.main[:key] = 'val' + rescue => e + fails += 1 if e.message =~ /Cannot set ractor local/ + end + fails + }.take +} + ### ### Synchronization tests ### @@ -2276,3 +2306,97 @@ assert_equal 'ok', %q{ 'ok' } + +# There are some bugs in Windows with multiple threads in same ractor calling ractor actions +# Ex: https://github.com/ruby/ruby/actions/runs/14998660285/job/42139383905 +unless /mswin/ =~ RUBY_PLATFORM + # r.send and r.take from multiple threads + # [Bug #21037] + assert_equal '[true, true]', %q{ + class Map + def initialize + @r = Ractor.new { + loop do + key = Ractor.receive + Ractor.yield key + end + } + end + + def fetch(key) + @r.send key + @r.take + end + end + + tm = Map.new + t1 = Thread.new { 10.times.map { tm.fetch("t1") } } + t2 = Thread.new { 10.times.map { tm.fetch("t2") } } + vals = t1.value + t2.value + [ + vals.first(10).all? { |v| v == "t1" }, + vals.last(10).all? { |v| v == "t2" } + ] + } + + # r.send and Ractor.select from multiple threads + assert_equal '[true, true]', %q{ + class Map + def initialize + @r = Ractor.new { + loop do + key = Ractor.receive + Ractor.yield key + end + } + end + + def fetch(key) + @r.send key + _r, val = Ractor.select(@r) + val + end + end + + tm = Map.new + t1 = Thread.new { 10.times.map { tm.fetch("t1") } } + t2 = Thread.new { 10.times.map { tm.fetch("t2") } } + vals = t1.value + t2.value + [ + vals.first(10).all? { |v| v == "t1" }, + vals.last(10).all? { |v| v == "t2" } + ] + } + + # Ractor.receive in multiple threads in same ractor + # [Bug #17624] + assert_equal '["T1 received", "T2 received"]', %q{ + r1 = Ractor.new do + output = [] + m = Mutex.new + # Start two listener threads + t1 = Thread.new do + Ractor.receive + m.synchronize do + output << "T1 received" + end + end + t2 = Thread.new do + Ractor.receive + m.synchronize do + output << "T2 received" + end + end + sleep 0.1 until [t1,t2].all? { |t| t.status == "sleep" } + Ractor.main.send(:both_blocking) + + [t1, t2].each(&:join) + output + end + + Ractor.receive # wait until both threads have blocked + r1.send(1) + r1.send(2) + r1.take.sort + } +end @@ -861,7 +861,10 @@ clean-platform distclean-platform realclean-platform: RUBYSPEC_CAPIEXT = spec/ruby/optional/capi/ext RUBYSPEC_CAPIEXT_SRCDIR = $(srcdir)/$(RUBYSPEC_CAPIEXT) -RUBYSPEC_CAPIEXT_DEPS = $(RUBYSPEC_CAPIEXT_SRCDIR)/rubyspec.h $(RUBY_H_INCLUDES) $(LIBRUBY) build-ext +RUBYSPEC_CAPIEXT_DEPS = $(RUBYSPEC_CAPIEXT_SRCDIR)/rubyspec.h $(RUBY_H_INCLUDES) $(LIBRUBY) + +rubyspec-capiext: build-ext $(DOT_WAIT) +# make-dependent rules should be included after this and built after build-ext. clean-spec: PHONY -$(Q) $(RM) $(RUBYSPEC_CAPIEXT)/*.$(OBJEXT) $(RUBYSPEC_CAPIEXT)/*.$(DLEXT) diff --git a/configure.ac b/configure.ac index 4bcd614f44..3c01b51239 100644 --- a/configure.ac +++ b/configure.ac @@ -1250,8 +1250,6 @@ main() ac_cv_func_gmtime_r=yes rb_cv_large_fd_select=yes ac_cv_type_struct_timeval=yes - ac_cv_func_clock_gettime=yes - ac_cv_func_clock_getres=yes ac_cv_func_malloc_usable_size=no ac_cv_type_off_t=yes ac_cv_sizeof_off_t=8 diff --git a/defs/gmake.mk b/defs/gmake.mk index 666e057c9d..6e696c4631 100644 --- a/defs/gmake.mk +++ b/defs/gmake.mk @@ -506,7 +506,8 @@ update-deps: # order-only-prerequisites doesn't work for $(RUBYSPEC_CAPIEXT) # because the same named directory exists in the source tree. -$(RUBYSPEC_CAPIEXT)/%.$(DLEXT): $(srcdir)/$(RUBYSPEC_CAPIEXT)/%.c $(RUBYSPEC_CAPIEXT_DEPS) +$(RUBYSPEC_CAPIEXT)/%.$(DLEXT): $(srcdir)/$(RUBYSPEC_CAPIEXT)/%.c $(RUBYSPEC_CAPIEXT_DEPS) \ + | build-ext $(ECHO) building $@ $(Q) $(MAKEDIRS) $(@D) $(Q) $(DLDSHARED) -L. $(XDLDFLAGS) $(XLDFLAGS) $(LDFLAGS) $(INCFLAGS) $(CPPFLAGS) $(OUTFLAG)$@ $< $(LIBRUBYARG) diff --git a/ext/-test-/thread_fd/depend b/ext/-test-/thread_fd/depend deleted file mode 100644 index 0fda9f6dbf..0000000000 --- a/ext/-test-/thread_fd/depend +++ /dev/null @@ -1,161 +0,0 @@ -# AUTOGENERATED DEPENDENCIES START -thread_fd.o: $(RUBY_EXTCONF_H) -thread_fd.o: $(arch_hdrdir)/ruby/config.h -thread_fd.o: $(hdrdir)/ruby/assert.h -thread_fd.o: $(hdrdir)/ruby/backward.h -thread_fd.o: $(hdrdir)/ruby/backward/2/assume.h -thread_fd.o: $(hdrdir)/ruby/backward/2/attributes.h -thread_fd.o: $(hdrdir)/ruby/backward/2/bool.h -thread_fd.o: $(hdrdir)/ruby/backward/2/inttypes.h -thread_fd.o: $(hdrdir)/ruby/backward/2/limits.h -thread_fd.o: $(hdrdir)/ruby/backward/2/long_long.h -thread_fd.o: $(hdrdir)/ruby/backward/2/stdalign.h -thread_fd.o: $(hdrdir)/ruby/backward/2/stdarg.h -thread_fd.o: $(hdrdir)/ruby/defines.h -thread_fd.o: $(hdrdir)/ruby/intern.h -thread_fd.o: $(hdrdir)/ruby/internal/abi.h -thread_fd.o: $(hdrdir)/ruby/internal/anyargs.h -thread_fd.o: $(hdrdir)/ruby/internal/arithmetic.h -thread_fd.o: $(hdrdir)/ruby/internal/arithmetic/char.h -thread_fd.o: $(hdrdir)/ruby/internal/arithmetic/double.h -thread_fd.o: $(hdrdir)/ruby/internal/arithmetic/fixnum.h -thread_fd.o: $(hdrdir)/ruby/internal/arithmetic/gid_t.h -thread_fd.o: $(hdrdir)/ruby/internal/arithmetic/int.h -thread_fd.o: $(hdrdir)/ruby/internal/arithmetic/intptr_t.h -thread_fd.o: $(hdrdir)/ruby/internal/arithmetic/long.h -thread_fd.o: $(hdrdir)/ruby/internal/arithmetic/long_long.h -thread_fd.o: $(hdrdir)/ruby/internal/arithmetic/mode_t.h -thread_fd.o: $(hdrdir)/ruby/internal/arithmetic/off_t.h -thread_fd.o: $(hdrdir)/ruby/internal/arithmetic/pid_t.h -thread_fd.o: $(hdrdir)/ruby/internal/arithmetic/short.h -thread_fd.o: $(hdrdir)/ruby/internal/arithmetic/size_t.h -thread_fd.o: $(hdrdir)/ruby/internal/arithmetic/st_data_t.h -thread_fd.o: $(hdrdir)/ruby/internal/arithmetic/uid_t.h -thread_fd.o: $(hdrdir)/ruby/internal/assume.h -thread_fd.o: $(hdrdir)/ruby/internal/attr/alloc_size.h -thread_fd.o: $(hdrdir)/ruby/internal/attr/artificial.h -thread_fd.o: $(hdrdir)/ruby/internal/attr/cold.h -thread_fd.o: $(hdrdir)/ruby/internal/attr/const.h -thread_fd.o: $(hdrdir)/ruby/internal/attr/constexpr.h -thread_fd.o: $(hdrdir)/ruby/internal/attr/deprecated.h -thread_fd.o: $(hdrdir)/ruby/internal/attr/diagnose_if.h -thread_fd.o: $(hdrdir)/ruby/internal/attr/enum_extensibility.h -thread_fd.o: $(hdrdir)/ruby/internal/attr/error.h -thread_fd.o: $(hdrdir)/ruby/internal/attr/flag_enum.h -thread_fd.o: $(hdrdir)/ruby/internal/attr/forceinline.h -thread_fd.o: $(hdrdir)/ruby/internal/attr/format.h -thread_fd.o: $(hdrdir)/ruby/internal/attr/maybe_unused.h -thread_fd.o: $(hdrdir)/ruby/internal/attr/noalias.h -thread_fd.o: $(hdrdir)/ruby/internal/attr/nodiscard.h -thread_fd.o: $(hdrdir)/ruby/internal/attr/noexcept.h -thread_fd.o: $(hdrdir)/ruby/internal/attr/noinline.h -thread_fd.o: $(hdrdir)/ruby/internal/attr/nonnull.h -thread_fd.o: $(hdrdir)/ruby/internal/attr/noreturn.h -thread_fd.o: $(hdrdir)/ruby/internal/attr/packed_struct.h -thread_fd.o: $(hdrdir)/ruby/internal/attr/pure.h -thread_fd.o: $(hdrdir)/ruby/internal/attr/restrict.h -thread_fd.o: $(hdrdir)/ruby/internal/attr/returns_nonnull.h -thread_fd.o: $(hdrdir)/ruby/internal/attr/warning.h -thread_fd.o: $(hdrdir)/ruby/internal/attr/weakref.h -thread_fd.o: $(hdrdir)/ruby/internal/cast.h -thread_fd.o: $(hdrdir)/ruby/internal/compiler_is.h -thread_fd.o: $(hdrdir)/ruby/internal/compiler_is/apple.h -thread_fd.o: $(hdrdir)/ruby/internal/compiler_is/clang.h -thread_fd.o: $(hdrdir)/ruby/internal/compiler_is/gcc.h -thread_fd.o: $(hdrdir)/ruby/internal/compiler_is/intel.h -thread_fd.o: $(hdrdir)/ruby/internal/compiler_is/msvc.h -thread_fd.o: $(hdrdir)/ruby/internal/compiler_is/sunpro.h -thread_fd.o: $(hdrdir)/ruby/internal/compiler_since.h -thread_fd.o: $(hdrdir)/ruby/internal/config.h -thread_fd.o: $(hdrdir)/ruby/internal/constant_p.h -thread_fd.o: $(hdrdir)/ruby/internal/core.h -thread_fd.o: $(hdrdir)/ruby/internal/core/rarray.h -thread_fd.o: $(hdrdir)/ruby/internal/core/rbasic.h -thread_fd.o: $(hdrdir)/ruby/internal/core/rbignum.h -thread_fd.o: $(hdrdir)/ruby/internal/core/rclass.h -thread_fd.o: $(hdrdir)/ruby/internal/core/rdata.h -thread_fd.o: $(hdrdir)/ruby/internal/core/rfile.h -thread_fd.o: $(hdrdir)/ruby/internal/core/rhash.h -thread_fd.o: $(hdrdir)/ruby/internal/core/robject.h -thread_fd.o: $(hdrdir)/ruby/internal/core/rregexp.h -thread_fd.o: $(hdrdir)/ruby/internal/core/rstring.h -thread_fd.o: $(hdrdir)/ruby/internal/core/rstruct.h -thread_fd.o: $(hdrdir)/ruby/internal/core/rtypeddata.h -thread_fd.o: $(hdrdir)/ruby/internal/ctype.h -thread_fd.o: $(hdrdir)/ruby/internal/dllexport.h -thread_fd.o: $(hdrdir)/ruby/internal/dosish.h -thread_fd.o: $(hdrdir)/ruby/internal/error.h -thread_fd.o: $(hdrdir)/ruby/internal/eval.h -thread_fd.o: $(hdrdir)/ruby/internal/event.h -thread_fd.o: $(hdrdir)/ruby/internal/fl_type.h -thread_fd.o: $(hdrdir)/ruby/internal/gc.h -thread_fd.o: $(hdrdir)/ruby/internal/glob.h -thread_fd.o: $(hdrdir)/ruby/internal/globals.h -thread_fd.o: $(hdrdir)/ruby/internal/has/attribute.h -thread_fd.o: $(hdrdir)/ruby/internal/has/builtin.h -thread_fd.o: $(hdrdir)/ruby/internal/has/c_attribute.h -thread_fd.o: $(hdrdir)/ruby/internal/has/cpp_attribute.h -thread_fd.o: $(hdrdir)/ruby/internal/has/declspec_attribute.h -thread_fd.o: $(hdrdir)/ruby/internal/has/extension.h -thread_fd.o: $(hdrdir)/ruby/internal/has/feature.h -thread_fd.o: $(hdrdir)/ruby/internal/has/warning.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/array.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/bignum.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/class.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/compar.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/complex.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/cont.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/dir.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/enum.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/enumerator.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/error.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/eval.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/file.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/hash.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/io.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/load.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/marshal.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/numeric.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/object.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/parse.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/proc.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/process.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/random.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/range.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/rational.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/re.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/ruby.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/select.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/select/largesize.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/signal.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/sprintf.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/string.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/struct.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/thread.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/time.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/variable.h -thread_fd.o: $(hdrdir)/ruby/internal/intern/vm.h -thread_fd.o: $(hdrdir)/ruby/internal/interpreter.h -thread_fd.o: $(hdrdir)/ruby/internal/iterator.h -thread_fd.o: $(hdrdir)/ruby/internal/memory.h -thread_fd.o: $(hdrdir)/ruby/internal/method.h -thread_fd.o: $(hdrdir)/ruby/internal/module.h -thread_fd.o: $(hdrdir)/ruby/internal/newobj.h -thread_fd.o: $(hdrdir)/ruby/internal/scan_args.h -thread_fd.o: $(hdrdir)/ruby/internal/special_consts.h -thread_fd.o: $(hdrdir)/ruby/internal/static_assert.h -thread_fd.o: $(hdrdir)/ruby/internal/stdalign.h -thread_fd.o: $(hdrdir)/ruby/internal/stdbool.h -thread_fd.o: $(hdrdir)/ruby/internal/stdckdint.h -thread_fd.o: $(hdrdir)/ruby/internal/symbol.h -thread_fd.o: $(hdrdir)/ruby/internal/value.h -thread_fd.o: $(hdrdir)/ruby/internal/value_type.h -thread_fd.o: $(hdrdir)/ruby/internal/variable.h -thread_fd.o: $(hdrdir)/ruby/internal/warning_push.h -thread_fd.o: $(hdrdir)/ruby/internal/xmalloc.h -thread_fd.o: $(hdrdir)/ruby/missing.h -thread_fd.o: $(hdrdir)/ruby/ruby.h -thread_fd.o: $(hdrdir)/ruby/st.h -thread_fd.o: $(hdrdir)/ruby/subst.h -thread_fd.o: thread_fd.c -# AUTOGENERATED DEPENDENCIES END diff --git a/ext/-test-/thread_fd/extconf.rb b/ext/-test-/thread_fd/extconf.rb deleted file mode 100644 index a8bbe9d169..0000000000 --- a/ext/-test-/thread_fd/extconf.rb +++ /dev/null @@ -1,2 +0,0 @@ -# frozen_string_literal: true -create_makefile('-test-/thread_fd') diff --git a/ext/-test-/thread_fd/thread_fd.c b/ext/-test-/thread_fd/thread_fd.c deleted file mode 100644 index 042b799dc8..0000000000 --- a/ext/-test-/thread_fd/thread_fd.c +++ /dev/null @@ -1,30 +0,0 @@ -#include "ruby/ruby.h" - -static VALUE -thread_fd_close(VALUE ign, VALUE fd) -{ - rb_thread_fd_close(NUM2INT(fd)); - return Qnil; -} - -static VALUE -thread_fd_wait(VALUE ign, VALUE fd) -{ - int ret = rb_thread_wait_fd(NUM2INT(fd)); - return INT2NUM(ret); -} - -static VALUE -thread_fd_writable(VALUE ign, VALUE fd) -{ - int ret = rb_thread_fd_writable(NUM2INT(fd)); - return INT2NUM(ret); -} - -void -Init_thread_fd(void) -{ - rb_define_singleton_method(rb_cIO, "thread_fd_close", thread_fd_close, 1); - rb_define_singleton_method(rb_cIO, "thread_fd_wait", thread_fd_wait, 1); - rb_define_singleton_method(rb_cIO, "thread_fd_writable", thread_fd_writable, 1); -} diff --git a/ext/erb/escape/escape.c b/ext/erb/escape/escape.c index 67b2d1ef34..db2abd9773 100644 --- a/ext/erb/escape/escape.c +++ b/ext/erb/escape/escape.c @@ -65,9 +65,12 @@ optimized_escape_html(VALUE str) return escaped; } -// ERB::Util.html_escape is different from CGI.escapeHTML in the following two parts: -// * ERB::Util.html_escape converts an argument with #to_s first (only if it's not T_STRING) -// * ERB::Util.html_escape does not allocate a new string when nothing needs to be escaped +/* + * ERB::Util.html_escape is similar to CGI.escapeHTML but different in the following two parts: + * + * * ERB::Util.html_escape converts an argument with #to_s first (only if it's not T_STRING) + * * ERB::Util.html_escape does not allocate a new string when nothing needs to be escaped + */ static VALUE erb_escape_html(VALUE self, VALUE str) { diff --git a/ext/json/lib/json/common.rb b/ext/json/lib/json/common.rb index 98bc0ac85a..7627761b52 100644 --- a/ext/json/lib/json/common.rb +++ b/ext/json/lib/json/common.rb @@ -230,7 +230,9 @@ module JSON class JSONError < StandardError; end # This exception is raised if a parser error occurs. - class ParserError < JSONError; end + class ParserError < JSONError + attr_reader :line, :column + end # This exception is raised if the nesting of parsed data structures is too # deep. diff --git a/ext/json/lib/json/version.rb b/ext/json/lib/json/version.rb index c8744c9732..45d2b0a1fb 100644 --- a/ext/json/lib/json/version.rb +++ b/ext/json/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.11.3' + VERSION = '2.12.0' end diff --git a/ext/json/parser/parser.c b/ext/json/parser/parser.c index f20769a365..c5f300183d 100644 --- a/ext/json/parser/parser.c +++ b/ext/json/parser/parser.c @@ -337,33 +337,128 @@ static size_t strnlen(const char *s, size_t maxlen) } #endif +static int convert_UTF32_to_UTF8(char *buf, uint32_t ch) +{ + int len = 1; + if (ch <= 0x7F) { + buf[0] = (char) ch; + } else if (ch <= 0x07FF) { + buf[0] = (char) ((ch >> 6) | 0xC0); + buf[1] = (char) ((ch & 0x3F) | 0x80); + len++; + } else if (ch <= 0xFFFF) { + buf[0] = (char) ((ch >> 12) | 0xE0); + buf[1] = (char) (((ch >> 6) & 0x3F) | 0x80); + buf[2] = (char) ((ch & 0x3F) | 0x80); + len += 2; + } else if (ch <= 0x1fffff) { + buf[0] =(char) ((ch >> 18) | 0xF0); + buf[1] =(char) (((ch >> 12) & 0x3F) | 0x80); + buf[2] =(char) (((ch >> 6) & 0x3F) | 0x80); + buf[3] =(char) ((ch & 0x3F) | 0x80); + len += 3; + } else { + buf[0] = '?'; + } + return len; +} + +typedef struct JSON_ParserStruct { + VALUE on_load_proc; + VALUE decimal_class; + ID decimal_method_id; + int max_nesting; + bool allow_nan; + bool allow_trailing_comma; + bool parsing_name; + bool symbolize_names; + bool freeze; +} JSON_ParserConfig; + +typedef struct JSON_ParserStateStruct { + VALUE stack_handle; + const char *start; + const char *cursor; + const char *end; + rvalue_stack *stack; + rvalue_cache name_cache; + int in_array; + int current_nesting; +} JSON_ParserState; + + #define PARSE_ERROR_FRAGMENT_LEN 32 #ifdef RBIMPL_ATTR_NORETURN RBIMPL_ATTR_NORETURN() #endif -static void raise_parse_error(const char *format, const char *start) +static void raise_parse_error(const char *format, JSON_ParserState *state) { - unsigned char buffer[PARSE_ERROR_FRAGMENT_LEN + 1]; + unsigned char buffer[PARSE_ERROR_FRAGMENT_LEN + 3]; - size_t len = start ? strnlen(start, PARSE_ERROR_FRAGMENT_LEN) : 0; - const char *ptr = start; + const char *cursor = state->cursor; + long column = 0; + long line = 1; - if (len == PARSE_ERROR_FRAGMENT_LEN) { - MEMCPY(buffer, start, char, PARSE_ERROR_FRAGMENT_LEN); + while (cursor >= state->start) { + if (*cursor-- == '\n') { + break; + } + column++; + } - while (buffer[len - 1] >= 0x80 && buffer[len - 1] < 0xC0) { // Is continuation byte - len--; + while (cursor >= state->start) { + if (*cursor-- == '\n') { + line++; } + } - if (buffer[len - 1] >= 0xC0) { // multibyte character start - len--; + const char *ptr = "EOF"; + if (state->cursor && state->cursor < state->end) { + ptr = state->cursor; + size_t len = 0; + while (len < PARSE_ERROR_FRAGMENT_LEN) { + char ch = ptr[len]; + if (!ch || ch == '\n' || ch == ' ' || ch == '\t' || ch == '\r') { + break; + } + len++; } - buffer[len] = '\0'; - ptr = (const char *)buffer; + if (len) { + buffer[0] = '\''; + MEMCPY(buffer + 1, ptr, char, len); + + while (buffer[len] >= 0x80 && buffer[len] < 0xC0) { // Is continuation byte + len--; + } + + if (buffer[len] >= 0xC0) { // multibyte character start + len--; + } + + buffer[len + 1] = '\''; + buffer[len + 2] = '\0'; + ptr = (const char *)buffer; + } } - rb_enc_raise(enc_utf8, rb_path2class("JSON::ParserError"), format, ptr); + VALUE msg = rb_sprintf(format, ptr); + VALUE message = rb_enc_sprintf(enc_utf8, "%s at line %ld column %ld", RSTRING_PTR(msg), line, column); + RB_GC_GUARD(msg); + + VALUE exc = rb_exc_new_str(rb_path2class("JSON::ParserError"), message); + rb_ivar_set(exc, rb_intern("@line"), LONG2NUM(line)); + rb_ivar_set(exc, rb_intern("@column"), LONG2NUM(column)); + rb_exc_raise(exc); +} + +#ifdef RBIMPL_ATTR_NORETURN +RBIMPL_ATTR_NORETURN() +#endif +static void raise_parse_error_at(const char *format, JSON_ParserState *state, const char *at) +{ + state->cursor = at; + raise_parse_error(format, state); } /* unicode */ @@ -385,73 +480,25 @@ static const signed char digit_values[256] = { -1, -1, -1, -1, -1, -1, -1 }; -static uint32_t unescape_unicode(const unsigned char *p) +static uint32_t unescape_unicode(JSON_ParserState *state, const unsigned char *p) { signed char b; uint32_t result = 0; b = digit_values[p[0]]; - if (b < 0) raise_parse_error("incomplete unicode character escape sequence at '%s'", (char *)p - 2); + if (b < 0) raise_parse_error_at("incomplete unicode character escape sequence at %s", state, (char *)p - 2); result = (result << 4) | (unsigned char)b; b = digit_values[p[1]]; - if (b < 0) raise_parse_error("incomplete unicode character escape sequence at '%s'", (char *)p - 2); + if (b < 0) raise_parse_error_at("incomplete unicode character escape sequence at %s", state, (char *)p - 2); result = (result << 4) | (unsigned char)b; b = digit_values[p[2]]; - if (b < 0) raise_parse_error("incomplete unicode character escape sequence at '%s'", (char *)p - 2); + if (b < 0) raise_parse_error_at("incomplete unicode character escape sequence at %s", state, (char *)p - 2); result = (result << 4) | (unsigned char)b; b = digit_values[p[3]]; - if (b < 0) raise_parse_error("incomplete unicode character escape sequence at '%s'", (char *)p - 2); + if (b < 0) raise_parse_error_at("incomplete unicode character escape sequence at %s", state, (char *)p - 2); result = (result << 4) | (unsigned char)b; return result; } -static int convert_UTF32_to_UTF8(char *buf, uint32_t ch) -{ - int len = 1; - if (ch <= 0x7F) { - buf[0] = (char) ch; - } else if (ch <= 0x07FF) { - buf[0] = (char) ((ch >> 6) | 0xC0); - buf[1] = (char) ((ch & 0x3F) | 0x80); - len++; - } else if (ch <= 0xFFFF) { - buf[0] = (char) ((ch >> 12) | 0xE0); - buf[1] = (char) (((ch >> 6) & 0x3F) | 0x80); - buf[2] = (char) ((ch & 0x3F) | 0x80); - len += 2; - } else if (ch <= 0x1fffff) { - buf[0] =(char) ((ch >> 18) | 0xF0); - buf[1] =(char) (((ch >> 12) & 0x3F) | 0x80); - buf[2] =(char) (((ch >> 6) & 0x3F) | 0x80); - buf[3] =(char) ((ch & 0x3F) | 0x80); - len += 3; - } else { - buf[0] = '?'; - } - return len; -} - -typedef struct JSON_ParserStruct { - VALUE on_load_proc; - VALUE decimal_class; - ID decimal_method_id; - int max_nesting; - bool allow_nan; - bool allow_trailing_comma; - bool parsing_name; - bool symbolize_names; - bool freeze; -} JSON_ParserConfig; - -typedef struct JSON_ParserStateStruct { - VALUE stack_handle; - const char *cursor; - const char *end; - rvalue_stack *stack; - rvalue_cache name_cache; - int in_array; - int current_nesting; -} JSON_ParserState; - #define GET_PARSER_CONFIG \ JSON_ParserConfig *config; \ TypedData_Get_Struct(self, JSON_ParserConfig, &JSON_ParserConfig_type, config) @@ -485,8 +532,7 @@ json_eat_comments(JSON_ParserState *state) while (true) { state->cursor = memchr(state->cursor, '*', state->end - state->cursor); if (!state->cursor) { - state->cursor = state->end; - raise_parse_error("unexpected end of input, expected closing '*/'", state->cursor); + raise_parse_error_at("unexpected end of input, expected closing '*/'", state, state->end); } else { state->cursor++; if (state->cursor < state->end && *state->cursor == '/') { @@ -498,11 +544,11 @@ json_eat_comments(JSON_ParserState *state) break; } default: - raise_parse_error("unexpected token at '%s'", state->cursor); + raise_parse_error("unexpected token %s", state); break; } } else { - raise_parse_error("unexpected token at '%s'", state->cursor); + raise_parse_error("unexpected token %s", state); } } @@ -621,9 +667,9 @@ static VALUE json_string_unescape(JSON_ParserState *state, const char *string, c break; case 'u': if (pe > stringEnd - 5) { - raise_parse_error("incomplete unicode character escape sequence at '%s'", p); + raise_parse_error_at("incomplete unicode character escape sequence at %s", state, p); } else { - uint32_t ch = unescape_unicode((unsigned char *) ++pe); + uint32_t ch = unescape_unicode(state, (unsigned char *) ++pe); pe += 3; /* To handle values above U+FFFF, we take a sequence of * \uXXXX escapes in the U+D800..U+DBFF then @@ -638,10 +684,10 @@ static VALUE json_string_unescape(JSON_ParserState *state, const char *string, c if ((ch & 0xFC00) == 0xD800) { pe++; if (pe > stringEnd - 6) { - raise_parse_error("incomplete surrogate pair at '%s'", p); + raise_parse_error_at("incomplete surrogate pair at %s", state, p); } if (pe[0] == '\\' && pe[1] == 'u') { - uint32_t sur = unescape_unicode((unsigned char *) pe + 2); + uint32_t sur = unescape_unicode(state, (unsigned char *) pe + 2); ch = (((ch & 0x3F) << 10) | ((((ch >> 6) & 0xF) + 1) << 16) | (sur & 0x3FF)); pe += 5; @@ -829,12 +875,12 @@ static inline VALUE json_parse_string(JSON_ParserState *state, JSON_ParserConfig state->cursor++; escaped = true; if ((unsigned char)*state->cursor < 0x20) { - raise_parse_error("invalid ASCII control character in string: %s", state->cursor); + raise_parse_error("invalid ASCII control character in string: %s", state); } break; } default: - raise_parse_error("invalid ASCII control character in string: %s", state->cursor); + raise_parse_error("invalid ASCII control character in string: %s", state); break; } } @@ -842,7 +888,7 @@ static inline VALUE json_parse_string(JSON_ParserState *state, JSON_ParserConfig state->cursor++; } - raise_parse_error("unexpected end of input, expected closing \"", state->cursor); + raise_parse_error("unexpected end of input, expected closing \"", state); return Qfalse; } @@ -850,7 +896,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) { json_eat_whitespace(state); if (state->cursor >= state->end) { - raise_parse_error("unexpected end of input", state->cursor); + raise_parse_error("unexpected end of input", state); } switch (*state->cursor) { @@ -860,7 +906,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) return json_push_value(state, config, Qnil); } - raise_parse_error("unexpected token at '%s'", state->cursor); + raise_parse_error("unexpected token %s", state); break; case 't': if ((state->end - state->cursor >= 4) && (memcmp(state->cursor, "true", 4) == 0)) { @@ -868,7 +914,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) return json_push_value(state, config, Qtrue); } - raise_parse_error("unexpected token at '%s'", state->cursor); + raise_parse_error("unexpected token %s", state); break; case 'f': // Note: memcmp with a small power of two compile to an integer comparison @@ -877,7 +923,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) return json_push_value(state, config, Qfalse); } - raise_parse_error("unexpected token at '%s'", state->cursor); + raise_parse_error("unexpected token %s", state); break; case 'N': // Note: memcmp with a small power of two compile to an integer comparison @@ -886,7 +932,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) return json_push_value(state, config, CNaN); } - raise_parse_error("unexpected token at '%s'", state->cursor); + raise_parse_error("unexpected token %s", state); break; case 'I': if (config->allow_nan && (state->end - state->cursor >= 8) && (memcmp(state->cursor, "Infinity", 8) == 0)) { @@ -894,7 +940,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) return json_push_value(state, config, CInfinity); } - raise_parse_error("unexpected token at '%s'", state->cursor); + raise_parse_error("unexpected token %s", state); break; case '-': // Note: memcmp with a small power of two compile to an integer comparison @@ -903,7 +949,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) state->cursor += 9; return json_push_value(state, config, CMinusInfinity); } else { - raise_parse_error("unexpected token at '%s'", state->cursor); + raise_parse_error("unexpected token %s", state); } } // Fallthrough @@ -921,11 +967,11 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) long integer_length = state->cursor - start; if (RB_UNLIKELY(start[0] == '0' && integer_length > 1)) { - raise_parse_error("invalid number: %s", start); + raise_parse_error_at("invalid number: %s", state, start); } else if (RB_UNLIKELY(integer_length > 2 && start[0] == '-' && start[1] == '0')) { - raise_parse_error("invalid number: %s", start); + raise_parse_error_at("invalid number: %s", state, start); } else if (RB_UNLIKELY(integer_length == 1 && start[0] == '-')) { - raise_parse_error("invalid number: %s", start); + raise_parse_error_at("invalid number: %s", state, start); } if ((state->cursor < state->end) && (*state->cursor == '.')) { @@ -933,7 +979,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) state->cursor++; if (state->cursor == state->end || *state->cursor < '0' || *state->cursor > '9') { - raise_parse_error("invalid number: %s", state->cursor); + raise_parse_error("invalid number: %s", state); } while ((state->cursor < state->end) && (*state->cursor >= '0') && (*state->cursor <= '9')) { @@ -949,7 +995,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) } if (state->cursor == state->end || *state->cursor < '0' || *state->cursor > '9') { - raise_parse_error("invalid number: %s", state->cursor); + raise_parse_error("invalid number: %s", state); } while ((state->cursor < state->end) && (*state->cursor >= '0') && (*state->cursor <= '9')) { @@ -1009,7 +1055,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) } } - raise_parse_error("expected ',' or ']' after array value", state->cursor); + raise_parse_error("expected ',' or ']' after array value", state); } break; } @@ -1028,13 +1074,13 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) } if (*state->cursor != '"') { - raise_parse_error("expected object key, got '%s", state->cursor); + raise_parse_error("expected object key, got %s", state); } json_parse_string(state, config, true); json_eat_whitespace(state); if ((state->cursor >= state->end) || (*state->cursor != ':')) { - raise_parse_error("expected ':' after object key", state->cursor); + raise_parse_error("expected ':' after object key", state); } state->cursor++; @@ -1063,13 +1109,13 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) } if (*state->cursor != '"') { - raise_parse_error("expected object key, got: '%s'", state->cursor); + raise_parse_error("expected object key, got: %s", state); } json_parse_string(state, config, true); json_eat_whitespace(state); if ((state->cursor >= state->end) || (*state->cursor != ':')) { - raise_parse_error("expected ':' after object key, got: '%s", state->cursor); + raise_parse_error("expected ':' after object key, got: %s", state); } state->cursor++; @@ -1079,24 +1125,24 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) } } - raise_parse_error("expected ',' or '}' after object value, got: '%s'", state->cursor); + raise_parse_error("expected ',' or '}' after object value, got: %s", state); } break; } default: - raise_parse_error("unexpected character: '%s'", state->cursor); + raise_parse_error("unexpected character: %s", state); break; } - raise_parse_error("unreacheable: '%s'", state->cursor); + raise_parse_error("unreacheable: %s", state); } static void json_ensure_eof(JSON_ParserState *state) { json_eat_whitespace(state); if (state->cursor != state->end) { - raise_parse_error("unexpected token at end of stream '%s'", state->cursor); + raise_parse_error("unexpected token at end of stream %s", state); } } @@ -1232,9 +1278,14 @@ static VALUE cParser_parse(JSON_ParserConfig *config, VALUE Vsource) .capa = RVALUE_STACK_INITIAL_CAPA, }; + long len; + const char *start; + RSTRING_GETMEM(Vsource, start, len); + JSON_ParserState _state = { - .cursor = RSTRING_PTR(Vsource), - .end = RSTRING_END(Vsource), + .start = start, + .cursor = start, + .end = start + len, .stack = &stack, }; JSON_ParserState *state = &_state; diff --git a/ext/json/vendor/fpconv.c b/ext/json/vendor/fpconv.c index 854cae288c..1bbca28739 100644 --- a/ext/json/vendor/fpconv.c +++ b/ext/json/vendor/fpconv.c @@ -92,7 +92,7 @@ static Fp find_cachedpow10(int exp, int* k) { const double one_log_ten = 0.30102999566398114; - int approx = -(exp + npowers) * one_log_ten; + int approx = (int)(-(exp + npowers) * one_log_ten); int idx = (approx - firstpower) / steppowers; while(1) { @@ -340,7 +340,7 @@ static int emit_digits(char* digits, int ndigits, char* dest, int K, bool neg) } /* write decimal w/o scientific notation */ - if(K < 0 && (K > -7 || exp < 4)) { + if(K < 0 && (K > -7 || exp < 10)) { int offset = ndigits - absv(K); /* fp < 1.0 -> write leading zero */ if(offset <= 0) { diff --git a/ext/psych/lib/psych/versions.rb b/ext/psych/lib/psych/versions.rb index 6c892fbcb4..3202b10296 100644 --- a/ext/psych/lib/psych/versions.rb +++ b/ext/psych/lib/psych/versions.rb @@ -2,7 +2,7 @@ module Psych # The version of Psych you are using - VERSION = '5.2.5' + VERSION = '5.2.6' if RUBY_ENGINE == 'jruby' DEFAULT_SNAKEYAML_VERSION = '2.9'.freeze diff --git a/ext/psych/lib/psych/visitors/yaml_tree.rb b/ext/psych/lib/psych/visitors/yaml_tree.rb index d7958a8431..b6c86f4c94 100644 --- a/ext/psych/lib/psych/visitors/yaml_tree.rb +++ b/ext/psych/lib/psych/visitors/yaml_tree.rb @@ -198,7 +198,7 @@ module Psych @emitter.end_mapping end - end + end unless RUBY_VERSION < "3.2" def visit_Struct o tag = ['!ruby/struct', o.class.name].compact.join(':') diff --git a/ext/stringio/depend b/ext/stringio/depend index b9fba7e9fa..9a8979829b 100644 --- a/ext/stringio/depend +++ b/ext/stringio/depend @@ -171,5 +171,6 @@ stringio.o: $(hdrdir)/ruby/oniguruma.h stringio.o: $(hdrdir)/ruby/ruby.h stringio.o: $(hdrdir)/ruby/st.h stringio.o: $(hdrdir)/ruby/subst.h +stringio.o: $(hdrdir)/ruby/version.h stringio.o: stringio.c # AUTOGENERATED DEPENDENCIES END diff --git a/ext/stringio/stringio.c b/ext/stringio/stringio.c index dd23daa776..26287dd1b8 100644 --- a/ext/stringio/stringio.c +++ b/ext/stringio/stringio.c @@ -20,6 +20,7 @@ STRINGIO_VERSION = "3.1.8.dev"; #include "ruby.h" #include "ruby/io.h" #include "ruby/encoding.h" +#include "ruby/version.h" #if defined(HAVE_FCNTL_H) || defined(_WIN32) #include <fcntl.h> #elif defined(HAVE_SYS_FCNTL_H) @@ -1864,7 +1865,14 @@ strio_set_encoding(int argc, VALUE *argv, VALUE self) } } ptr->enc = enc; - if (!NIL_P(ptr->string) && WRITABLE(self)) { + if (!NIL_P(ptr->string) && WRITABLE(self) +#if (RUBY_API_VERSION_MAJOR == 3 && RUBY_API_VERSION_MINOR >= 4) || RUBY_API_VERSION_MAJOR >= 4 + // Do not attempt to modify chilled strings on Ruby 3.4+ + // RUBY_FL_USER2 == STR_CHILLED_LITERAL + // RUBY_FL_USER3 == STR_CHILLED_SYMBOL_TO_S + && !FL_TEST_RAW(ptr->string, RUBY_FL_USER2 | RUBY_FL_USER3) +#endif + ) { rb_enc_associate(ptr->string, enc); } @@ -1865,7 +1865,6 @@ static VALUE object_id(VALUE obj) { VALUE id = Qfalse; - rb_shape_t *shape = rb_obj_shape(obj); unsigned int lock_lev; // We could avoid locking if the object isn't shareable @@ -1873,16 +1872,17 @@ object_id(VALUE obj) // we'd at least need to generate the object_id using atomics. lock_lev = rb_gc_vm_lock(); - if (rb_shape_has_object_id(shape)) { - rb_shape_t *object_id_shape = rb_shape_object_id_shape(obj); - id = rb_obj_field_get(obj, object_id_shape); + shape_id_t shape_id = rb_obj_shape_id(obj); + shape_id_t object_id_shape_id = rb_shape_transition_object_id(obj); + + if (shape_id >= object_id_shape_id) { + id = rb_obj_field_get(obj, object_id_shape_id); } else { id = ULL2NUM(next_object_id); next_object_id += OBJ_ID_INCREMENT; - rb_shape_t *object_id_shape = rb_shape_object_id_shape(obj); - rb_obj_field_set(obj, object_id_shape, id); + rb_obj_field_set(obj, object_id_shape_id, id); if (RB_UNLIKELY(id_to_obj_tbl)) { st_insert(id_to_obj_tbl, (st_data_t)id, (st_data_t)obj); } @@ -3204,6 +3204,7 @@ rb_gc_mark_children(void *objspace, VALUE obj) gc_mark_internal(RFILE(obj)->fptr->encs.ecopts); gc_mark_internal(RFILE(obj)->fptr->write_lock); gc_mark_internal(RFILE(obj)->fptr->timeout); + gc_mark_internal(RFILE(obj)->fptr->wakeup_mutex); } break; @@ -3717,12 +3718,14 @@ update_superclasses(rb_objspace_t *objspace, rb_classext_t *ext) } static void -update_classext_values(rb_objspace_t *objspace, rb_classext_t *ext) +update_classext_values(rb_objspace_t *objspace, rb_classext_t *ext, bool is_iclass) { UPDATE_IF_MOVED(objspace, RCLASSEXT_ORIGIN(ext)); UPDATE_IF_MOVED(objspace, RCLASSEXT_REFINED_CLASS(ext)); - UPDATE_IF_MOVED(objspace, RCLASSEXT_INCLUDER(ext)); UPDATE_IF_MOVED(objspace, RCLASSEXT_CLASSPATH(ext)); + if (is_iclass) { + UPDATE_IF_MOVED(objspace, RCLASSEXT_INCLUDER(ext)); + } } static void @@ -3756,7 +3759,7 @@ update_classext(rb_classext_t *ext, bool is_prime, VALUE namespace, void *arg) update_superclasses(objspace, ext); update_subclasses(objspace, ext); - update_classext_values(objspace, ext); + update_classext_values(objspace, ext, false); } static void @@ -3773,7 +3776,7 @@ update_iclass_classext(rb_classext_t *ext, bool is_prime, VALUE namespace, void update_cc_tbl(objspace, RCLASSEXT_CC_TBL(ext)); update_subclasses(objspace, ext); - update_classext_values(objspace, ext); + update_classext_values(objspace, ext, true); } extern rb_symbols_t ruby_global_symbols; @@ -4185,6 +4188,8 @@ rb_gc_update_object_references(void *objspace, VALUE obj) UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->writeconv_pre_ecopts); UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->encs.ecopts); UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->write_lock); + UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->timeout); + UPDATE_IF_MOVED(objspace, RFILE(obj)->fptr->wakeup_mutex); } break; case T_REGEXP: @@ -4913,12 +4918,6 @@ rb_asan_poisoned_object_p(VALUE obj) return __asan_region_is_poisoned(ptr, rb_gc_obj_slot_size(obj)); } -#define asan_unpoisoning_object(obj) \ - for (void *poisoned = asan_unpoison_object_temporary(obj), \ - *unpoisoning = &poisoned; /* flag to loop just once */ \ - unpoisoning; \ - unpoisoning = asan_poison_object_restore(obj, poisoned)) - const char * rb_raw_obj_info(char *const buff, const size_t buff_size, VALUE obj) { @@ -880,7 +880,7 @@ ar_general_foreach(VALUE hash, st_foreach_check_callback_func *func, st_update_c return 0; case ST_REPLACE: if (replace) { - retval = (*replace)(&key, &val, arg, TRUE); + (*replace)(&key, &val, arg, TRUE); // TODO: pair should be same as pair before. pair = RHASH_AR_TABLE_REF(hash, i); @@ -951,7 +951,7 @@ ar_foreach_check(VALUE hash, st_foreach_check_callback_func *func, st_data_t arg if (pair->key == never) break; ret = ar_find_entry_hint(hash, hint, key); if (ret == RHASH_AR_TABLE_MAX_BOUND) { - retval = (*func)(0, 0, arg, 1); + (*func)(0, 0, arg, 1); return 2; } } @@ -1404,6 +1404,7 @@ hash_foreach_ensure(VALUE hash) return 0; } +/* This does not manage iteration level */ int rb_hash_stlike_foreach(VALUE hash, st_foreach_callback_func *func, st_data_t arg) { @@ -1415,6 +1416,7 @@ rb_hash_stlike_foreach(VALUE hash, st_foreach_callback_func *func, st_data_t arg } } +/* This does not manage iteration level */ int rb_hash_stlike_foreach_with_replace(VALUE hash, st_foreach_check_callback_func *func, st_update_callback_func *replace, st_data_t arg) { @@ -3486,6 +3488,20 @@ transform_values_foreach_replace(st_data_t *key, st_data_t *value, st_data_t arg return ST_CONTINUE; } +static VALUE +transform_values_call(VALUE hash) +{ + rb_hash_stlike_foreach_with_replace(hash, transform_values_foreach_func, transform_values_foreach_replace, hash); + return hash; +} + +static void +transform_values(VALUE hash) +{ + hash_iter_lev_inc(hash); + rb_ensure(transform_values_call, hash, hash_foreach_ensure, hash); +} + /* * call-seq: * transform_values {|value| ... } -> new_hash @@ -3514,7 +3530,7 @@ rb_hash_transform_values(VALUE hash) SET_DEFAULT(result, Qnil); if (!RHASH_EMPTY_P(hash)) { - rb_hash_stlike_foreach_with_replace(result, transform_values_foreach_func, transform_values_foreach_replace, result); + transform_values(result); compact_after_delete(result); } @@ -3549,7 +3565,7 @@ rb_hash_transform_values_bang(VALUE hash) rb_hash_modify_check(hash); if (!RHASH_TABLE_EMPTY_P(hash)) { - rb_hash_stlike_foreach_with_replace(hash, transform_values_foreach_func, transform_values_foreach_replace, hash); + transform_values(hash); } return hash; diff --git a/include/ruby/internal/fl_type.h b/include/ruby/internal/fl_type.h index 0cad5b673d..701118ef25 100644 --- a/include/ruby/internal/fl_type.h +++ b/include/ruby/internal/fl_type.h @@ -62,7 +62,6 @@ #define FL_TAINT RBIMPL_CAST((VALUE)RUBY_FL_TAINT) /**< @old{RUBY_FL_TAINT} */ #define FL_SHAREABLE RBIMPL_CAST((VALUE)RUBY_FL_SHAREABLE) /**< @old{RUBY_FL_SHAREABLE} */ #define FL_UNTRUSTED RBIMPL_CAST((VALUE)RUBY_FL_UNTRUSTED) /**< @old{RUBY_FL_UNTRUSTED} */ -#define FL_SEEN_OBJ_ID RBIMPL_CAST((VALUE)RUBY_FL_SEEN_OBJ_ID) /**< @old{RUBY_FL_SEEN_OBJ_ID} */ #define FL_EXIVAR RBIMPL_CAST((VALUE)RUBY_FL_EXIVAR) /**< @old{RUBY_FL_EXIVAR} */ #define FL_FREEZE RBIMPL_CAST((VALUE)RUBY_FL_FREEZE) /**< @old{RUBY_FL_FREEZE} */ diff --git a/include/ruby/internal/intern/thread.h b/include/ruby/internal/intern/thread.h index 716375acd7..4d87452745 100644 --- a/include/ruby/internal/intern/thread.h +++ b/include/ruby/internal/intern/thread.h @@ -61,10 +61,10 @@ int rb_thread_wait_fd(int fd); int rb_thread_fd_writable(int fd); /** - * Notifies a closing of a file descriptor to other threads. Multiple threads - * can wait for the given file descriptor at once. If such file descriptor is - * closed, threads need to start propagating their exceptions. This is the API - * to kick that process. + * This funciton is now a no-op. It was previously used to interrupt threads + * that were using the given file descriptor and wait for them to finish. + * + * @deprecated Use IO with RUBY_IO_MODE_EXTERNAL and `rb_io_close` instead. * * @param[in] fd A file descriptor. * @note This function blocks until all the threads waiting for such fd diff --git a/internal/class.h b/internal/class.h index 1652f3b70e..25b02ca3e0 100644 --- a/internal/class.h +++ b/internal/class.h @@ -116,8 +116,10 @@ struct rb_classext_struct { struct { VALUE attached_object; } singleton_class; + struct { + const VALUE includer; + } iclass; } as; - const VALUE includer; attr_index_t max_iv_count; unsigned char variation_count; bool permanent_classpath : 1; @@ -184,7 +186,7 @@ static inline rb_classext_t * RCLASS_EXT_WRITABLE(VALUE obj); #define RCLASSEXT_ORIGIN(ext) (ext->origin_) #define RCLASSEXT_REFINED_CLASS(ext) (ext->refined_class) // class.allocator/singleton_class.attached_object are not accessed directly via RCLASSEXT_* -#define RCLASSEXT_INCLUDER(ext) (ext->includer) +#define RCLASSEXT_INCLUDER(ext) (ext->as.iclass.includer) #define RCLASSEXT_MAX_IV_COUNT(ext) (ext->max_iv_count) #define RCLASSEXT_VARIATION_COUNT(ext) (ext->variation_count) #define RCLASSEXT_PERMANENT_CLASSPATH(ext) (ext->permanent_classpath) @@ -237,7 +239,7 @@ static inline void RCLASSEXT_SET_INCLUDER(rb_classext_t *ext, VALUE klass, VALUE // namespaces don't make changes on these refined_class/attached_object/includer #define RCLASS_REFINED_CLASS(c) (RCLASS_EXT_PRIME(c)->refined_class) #define RCLASS_ATTACHED_OBJECT(c) (RCLASS_EXT_PRIME(c)->as.singleton_class.attached_object) -#define RCLASS_INCLUDER(c) (RCLASS_EXT_PRIME(c)->includer) +#define RCLASS_INCLUDER(c) (RCLASS_EXT_PRIME(c)->as.iclass.includer) // Writable classext entries (instead of RCLASS_SET_*) because member data will be operated directly #define RCLASS_WRITABLE_M_TBL(c) (RCLASS_EXT_WRITABLE(c)->m_tbl) @@ -459,6 +461,7 @@ RCLASSEXT_SET_ORIGIN(rb_classext_t *ext, VALUE klass, VALUE origin) static inline void RCLASSEXT_SET_INCLUDER(rb_classext_t *ext, VALUE klass, VALUE includer) { + RUBY_ASSERT(RB_TYPE_P(klass, T_ICLASS)); RB_OBJ_WRITE(klass, &(RCLASSEXT_INCLUDER(ext)), includer); } @@ -651,7 +654,7 @@ RCLASS_SET_REFINED_CLASS(VALUE klass, VALUE refined) static inline rb_alloc_func_t RCLASS_ALLOCATOR(VALUE klass) { - if (RCLASS_SINGLETON_P(klass)) { + if (RCLASS_SINGLETON_P(klass) || RB_TYPE_P(klass, T_ICLASS)) { return 0; } return RCLASS_EXT_PRIME(klass)->as.class.allocator; @@ -702,6 +705,7 @@ RICLASS_OWNS_M_TBL_P(VALUE iclass) static inline void RCLASS_SET_INCLUDER(VALUE iclass, VALUE klass) { + RUBY_ASSERT(RB_TYPE_P(iclass, T_ICLASS)); RB_OBJ_WRITE(iclass, &RCLASS_INCLUDER(iclass), klass); } diff --git a/internal/io.h b/internal/io.h index 0fcb72ffab..55a8a1a175 100644 --- a/internal/io.h +++ b/internal/io.h @@ -14,10 +14,20 @@ struct rb_io; #include "ruby/io.h" /* for rb_io_t */ +#include "ccan/list/list.h" #define IO_WITHOUT_GVL(func, arg) rb_nogvl(func, arg, RUBY_UBF_IO, 0, RB_NOGVL_OFFLOAD_SAFE) #define IO_WITHOUT_GVL_INT(func, arg) (int)(VALUE)IO_WITHOUT_GVL(func, arg) +// Represents an in-flight blocking operation: +struct rb_io_blocking_operation { + // The linked list data structure. + struct ccan_list_node list; + + // The execution context of the blocking operation: + struct rb_execution_context_struct *ec; +}; + /** Ruby's IO, metadata and buffers. */ struct rb_io { @@ -111,6 +121,15 @@ struct rb_io { * The timeout associated with this IO when performing blocking operations. */ VALUE timeout; + + /** + * Threads that are performing a blocking operation without the GVL using + * this IO. On calling IO#close, these threads will be interrupted so that + * the operation can be cancelled. + */ + struct ccan_list_head blocking_operations; + struct rb_execution_context_struct *closing_ec; + VALUE wakeup_mutex; }; /* io.c */ @@ -125,7 +144,7 @@ VALUE rb_io_prep_stdin(void); VALUE rb_io_prep_stdout(void); VALUE rb_io_prep_stderr(void); -int rb_io_fptr_finalize(struct rb_io *fptr); +int rb_io_notify_close(struct rb_io *fptr); RUBY_SYMBOL_EXPORT_BEGIN /* io.c (export) */ diff --git a/internal/thread.h b/internal/thread.h index 57708455d9..5406a617e4 100644 --- a/internal/thread.h +++ b/internal/thread.h @@ -13,6 +13,7 @@ #include "ccan/list/list.h" /* for list in rb_io_close_wait_list */ struct rb_thread_struct; /* in vm_core.h */ +struct rb_io; #define RB_VM_SAVE_MACHINE_CONTEXT(th) \ do { \ @@ -58,14 +59,8 @@ void ruby_mn_threads_params(void); int rb_thread_io_wait(struct rb_io *io, int events, struct timeval * timeout); int rb_thread_wait_for_single_fd(int fd, int events, struct timeval * timeout); -struct rb_io_close_wait_list { - struct ccan_list_head pending_fd_users; - VALUE closing_thread; - VALUE closing_fiber; - VALUE wakeup_mutex; -}; -int rb_notify_fd_close(int fd, struct rb_io_close_wait_list *busy); -void rb_notify_fd_close_wait(struct rb_io_close_wait_list *busy); +size_t rb_thread_io_close_interrupt(struct rb_io *); +void rb_thread_io_close_wait(struct rb_io *); void rb_ec_check_ints(struct rb_execution_context_struct *ec); diff --git a/internal/variable.h b/internal/variable.h index 31a086f42b..d2432fe22e 100644 --- a/internal/variable.h +++ b/internal/variable.h @@ -53,9 +53,9 @@ void rb_obj_copy_ivs_to_hash_table(VALUE obj, st_table *table); void rb_obj_init_too_complex(VALUE obj, st_table *table); void rb_evict_ivars_to_hash(VALUE obj); void rb_evict_fields_to_hash(VALUE obj); -VALUE rb_obj_field_get(VALUE obj, rb_shape_t *target_shape); +VALUE rb_obj_field_get(VALUE obj, shape_id_t target_shape_id); void rb_ivar_set_internal(VALUE obj, ID id, VALUE val); -void rb_obj_field_set(VALUE obj, rb_shape_t *target_shape, VALUE val); +void rb_obj_field_set(VALUE obj, shape_id_t target_shape_id, VALUE val); RUBY_SYMBOL_EXPORT_BEGIN /* variable.c (export) */ @@ -5517,8 +5517,7 @@ maygvl_fclose(FILE *file, int keepgvl) static void free_io_buffer(rb_io_buffer_t *buf); static void -fptr_finalize_flush(rb_io_t *fptr, int noraise, int keepgvl, - struct rb_io_close_wait_list *busy) +fptr_finalize_flush(rb_io_t *fptr, int noraise, int keepgvl) { VALUE error = Qnil; int fd = fptr->fd; @@ -5558,11 +5557,8 @@ fptr_finalize_flush(rb_io_t *fptr, int noraise, int keepgvl, fptr->stdio_file = 0; fptr->mode &= ~(FMODE_READABLE|FMODE_WRITABLE); - // Ensure waiting_fd users do not hit EBADF. - if (busy) { - // Wait for them to exit before we call close(). - rb_notify_fd_close_wait(busy); - } + // wait for blocking operations to ensure they do not hit EBADF: + rb_thread_io_close_wait(fptr); // Disable for now. // if (!done && fd >= 0) { @@ -5610,7 +5606,7 @@ fptr_finalize_flush(rb_io_t *fptr, int noraise, int keepgvl, static void fptr_finalize(rb_io_t *fptr, int noraise) { - fptr_finalize_flush(fptr, noraise, FALSE, 0); + fptr_finalize_flush(fptr, noraise, FALSE); free_io_buffer(&fptr->rbuf); free_io_buffer(&fptr->wbuf); clear_codeconv(fptr); @@ -5686,14 +5682,20 @@ rb_io_fptr_finalize(struct rb_io *io) } size_t -rb_io_memsize(const rb_io_t *fptr) +rb_io_memsize(const rb_io_t *io) { size_t size = sizeof(rb_io_t); - size += fptr->rbuf.capa; - size += fptr->wbuf.capa; - size += fptr->cbuf.capa; - if (fptr->readconv) size += rb_econv_memsize(fptr->readconv); - if (fptr->writeconv) size += rb_econv_memsize(fptr->writeconv); + size += io->rbuf.capa; + size += io->wbuf.capa; + size += io->cbuf.capa; + if (io->readconv) size += rb_econv_memsize(io->readconv); + if (io->writeconv) size += rb_econv_memsize(io->writeconv); + + struct rb_io_blocking_operation *blocking_operation = 0; + ccan_list_for_each(&io->blocking_operations, blocking_operation, list) { + size += sizeof(struct rb_io_blocking_operation); + } + return size; } @@ -5710,7 +5712,6 @@ io_close_fptr(VALUE io) rb_io_t *fptr; VALUE write_io; rb_io_t *write_fptr; - struct rb_io_close_wait_list busy; write_io = GetWriteIO(io); if (io != write_io) { @@ -5724,9 +5725,9 @@ io_close_fptr(VALUE io) if (!fptr) return 0; if (fptr->fd < 0) return 0; - if (rb_notify_fd_close(fptr->fd, &busy)) { + if (rb_thread_io_close_interrupt(fptr)) { /* calls close(fptr->fd): */ - fptr_finalize_flush(fptr, FALSE, KEEPGVL, &busy); + fptr_finalize_flush(fptr, FALSE, KEEPGVL); } rb_io_fptr_cleanup(fptr, FALSE); return fptr; @@ -8369,6 +8370,10 @@ io_reopen(VALUE io, VALUE nfile) fd = fptr->fd; fd2 = orig->fd; if (fd != fd2) { + // Interrupt all usage of the old file descriptor: + rb_thread_io_close_interrupt(fptr); + rb_thread_io_close_wait(fptr); + if (RUBY_IO_EXTERNAL_P(fptr) || fd <= 2 || !fptr->stdio_file) { /* need to keep FILE objects of stdin, stdout and stderr */ if (rb_cloexec_dup2(fd2, fd) < 0) @@ -8384,7 +8389,7 @@ io_reopen(VALUE io, VALUE nfile) rb_update_max_fd(fd); fptr->fd = fd; } - rb_thread_fd_close(fd); + if ((orig->mode & FMODE_READABLE) && pos >= 0) { if (io_seek(fptr, pos, SEEK_SET) < 0 && errno) { rb_sys_fail_path(fptr->pathv); @@ -8561,6 +8566,11 @@ rb_io_init_copy(VALUE dest, VALUE io) fptr->pid = orig->pid; fptr->lineno = orig->lineno; fptr->timeout = orig->timeout; + + ccan_list_head_init(&fptr->blocking_operations); + fptr->closing_ec = NULL; + fptr->wakeup_mutex = Qnil; + if (!NIL_P(orig->pathv)) fptr->pathv = orig->pathv; fptr_copy_finalizer(fptr, orig); @@ -9298,6 +9308,10 @@ rb_io_open_descriptor(VALUE klass, int descriptor, int mode, VALUE path, VALUE t io->timeout = timeout; + ccan_list_head_init(&io->blocking_operations); + io->closing_ec = NULL; + io->wakeup_mutex = Qnil; + if (encoding) { io->encs = *encoding; } @@ -9437,6 +9451,9 @@ rb_io_fptr_new(void) fp->encs.ecopts = Qnil; fp->write_lock = Qnil; fp->timeout = Qnil; + ccan_list_head_init(&fp->blocking_operations); + fp->closing_ec = NULL; + fp->wakeup_mutex = Qnil; return fp; } @@ -9567,6 +9584,9 @@ io_initialize(VALUE io, VALUE fnum, VALUE vmode, VALUE opt) fp->encs = convconfig; fp->pathv = path; fp->timeout = Qnil; + ccan_list_head_init(&fp->blocking_operations); + fp->closing_ec = NULL; + fp->wakeup_mutex = Qnil; clear_codeconv(fp); io_check_tty(fp); if (fileno(stdin) == fd) @@ -3778,7 +3778,7 @@ rb_vm_encoded_insn_data_table_init(void) const void * const *table = rb_vm_get_insns_address_table(); #define INSN_CODE(insn) ((VALUE)table[insn]) #else -#define INSN_CODE(insn) (insn) +#define INSN_CODE(insn) ((VALUE)(insn)) #endif encoded_insn_data = st_init_numtable_with_size(VM_BARE_INSTRUCTION_SIZE); diff --git a/lib/bundler/cli/doctor/diagnose.rb b/lib/bundler/cli/doctor/diagnose.rb index c5da23acb8..a878025dda 100644 --- a/lib/bundler/cli/doctor/diagnose.rb +++ b/lib/bundler/cli/doctor/diagnose.rb @@ -24,7 +24,7 @@ module Bundler def dylibs_darwin(path) output = `/usr/bin/otool -L #{path.shellescape}`.chomp - dylibs = output.split("\n")[1..-1].map {|l| l.match(DARWIN_REGEX).captures[0] }.uniq + dylibs = output.split("\n")[1..-1].filter_map {|l| l.match(DARWIN_REGEX)&.match(1) }.uniq # ignore @rpath and friends dylibs.reject {|dylib| dylib.start_with? "@" } end diff --git a/lib/bundler/fetcher/dependency.rb b/lib/bundler/fetcher/dependency.rb index c23fa160f4..994b415e9c 100644 --- a/lib/bundler/fetcher/dependency.rb +++ b/lib/bundler/fetcher/dependency.rb @@ -1,11 +1,8 @@ # frozen_string_literal: true require_relative "base" -begin - require "cgi/escape" -rescue LoadError - require "cgi/util" -end +require "cgi/escape" +require "cgi/util" unless defined?(CGI::EscapeExt) module Bundler class Fetcher diff --git a/lib/bundler/friendly_errors.rb b/lib/bundler/friendly_errors.rb index 99893ed1f1..8a5ab2e025 100644 --- a/lib/bundler/friendly_errors.rb +++ b/lib/bundler/friendly_errors.rb @@ -102,11 +102,8 @@ module Bundler def issues_url(exception) message = exception.message.lines.first.tr(":", " ").chomp message = message.split("-").first if exception.is_a?(Errno) - begin - require "cgi/escape" - rescue LoadError - require "cgi/util" - end + require "cgi/escape" + require "cgi/util" unless defined?(CGI::EscapeExt) "https://github.com/rubygems/rubygems/search?q=" \ "#{CGI.escape(message)}&type=Issues" end diff --git a/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent.rb b/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent.rb index 115b49f452..03909a298b 100644 --- a/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent.rb +++ b/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent.rb @@ -1,10 +1,7 @@ require_relative '../../../../../vendored_net_http' require_relative '../../../../../vendored_uri' -begin - require 'cgi/escape' -rescue LoadError - require 'cgi/util' # for escaping -end +require 'cgi/escape' +require 'cgi/util' unless defined?(CGI::EscapeExt) require_relative '../../../../connection_pool/lib/connection_pool' autoload :OpenSSL, 'openssl' @@ -46,9 +43,8 @@ autoload :OpenSSL, 'openssl' # # perform the POST, the Gem::URI is always required # response http.request post_uri, post # -# Note that for GET, HEAD and other requests that do not have a body you want -# to use Gem::URI#request_uri not Gem::URI#path. The request_uri contains the query -# params which are sent in the body for other requests. +# ⚠ Note that for GET, HEAD and other requests that do not have a body, +# it uses Gem::URI#request_uri as default to send query params # # == TLS/SSL # @@ -64,6 +60,7 @@ autoload :OpenSSL, 'openssl' # #ca_path :: Directory with certificate-authorities # #cert_store :: An SSL certificate store # #ciphers :: List of SSl ciphers allowed +# #extra_chain_cert :: Extra certificates to be added to the certificate chain # #private_key :: The client's SSL private key # #reuse_ssl_sessions :: Reuse a previously opened SSL session for a new # connection @@ -180,7 +177,7 @@ class Gem::Net::HTTP::Persistent ## # The version of Gem::Net::HTTP::Persistent you are using - VERSION = '4.0.4' + VERSION = '4.0.6' ## # Error class for errors raised by Gem::Net::HTTP::Persistent. Various @@ -272,6 +269,11 @@ class Gem::Net::HTTP::Persistent attr_reader :ciphers ## + # Extra certificates to be added to the certificate chain + + attr_reader :extra_chain_cert + + ## # Sends debug_output to this IO via Gem::Net::HTTP#set_debug_output. # # Never use this method in production code, it causes a serious security @@ -591,6 +593,21 @@ class Gem::Net::HTTP::Persistent reconnect_ssl end + if Gem::Net::HTTP.method_defined?(:extra_chain_cert=) + ## + # Extra certificates to be added to the certificate chain. + # It is only supported starting from Gem::Net::HTTP version 0.1.1 + def extra_chain_cert= extra_chain_cert + @extra_chain_cert = extra_chain_cert + + reconnect_ssl + end + else + def extra_chain_cert= _extra_chain_cert + raise "extra_chain_cert= is not supported by this version of Gem::Net::HTTP" + end + end + ## # Creates a new connection for +uri+ @@ -609,47 +626,49 @@ class Gem::Net::HTTP::Persistent connection = @pool.checkout net_http_args - http = connection.http + begin + http = connection.http - connection.ressl @ssl_generation if - connection.ssl_generation != @ssl_generation + connection.ressl @ssl_generation if + connection.ssl_generation != @ssl_generation - if not http.started? then - ssl http if use_ssl - start http - elsif expired? connection then - reset connection - end + if not http.started? then + ssl http if use_ssl + start http + elsif expired? connection then + reset connection + end - http.keep_alive_timeout = @idle_timeout if @idle_timeout - http.max_retries = @max_retries if http.respond_to?(:max_retries=) - http.read_timeout = @read_timeout if @read_timeout - http.write_timeout = @write_timeout if - @write_timeout && http.respond_to?(:write_timeout=) + http.keep_alive_timeout = @idle_timeout if @idle_timeout + http.max_retries = @max_retries if http.respond_to?(:max_retries=) + http.read_timeout = @read_timeout if @read_timeout + http.write_timeout = @write_timeout if + @write_timeout && http.respond_to?(:write_timeout=) + + return yield connection + rescue Errno::ECONNREFUSED + if http.proxy? + address = http.proxy_address + port = http.proxy_port + else + address = http.address + port = http.port + end - return yield connection - rescue Errno::ECONNREFUSED - if http.proxy? - address = http.proxy_address - port = http.proxy_port - else - address = http.address - port = http.port - end + raise Error, "connection refused: #{address}:#{port}" + rescue Errno::EHOSTDOWN + if http.proxy? + address = http.proxy_address + port = http.proxy_port + else + address = http.address + port = http.port + end - raise Error, "connection refused: #{address}:#{port}" - rescue Errno::EHOSTDOWN - if http.proxy? - address = http.proxy_address - port = http.proxy_port - else - address = http.address - port = http.port + raise Error, "host down: #{address}:#{port}" + ensure + @pool.checkin net_http_args end - - raise Error, "host down: #{address}:#{port}" - ensure - @pool.checkin net_http_args end ## @@ -957,7 +976,8 @@ class Gem::Net::HTTP::Persistent end ## - # Shuts down all connections + # Shuts down all connections. Attempting to checkout a connection after + # shutdown will raise an error. # # *NOTE*: Calling shutdown for can be dangerous! # @@ -969,6 +989,17 @@ class Gem::Net::HTTP::Persistent end ## + # Discard all existing connections. Subsequent checkouts will create + # new connections as needed. + # + # If any thread is still using a connection it may cause an error! Call + # #reload when you are completely done making requests! + + def reload + @pool.reload { |http| http.finish } + end + + ## # Enables SSL on +connection+ def ssl connection @@ -1025,6 +1056,10 @@ application: connection.key = @private_key end + if defined?(@extra_chain_cert) and @extra_chain_cert + connection.extra_chain_cert = @extra_chain_cert + end + connection.cert_store = if @cert_store then @cert_store else diff --git a/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/timed_stack_multi.rb b/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/timed_stack_multi.rb index 214804fcd9..034fbe39b8 100644 --- a/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/timed_stack_multi.rb +++ b/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/timed_stack_multi.rb @@ -63,7 +63,8 @@ class Gem::Net::HTTP::Persistent::TimedStackMulti < Bundler::ConnectionPool::Tim if @created >= @max && @enqueued >= 1 oldest, = @lru.first @lru.delete oldest - @ques[oldest].pop + connection = @ques[oldest].pop + connection.close if connection.respond_to?(:close) @created -= 1 end diff --git a/lib/erb.gemspec b/lib/erb.gemspec index 94a8fd5c3e..0a59abad53 100644 --- a/lib/erb.gemspec +++ b/lib/erb.gemspec @@ -7,14 +7,13 @@ end Gem::Specification.new do |spec| spec.name = 'erb' - spec.version = ERB.const_get(:VERSION, false) + spec.version = ERB::VERSION spec.authors = ['Masatoshi SEKI', 'Takashi Kokubun'] spec.email = ['[email protected]', '[email protected]'] spec.summary = %q{An easy to use but powerful templating system for Ruby.} spec.description = %q{An easy to use but powerful templating system for Ruby.} spec.homepage = 'https://github.com/ruby/erb' - spec.required_ruby_version = Gem::Requirement.new('>= 2.5.0') spec.licenses = ['Ruby', 'BSD-2-Clause'] spec.metadata['homepage_uri'] = spec.homepage @@ -27,12 +26,11 @@ Gem::Specification.new do |spec| spec.executables = ['erb'] spec.require_paths = ['lib'] + spec.required_ruby_version = '>= 3.2.0' + if RUBY_ENGINE == 'jruby' spec.platform = 'java' else - spec.required_ruby_version = '>= 2.7.0' spec.extensions = ['ext/erb/escape/extconf.rb'] end - - spec.add_dependency 'cgi', '>= 0.3.3' end diff --git a/lib/erb.rb b/lib/erb.rb index dffc6d943e..ebf91e4792 100644 --- a/lib/erb.rb +++ b/lib/erb.rb @@ -12,11 +12,6 @@ # # You can redistribute it and/or modify it under the same terms as Ruby. -begin - require 'cgi/escape' -rescue LoadError - require 'cgi/util' -end require 'erb/version' require 'erb/compiler' require 'erb/def_method' diff --git a/lib/erb/def_method.rb b/lib/erb/def_method.rb index aee989a926..e503b37140 100644 --- a/lib/erb/def_method.rb +++ b/lib/erb/def_method.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true -#-- + # ERB::DefMethod # # Utility module to define eRuby script as instance method. diff --git a/lib/erb/util.rb b/lib/erb/util.rb index 1d2a36275d..42c7a57622 100644 --- a/lib/erb/util.rb +++ b/lib/erb/util.rb @@ -1,17 +1,23 @@ # frozen_string_literal: true -#-- -# ERB::Escape -# -# A subset of ERB::Util. Unlike ERB::Util#html_escape, we expect/hope -# Rails will not monkey-patch ERB::Escape#html_escape. + +# Load CGI.escapeHTML and CGI.escapeURIComponent. +# CRuby: +# cgi.gem v0.1.0+ (Ruby 2.7-3.4) and Ruby 3.5+ stdlib have 'cgi/escape' and CGI.escapeHTML. +# cgi.gem v0.3.3+ (Ruby 3.2-3.4) and Ruby 3.5+ stdlib have CGI.escapeURIComponent. +# JRuby: cgi.gem has a Java extension 'cgi/escape'. +# TruffleRuby: lib/truffle/cgi/escape.rb requires 'cgi/util'. +require 'cgi/escape' + +# Load or define ERB::Escape#html_escape. +# We don't build the C extention 'cgi/escape' for JRuby, TruffleRuby, and WASM. +# miniruby (used by CRuby build scripts) also fails to load erb/escape.so. begin - # We don't build the C extension for JRuby, TruffleRuby, and WASM - if $LOAD_PATH.resolve_feature_path('erb/escape') - require 'erb/escape' - end -rescue LoadError # resolve_feature_path raises LoadError on TruffleRuby 22.3.0 -end -unless defined?(ERB::Escape) + require 'erb/escape' +rescue LoadError + # ERB::Escape + # + # A subset of ERB::Util. Unlike ERB::Util#html_escape, we expect/hope + # Rails will not monkey-patch ERB::Escape#html_escape. module ERB::Escape def html_escape(s) CGI.escapeHTML(s.to_s) @@ -20,7 +26,6 @@ unless defined?(ERB::Escape) end end -#-- # ERB::Util # # A utility module for conversion routines, often handy in HTML generation. @@ -54,8 +59,16 @@ module ERB::Util # # Programming%20Ruby%3A%20%20The%20Pragmatic%20Programmer%27s%20Guide # - def url_encode(s) - CGI.escapeURIComponent(s.to_s) + if CGI.respond_to?(:escapeURIComponent) + def url_encode(s) + CGI.escapeURIComponent(s.to_s) + end + else # cgi.gem <= v0.3.2 + def url_encode(s) + s.to_s.b.gsub(/[^a-zA-Z0-9_\-.~]/n) do |m| + sprintf("%%%02X", m.unpack1("C")) + end + end end alias u url_encode module_function :u diff --git a/lib/erb/version.rb b/lib/erb/version.rb index b5fe39b330..6090303add 100644 --- a/lib/erb/version.rb +++ b/lib/erb/version.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true class ERB - VERSION = '4.0.4' - private_constant :VERSION + VERSION = '5.0.0' end diff --git a/lib/net/http.rb b/lib/net/http.rb index 40ff06edab..635f756b41 100644 --- a/lib/net/http.rb +++ b/lib/net/http.rb @@ -1917,11 +1917,8 @@ module Net #:nodoc: private def unescape(value) - begin - require "cgi/escape" - rescue LoadError - require "cgi/util" - end + require 'cgi/escape' + require 'cgi/util' unless defined?(CGI::EscapeExt) CGI.unescape(value) end diff --git a/lib/rubygems/package.rb b/lib/rubygems/package.rb index 60a8ab24ba..c855423ed7 100644 --- a/lib/rubygems/package.rb +++ b/lib/rubygems/package.rb @@ -219,7 +219,6 @@ class Gem::Package # Adds a checksum for each entry in the gem to checksums.yaml.gz. def add_checksums(tar) - require 'psych' Gem.load_yaml checksums_by_algorithm = Hash.new {|h, algorithm| h[algorithm] = {} } @@ -553,7 +552,6 @@ EOM # Reads and loads checksums.yaml.gz from the tar file +gem+ def read_checksums(gem) - require_relative 'safe_yaml' Gem.load_yaml @checksums = gem.seek "checksums.yaml.gz" do |entry| diff --git a/lib/rubygems/safe_yaml.rb b/lib/rubygems/safe_yaml.rb index ba1a5bcbed..6a02a48230 100644 --- a/lib/rubygems/safe_yaml.rb +++ b/lib/rubygems/safe_yaml.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'psych' - module Gem ### # This module is used for safely loading YAML specs from a gem. The diff --git a/lib/rubygems/uri_formatter.rb b/lib/rubygems/uri_formatter.rb index 2022757689..8856fdadd2 100644 --- a/lib/rubygems/uri_formatter.rb +++ b/lib/rubygems/uri_formatter.rb @@ -17,11 +17,8 @@ class Gem::UriFormatter # Creates a new URI formatter for +uri+. def initialize(uri) - begin - require "cgi/escape" - rescue LoadError - require "cgi/util" - end + require "cgi/escape" + require "cgi/util" unless defined?(CGI::EscapeExt) @uri = uri end diff --git a/lib/rubygems/vendor/net-http/lib/net/http.rb b/lib/rubygems/vendor/net-http/lib/net/http.rb index 2edfcb1723..0e86056614 100644 --- a/lib/rubygems/vendor/net-http/lib/net/http.rb +++ b/lib/rubygems/vendor/net-http/lib/net/http.rb @@ -46,7 +46,7 @@ module Gem::Net #:nodoc: # == Strategies # # - If you will make only a few GET requests, - # consider using {OpenURI}[rdoc-ref:OpenURI]. + # consider using {OpenURI}[https://docs.ruby-lang.org/en/master/OpenURI.html]. # - If you will make only a few requests of all kinds, # consider using the various singleton convenience methods in this class. # Each of the following methods automatically starts and finishes @@ -108,7 +108,7 @@ module Gem::Net #:nodoc: # It consists of some or all of: scheme, hostname, path, query, and fragment; # see {URI syntax}[https://en.wikipedia.org/wiki/Uniform_Resource_Identifier#Syntax]. # - # A Ruby {Gem::URI::Generic}[rdoc-ref:Gem::URI::Generic] object + # A Ruby {Gem::URI::Generic}[https://docs.ruby-lang.org/en/master/Gem/URI/Generic.html] object # represents an internet URI. # It provides, among others, methods # +scheme+, +hostname+, +path+, +query+, and +fragment+. @@ -460,7 +460,7 @@ module Gem::Net #:nodoc: # # First, what's elsewhere. Class Gem::Net::HTTP: # - # - Inherits from {class Object}[rdoc-ref:Object@What-27s+Here]. + # - Inherits from {class Object}[https://docs.ruby-lang.org/en/master/Object.html#class-Object-label-What-27s+Here]. # # This is a categorized summary of methods and attributes. # @@ -475,8 +475,7 @@ module Gem::Net #:nodoc: # # - {::start}[rdoc-ref:Gem::Net::HTTP.start]: # Begins a new session in a new \Gem::Net::HTTP object. - # - {#started?}[rdoc-ref:Gem::Net::HTTP#started?] - # (aliased as {#active?}[rdoc-ref:Gem::Net::HTTP#active?]): + # - {#started?}[rdoc-ref:Gem::Net::HTTP#started?]: # Returns whether in a session. # - {#finish}[rdoc-ref:Gem::Net::HTTP#finish]: # Ends an active session. @@ -556,18 +555,15 @@ module Gem::Net #:nodoc: # Sends a PUT request and returns a response object. # - {#request}[rdoc-ref:Gem::Net::HTTP#request]: # Sends a request and returns a response object. - # - {#request_get}[rdoc-ref:Gem::Net::HTTP#request_get] - # (aliased as {#get2}[rdoc-ref:Gem::Net::HTTP#get2]): + # - {#request_get}[rdoc-ref:Gem::Net::HTTP#request_get]: # Sends a GET request and forms a response object; # if a block given, calls the block with the object, # otherwise returns the object. - # - {#request_head}[rdoc-ref:Gem::Net::HTTP#request_head] - # (aliased as {#head2}[rdoc-ref:Gem::Net::HTTP#head2]): + # - {#request_head}[rdoc-ref:Gem::Net::HTTP#request_head]: # Sends a HEAD request and forms a response object; # if a block given, calls the block with the object, # otherwise returns the object. - # - {#request_post}[rdoc-ref:Gem::Net::HTTP#request_post] - # (aliased as {#post2}[rdoc-ref:Gem::Net::HTTP#post2]): + # - {#request_post}[rdoc-ref:Gem::Net::HTTP#request_post]: # Sends a POST request and forms a response object; # if a block given, calls the block with the object, # otherwise returns the object. @@ -605,8 +601,7 @@ module Gem::Net #:nodoc: # Returns whether +self+ is a proxy class. # - {#proxy?}[rdoc-ref:Gem::Net::HTTP#proxy?]: # Returns whether +self+ has a proxy. - # - {#proxy_address}[rdoc-ref:Gem::Net::HTTP#proxy_address] - # (aliased as {#proxyaddr}[rdoc-ref:Gem::Net::HTTP#proxyaddr]): + # - {#proxy_address}[rdoc-ref:Gem::Net::HTTP#proxy_address]: # Returns the proxy address. # - {#proxy_from_env?}[rdoc-ref:Gem::Net::HTTP#proxy_from_env?]: # Returns whether the proxy is taken from an environment variable. @@ -718,8 +713,7 @@ module Gem::Net #:nodoc: # === \HTTP Version # # - {::version_1_2?}[rdoc-ref:Gem::Net::HTTP.version_1_2?] - # (aliased as {::is_version_1_2?}[rdoc-ref:Gem::Net::HTTP.is_version_1_2?] - # and {::version_1_2}[rdoc-ref:Gem::Net::HTTP.version_1_2]): + # (aliased as {::version_1_2}[rdoc-ref:Gem::Net::HTTP.version_1_2]): # Returns true; retained for compatibility. # # === Debugging @@ -1293,7 +1287,7 @@ module Gem::Net #:nodoc: # - The name of an encoding. # - An alias for an encoding name. # - # See {Encoding}[rdoc-ref:Encoding]. + # See {Encoding}[https://docs.ruby-lang.org/en/master/Encoding.html]. # # Examples: # @@ -1552,11 +1546,11 @@ module Gem::Net #:nodoc: attr_accessor :cert_store # Sets or returns the available SSL ciphers. - # See {OpenSSL::SSL::SSLContext#ciphers=}[rdoc-ref:OpenSSL::SSL::SSLContext#ciphers-3D]. + # See {OpenSSL::SSL::SSLContext#ciphers=}[OpenSSL::SSL::SSL::Context#ciphers=]. attr_accessor :ciphers # Sets or returns the extra X509 certificates to be added to the certificate chain. - # See {OpenSSL::SSL::SSLContext#add_certificate}[rdoc-ref:OpenSSL::SSL::SSLContext#add_certificate]. + # See {OpenSSL::SSL::SSLContext#add_certificate}[OpenSSL::SSL::SSL::Context#add_certificate]. attr_accessor :extra_chain_cert # Sets or returns the OpenSSL::PKey::RSA or OpenSSL::PKey::DSA object. @@ -1566,15 +1560,15 @@ module Gem::Net #:nodoc: attr_accessor :ssl_timeout # Sets or returns the SSL version. - # See {OpenSSL::SSL::SSLContext#ssl_version=}[rdoc-ref:OpenSSL::SSL::SSLContext#ssl_version-3D]. + # See {OpenSSL::SSL::SSLContext#ssl_version=}[OpenSSL::SSL::SSL::Context#ssl_version=]. attr_accessor :ssl_version # Sets or returns the minimum SSL version. - # See {OpenSSL::SSL::SSLContext#min_version=}[rdoc-ref:OpenSSL::SSL::SSLContext#min_version-3D]. + # See {OpenSSL::SSL::SSLContext#min_version=}[OpenSSL::SSL::SSL::Context#min_version=]. attr_accessor :min_version # Sets or returns the maximum SSL version. - # See {OpenSSL::SSL::SSLContext#max_version=}[rdoc-ref:OpenSSL::SSL::SSLContext#max_version-3D]. + # See {OpenSSL::SSL::SSLContext#max_version=}[OpenSSL::SSL::SSL::Context#max_version=]. attr_accessor :max_version # Sets or returns the callback for the server certification verification. @@ -1590,7 +1584,7 @@ module Gem::Net #:nodoc: # Sets or returns whether to verify that the server certificate is valid # for the hostname. - # See {OpenSSL::SSL::SSLContext#verify_hostname=}[rdoc-ref:OpenSSL::SSL::SSLContext#attribute-i-verify_mode]. + # See {OpenSSL::SSL::SSLContext#verify_hostname=}[OpenSSL::SSL::SSL::Context#verify_hostname=]. attr_accessor :verify_hostname # Returns the X509 certificate chain (an array of strings) @@ -1923,11 +1917,8 @@ module Gem::Net #:nodoc: private def unescape(value) - begin - require "cgi/escape" - rescue LoadError - require "cgi/util" - end + require 'cgi/escape' + require 'cgi/util' unless defined?(CGI::EscapeExt) CGI.unescape(value) end diff --git a/misc/tsan_suppressions.txt b/misc/tsan_suppressions.txt new file mode 100644 index 0000000000..908de1c3d9 --- /dev/null +++ b/misc/tsan_suppressions.txt @@ -0,0 +1,120 @@ +# TSan: ThreadSanitizer +# https://github.com/google/sanitizers/wiki/threadsanitizersuppressions +# +# This file describes a number of places where TSAN detects problems in CRuby. +# Many of these indicate bugs. Others are benign (ex. data races that can be +# replaced with relaxed atomic loads) +# +# Usage: +# Configure with: +# ./configure cflags='-fsanitize=thread' CC=clang +# Build and run with: +# TSAN_OPTIONS="suppressions=$(pwd)/misc/tsan_suppressions.txt:die_after_fork=0" +# +# Other useful TSAN_OPTIONS: +# * halt_on_error=1 +# * strip_path_prefix=$(pwd)/ + +# Namespaces +race_top:push_subclass_entry_to_list + +# sub_nounderflow includes non-atomic read, possibly other issue +race:objspace_malloc_increase_body + +# Signals and ubf +race_top:rb_signal_buff_size +race:unregister_ubf_list + +# interrupt flag is set atomically, but read non-atomically +race_top:RUBY_VM_INTERRUPTED_ANY +race_top:unblock_function_set +race_top:threadptr_get_interrupts + +# system_working needs to be converted to atomic +race:system_working + +# It's already crashing. We're doing our best +signal:rb_vm_bugreport +race:check_reserved_signal_ + +race_top:rb_check_deadlock + +# lock_owner +race_top:thread_sched_setup_running_threads +race_top:vm_lock_enter +race_top:rb_ec_vm_lock_rec +race_top:vm_lock_enter +race_top:vm_locked + +# Ractors +race:ractor_take +race:ractor_register_take +race:ractor_check_take_basket +race:ractor_selector__wait + +# vm->ractor.sched.grq_cnt++ +race_top:ractor_sched_enq +race_top:ractor_sched_deq + +# Using VM lock instead of rb_native_mutex_unlock? +race:vm_remove_ractor + +# cr->sync.wait.wakeup_status +race_top:rb_ractor_sched_sleep + +# th->sched.finished at end of co_start +race_top:rb_thread_sched_mark_zombies + +# Races against timer thread setting th->sched.waiting_reason.flags +race_top:thread_sched_wait_events + +# At thread start +race_top:rb_ractor_set_current_ec_ + +# Possible deadlock between Ractor lock and UBF lock +deadlock:ractor_sleep_interrupt + +# RVALUE_AGE_SET manipulates flag bits on objects which may be accessed in Ractors +race_top:RVALUE_AGE_SET + +# Inline caches +race_top:vm_cc_call_set +race_top:vm_search_cc +race_top:vm_search_method_slowpath0 +race_top:rb_vm_opt_getconstant_path +race_top:vm_ic_attr_index_set + +# Shapes have problems with RCLASS_MAX_IV_COUNT and RCLASS_VARIATION_COUNT +# which are probably benign +race:shape_get_next + +# Non-atomic reads/writes +race:gccct_method_search + +# Ignore exit for now +race:rb_ec_finalize +race:rb_ec_cleanup + +# object_id races +race:object_id + +# Sets objspace->flags.dont_incremental while writebarrier may be running +race_top:objspace_each_exec +race_top:objspace_each_objects_ensure + +# Ractor autoload +race:rb_ractor_autoload_load + +# Non-atomic lazy initialized static variable +race_top:rbimpl_intern_const + +# Setting def->aliased bitfield non-atomically +race_top:method_definition_addref + +# Switching to setting up tracing. Likely other ractors should be stopped for this. +race_top:encoded_iseq_trace_instrument +race:rb_iseq_trace_set_all + +# We walk the machine stack looking for markable objects, a thread with the GVL +# released could by mutating the stack with non-Ruby-objects +race:rb_gc_mark_machine_context diff --git a/namespace.c b/namespace.c index b6af0ce407..3e9abe8df2 100644 --- a/namespace.c +++ b/namespace.c @@ -201,7 +201,7 @@ current_namespace(bool permit_calling_builtin) // calling = 0; } while (calling) { - const rb_namespace_t *proc_ns; + const rb_namespace_t *proc_ns = NULL; VALUE bh; if (VM_FRAME_NS_SWITCH_P(cfp)) { bh = rb_vm_frame_block_handler(cfp); @@ -307,6 +307,7 @@ rb_current_namespace_details(VALUE opt) RSTRING_PTR(part), th->namespaces ? RARRAY_LEN(th->namespaces) : 0, require_stack ? RARRAY_LEN(require_stack) : 0); + RB_GC_GUARD(part); rb_str_cat_cstr(str, buf); if (th->namespaces && RARRAY_LEN(th->namespaces) > 0) { @@ -314,6 +315,7 @@ rb_current_namespace_details(VALUE opt) nsobj = RARRAY_AREF(th->namespaces, i); part = rb_namespace_inspect(nsobj); snprintf(buf, 2048, " th->nss[%ld] %s\n", i, RSTRING_PTR(part)); + RB_GC_GUARD(part); rb_str_cat_cstr(str, buf); } } @@ -331,6 +333,7 @@ rb_current_namespace_details(VALUE opt) if (NAMESPACE_USER_P(ns)) { part = rb_namespace_inspect(proc_ns->ns_object); snprintf(buf, 2048, " cfp->ns:%s", RSTRING_PTR(part)); + RB_GC_GUARD(part); calling = 0; break; } @@ -350,6 +353,7 @@ rb_current_namespace_details(VALUE opt) rb_id2name(cme->def->original_id), RSTRING_PTR(part), path); + RB_GC_GUARD(part); rb_str_cat_cstr(str, buf); calling = 0; break; @@ -359,6 +363,7 @@ rb_current_namespace_details(VALUE opt) rb_id2name(cme->def->original_id), RSTRING_PTR(part), path); + RB_GC_GUARD(part); rb_str_cat_cstr(str, buf); } } @@ -745,7 +750,7 @@ copy_ext_file(char *src_path, char *dst_path) #else FILE *src, *dst; char buffer[1024]; - size_t read, wrote, written; + size_t read = 0, wrote, written = 0; size_t maxread = sizeof(buffer); int eof = 0; int clean_read = 1; @@ -987,7 +992,7 @@ rb_initialize_main_namespace(void) if (!namespace_experimental_warned) { rb_category_warn(RB_WARN_CATEGORY_EXPERIMENTAL, "Namespace is experimental, and the behavior may change in the future!\n" - "See doc/namespace.md for know issues, etc."); + "See doc/namespace.md for known issues, etc."); namespace_experimental_warned = 1; } @@ -2139,17 +2139,6 @@ static VALUE class_call_alloc_func(rb_alloc_func_t allocator, VALUE klass); */ static VALUE -rb_class_alloc_m(VALUE klass) -{ - rb_alloc_func_t allocator = class_get_alloc_func(klass); - if (!rb_obj_respond_to(klass, rb_intern("allocate"), 1)) { - rb_raise(rb_eTypeError, "calling %"PRIsVALUE".allocate is prohibited", - klass); - } - return class_call_alloc_func(allocator, klass); -} - -static VALUE rb_class_alloc(VALUE klass) { rb_alloc_func_t allocator = class_get_alloc_func(klass); @@ -4603,8 +4592,8 @@ InitVM_Object(void) rb_define_method(rb_cModule, "deprecate_constant", rb_mod_deprecate_constant, -1); /* in variable.c */ rb_define_method(rb_cModule, "singleton_class?", rb_mod_singleton_p, 0); - rb_define_method(rb_singleton_class(rb_cClass), "allocate", rb_class_alloc_m, 0); - rb_define_method(rb_cClass, "allocate", rb_class_alloc_m, 0); + rb_define_method(rb_singleton_class(rb_cClass), "allocate", rb_class_alloc, 0); + rb_define_method(rb_cClass, "allocate", rb_class_alloc, 0); rb_define_method(rb_cClass, "new", rb_class_new_instance_pass_kw, -1); rb_define_method(rb_cClass, "initialize", rb_class_initialize, -1); rb_define_method(rb_cClass, "superclass", rb_class_superclass, 0); diff --git a/prism/templates/lib/prism/dot_visitor.rb.erb b/prism/templates/lib/prism/dot_visitor.rb.erb index 6deaa3e726..cd2998fe61 100644 --- a/prism/templates/lib/prism/dot_visitor.rb.erb +++ b/prism/templates/lib/prism/dot_visitor.rb.erb @@ -1,8 +1,5 @@ -begin - require "cgi/escape" -rescue LoadError - require "cgi/util" -end +require "cgi/escape" +require "cgi/util" unless defined?(CGI::EscapeExt) module Prism # This visitor provides the ability to call Node#to_dot, which converts a @@ -210,8 +210,6 @@ ractor_mark(void *ptr) ractor_queue_mark(&r->sync.recv_queue); ractor_queue_mark(&r->sync.takers_queue); - rb_gc_mark(r->receiving_mutex); - rb_gc_mark(r->loc); rb_gc_mark(r->name); rb_gc_mark(r->r_stdin); @@ -242,9 +240,6 @@ ractor_free(void *ptr) rb_ractor_t *r = (rb_ractor_t *)ptr; RUBY_DEBUG_LOG("free r:%d", rb_ractor_id(r)); rb_native_mutex_destroy(&r->sync.lock); -#ifdef RUBY_THREAD_WIN32_H - rb_native_cond_destroy(&r->sync.cond); -#endif ractor_queue_free(&r->sync.recv_queue); ractor_queue_free(&r->sync.takers_queue); ractor_local_storage_free(r); @@ -432,6 +427,7 @@ ractor_queue_enq(rb_ractor_t *r, struct rb_ractor_queue *rq, struct rb_ractor_ba } rq->size *= 2; } + // copy basket into queue rq->baskets[(rq->start + rq->cnt++) % rq->size] = *basket; // fprintf(stderr, "%s %p->cnt:%d\n", RUBY_FUNCTION_NAME_STRING, (void *)rq, rq->cnt); } @@ -470,6 +466,7 @@ ractor_basket_accept(struct rb_ractor_basket *b) { VALUE v = ractor_basket_value(b); + // a ractor's main thread had an error and yielded us this exception during its dying moments if (b->p.send.exception) { VALUE cause = v; VALUE err = rb_exc_new_cstr(rb_eRactorRemoteError, "thrown by remote Ractor."); @@ -534,39 +531,59 @@ basket_type_name(enum rb_ractor_basket_type type) } #endif // USE_RUBY_DEBUG_LOG -static bool -ractor_sleeping_by(const rb_ractor_t *r, enum rb_ractor_wait_status wait_status) +static rb_thread_t * +ractor_sleeping_by(const rb_ractor_t *r, rb_thread_t *th, enum rb_ractor_wait_status wait_status) { - return (r->sync.wait.status & wait_status) && r->sync.wait.wakeup_status == wakeup_none; + if (th) { + if ((th->ractor_waiting.wait_status & wait_status) && th->ractor_waiting.wakeup_status == wakeup_none) { + return th; + } + } else { + // find any thread that has this ractor wait status that is blocked + ccan_list_for_each(&r->sync.wait.waiting_threads, th, ractor_waiting.waiting_node) { + if ((th->ractor_waiting.wait_status & wait_status) && th->ractor_waiting.wakeup_status == wakeup_none) { + return th; + } + } + } + return NULL; } #ifdef RUBY_THREAD_PTHREAD_H // thread_*.c -void rb_ractor_sched_wakeup(rb_ractor_t *r); +void rb_ractor_sched_wakeup(rb_ractor_t *r, rb_thread_t *th); #else +// win32 static void -rb_ractor_sched_wakeup(rb_ractor_t *r) +rb_ractor_sched_wakeup(rb_ractor_t *r, rb_thread_t *th) { - rb_native_cond_broadcast(&r->sync.cond); + (void)r; + ASSERT_ractor_locking(r); + rb_native_cond_signal(&th->ractor_waiting.cond); + } #endif +/* + * Wakeup `r` if the given `th` is blocked and has the given ractor `wait_status`. + * Wakeup any blocked thread in `r` with the given ractor `wait_status` if `th` is NULL. + */ static bool -ractor_wakeup(rb_ractor_t *r, enum rb_ractor_wait_status wait_status, enum rb_ractor_wakeup_status wakeup_status) +ractor_wakeup(rb_ractor_t *r, rb_thread_t *th /* can be NULL */, enum rb_ractor_wait_status wait_status, enum rb_ractor_wakeup_status wakeup_status) { ASSERT_ractor_locking(r); RUBY_DEBUG_LOG("r:%u wait_by:%s -> wait:%s wakeup:%s", rb_ractor_id(r), - wait_status_str(r->sync.wait.status), + wait_status_str(th->ractor_waiting.wait_status), wait_status_str(wait_status), wakeup_status_str(wakeup_status)); - if (ractor_sleeping_by(r, wait_status)) { - r->sync.wait.wakeup_status = wakeup_status; - rb_ractor_sched_wakeup(r); + if ((th = ractor_sleeping_by(r, th, wait_status)) != NULL) { + th->ractor_waiting.wakeup_status = wakeup_status; + rb_ractor_sched_wakeup(r, th); return true; } else { @@ -574,27 +591,33 @@ ractor_wakeup(rb_ractor_t *r, enum rb_ractor_wait_status wait_status, enum rb_ra } } +// unblock function (UBF). This gets called when another thread on this or another ractor sets our thread's interrupt flag. +// This is not async-safe. static void ractor_sleep_interrupt(void *ptr) { - rb_ractor_t *r = ptr; + rb_execution_context_t *ec = ptr; + rb_ractor_t *r = rb_ec_ractor_ptr(ec); + rb_thread_t *th = rb_ec_thread_ptr(ec); RACTOR_LOCK(r); { - ractor_wakeup(r, wait_receiving | wait_taking | wait_yielding, wakeup_by_interrupt); + ractor_wakeup(r, th, wait_receiving | wait_taking | wait_yielding, wakeup_by_interrupt); } RACTOR_UNLOCK(r); } typedef void (*ractor_sleep_cleanup_function)(rb_ractor_t *cr, void *p); +// Checks the current thread for ruby interrupts and runs the cleanup function `cf_func` with `cf_data` if +// `rb_ec_check_ints` is going to raise. See the `rb_threadptr_execute_interrupts` for info on when it can raise. static void -ractor_check_ints(rb_execution_context_t *ec, rb_ractor_t *cr, ractor_sleep_cleanup_function cf_func, void *cf_data) +ractor_check_ints(rb_execution_context_t *ec, rb_ractor_t *cr, rb_thread_t *cur_th, ractor_sleep_cleanup_function cf_func, void *cf_data) { - if (cr->sync.wait.status != wait_none) { - enum rb_ractor_wait_status prev_wait_status = cr->sync.wait.status; - cr->sync.wait.status = wait_none; - cr->sync.wait.wakeup_status = wakeup_by_interrupt; + if (cur_th->ractor_waiting.wait_status != wait_none) { + enum rb_ractor_wait_status prev_wait_status = cur_th->ractor_waiting.wait_status; + cur_th->ractor_waiting.wait_status = wait_none; + cur_th->ractor_waiting.wakeup_status = wakeup_by_interrupt; RACTOR_UNLOCK(cr); { @@ -607,7 +630,7 @@ ractor_check_ints(rb_execution_context_t *ec, rb_ractor_t *cr, ractor_sleep_clea EC_POP_TAG(); if (state) { - (*cf_func)(cr, cf_data); + (*cf_func)(cr, cf_data); // cleanup function is run after the ubf, if it had ubf EC_JUMP_TAG(ec, state); } } @@ -616,9 +639,8 @@ ractor_check_ints(rb_execution_context_t *ec, rb_ractor_t *cr, ractor_sleep_clea } } - // reachable? RACTOR_LOCK(cr); - cr->sync.wait.status = prev_wait_status; + cur_th->ractor_waiting.wait_status = prev_wait_status; } } @@ -626,15 +648,14 @@ ractor_check_ints(rb_execution_context_t *ec, rb_ractor_t *cr, ractor_sleep_clea void rb_ractor_sched_sleep(rb_execution_context_t *ec, rb_ractor_t *cr, rb_unblock_function_t *ubf); #else -// win32 static void -ractor_cond_wait(rb_ractor_t *r) +ractor_cond_wait(rb_ractor_t *r, rb_thread_t *th) { #if RACTOR_CHECK_MODE > 0 VALUE locked_by = r->sync.locked_by; r->sync.locked_by = Qnil; #endif - rb_native_cond_wait(&r->sync.cond, &r->sync.lock); + rb_native_cond_wait(&th->ractor_waiting.cond, &r->sync.lock); #if RACTOR_CHECK_MODE > 0 r->sync.locked_by = locked_by; @@ -645,77 +666,99 @@ static void * ractor_sleep_wo_gvl(void *ptr) { rb_ractor_t *cr = ptr; + rb_execution_context_t *ec = cr->threads.running_ec; + VM_ASSERT(GET_EC() == ec); + rb_thread_t *cur_th = rb_ec_thread_ptr(ec); RACTOR_LOCK_SELF(cr); { - VM_ASSERT(cr->sync.wait.status != wait_none); - if (cr->sync.wait.wakeup_status == wakeup_none) { - ractor_cond_wait(cr); + VM_ASSERT(cur_th->ractor_waiting.wait_status != wait_none); + // it's possible that another ractor has woken us up (ractor_wakeup), + // so check this condition + if (cur_th->ractor_waiting.wakeup_status == wakeup_none) { + cur_th->status = THREAD_STOPPED_FOREVER; + ractor_cond_wait(cr, cur_th); + cur_th->status = THREAD_RUNNABLE; + VM_ASSERT(cur_th->ractor_waiting.wakeup_status != wakeup_none); + } else { + RUBY_DEBUG_LOG("rare timing, no cond wait"); } - cr->sync.wait.status = wait_none; + cur_th->ractor_waiting.wait_status = wait_none; } RACTOR_UNLOCK_SELF(cr); return NULL; } static void -rb_ractor_sched_sleep(rb_execution_context_t *ec, rb_ractor_t *cr, rb_unblock_function_t *ubf) +rb_ractor_sched_sleep(rb_execution_context_t *ec, rb_ractor_t *cr, rb_unblock_function_t *ubf_ractor_sleep_interrupt) { + ASSERT_ractor_locking(cr); + rb_thread_t *th = rb_ec_thread_ptr(ec); + struct ccan_list_node *waitn = &th->ractor_waiting.waiting_node; + VM_ASSERT(waitn->next == waitn->prev && waitn->next == waitn); // it should be unlinked + ccan_list_add(&cr->sync.wait.waiting_threads, waitn); RACTOR_UNLOCK(cr); { - rb_nogvl(ractor_sleep_wo_gvl, cr, - ubf, cr, - RB_NOGVL_UBF_ASYNC_SAFE | RB_NOGVL_INTR_FAIL); + rb_nogvl(ractor_sleep_wo_gvl, cr, ubf_ractor_sleep_interrupt, ec, RB_NOGVL_INTR_FAIL); } RACTOR_LOCK(cr); + ccan_list_del_init(waitn); } #endif +/* + * Sleep the current ractor's current thread until another ractor wakes us up or another thread calls our unblock function. + * The following ractor actions can cause this function to be called: + * Ractor#take (wait_taking) + * Ractor.yield (wait_yielding) + * Ractor.receive (wait_receiving) + * Ractor.select (can be a combination of the above wait states, depending on the states of the ractors passed to Ractor.select) + */ static enum rb_ractor_wakeup_status -ractor_sleep_with_cleanup(rb_execution_context_t *ec, rb_ractor_t *cr, enum rb_ractor_wait_status wait_status, +ractor_sleep_with_cleanup(rb_execution_context_t *ec, rb_ractor_t *cr, rb_thread_t *cur_th, enum rb_ractor_wait_status wait_status, ractor_sleep_cleanup_function cf_func, void *cf_data) { + ASSERT_ractor_locking(cr); enum rb_ractor_wakeup_status wakeup_status; VM_ASSERT(GET_RACTOR() == cr); - // TODO: multi-threads - VM_ASSERT(cr->sync.wait.status == wait_none); + VM_ASSERT(cur_th->ractor_waiting.wait_status == wait_none); VM_ASSERT(wait_status != wait_none); - cr->sync.wait.status = wait_status; - cr->sync.wait.wakeup_status = wakeup_none; + cur_th->ractor_waiting.wait_status = wait_status; + cur_th->ractor_waiting.wakeup_status = wakeup_none; // fprintf(stderr, "%s r:%p status:%s, wakeup_status:%s\n", RUBY_FUNCTION_NAME_STRING, (void *)cr, // wait_status_str(cr->sync.wait.status), wakeup_status_str(cr->sync.wait.wakeup_status)); RUBY_DEBUG_LOG("sleep by %s", wait_status_str(wait_status)); - while (cr->sync.wait.wakeup_status == wakeup_none) { + while (cur_th->ractor_waiting.wakeup_status == wakeup_none) { rb_ractor_sched_sleep(ec, cr, ractor_sleep_interrupt); - ractor_check_ints(ec, cr, cf_func, cf_data); + ractor_check_ints(ec, cr, cur_th, cf_func, cf_data); } - cr->sync.wait.status = wait_none; + cur_th->ractor_waiting.wait_status = wait_none; - // TODO: multi-thread - wakeup_status = cr->sync.wait.wakeup_status; - cr->sync.wait.wakeup_status = wakeup_none; + wakeup_status = cur_th->ractor_waiting.wakeup_status; + cur_th->ractor_waiting.wakeup_status = wakeup_none; RUBY_DEBUG_LOG("wakeup %s", wakeup_status_str(wakeup_status)); + ASSERT_ractor_locking(cr); return wakeup_status; } static enum rb_ractor_wakeup_status -ractor_sleep(rb_execution_context_t *ec, rb_ractor_t *cr, enum rb_ractor_wait_status wait_status) +ractor_sleep(rb_execution_context_t *ec, rb_ractor_t *cr, rb_thread_t *cur_th, enum rb_ractor_wait_status wait_status) { - return ractor_sleep_with_cleanup(ec, cr, wait_status, 0, NULL); + return ractor_sleep_with_cleanup(ec, cr, cur_th, wait_status, 0, NULL); } // Ractor.receive static void -ractor_recursive_receive_if(rb_ractor_t *r) +ractor_recursive_receive_if(rb_thread_t *th) { - if (r->receiving_mutex && rb_mutex_owned_p(r->receiving_mutex)) { + if (th->ractor_waiting.receiving_mutex && rb_mutex_owned_p(th->ractor_waiting.receiving_mutex)) { rb_raise(rb_eRactorError, "can not call receive/receive_if recursively"); } } @@ -724,7 +767,7 @@ static VALUE ractor_try_receive(rb_execution_context_t *ec, rb_ractor_t *cr, struct rb_ractor_queue *rq) { struct rb_ractor_basket basket; - ractor_recursive_receive_if(cr); + ractor_recursive_receive_if(rb_ec_thread_ptr(ec)); bool received = false; RACTOR_LOCK_SELF(cr); @@ -749,12 +792,13 @@ static void ractor_wait_receive(rb_execution_context_t *ec, rb_ractor_t *cr, struct rb_ractor_queue *rq) { VM_ASSERT(cr == rb_ec_ractor_ptr(ec)); - ractor_recursive_receive_if(cr); + rb_thread_t *cur_th = rb_ec_thread_ptr(ec); + ractor_recursive_receive_if(cur_th); RACTOR_LOCK(cr); { while (ractor_queue_empty_p(cr, rq) && !cr->sync.incoming_port_closed) { - ractor_sleep(ec, cr, wait_receiving); + ractor_sleep(ec, cr, cur_th, wait_receiving); } } RACTOR_UNLOCK(cr); @@ -791,6 +835,7 @@ rq_dump(struct rb_ractor_queue *rq) struct receive_block_data { rb_ractor_t *cr; + rb_thread_t *th; struct rb_ractor_queue *rq; VALUE v; int index; @@ -798,11 +843,11 @@ struct receive_block_data { }; static void -ractor_receive_if_lock(rb_ractor_t *cr) +ractor_receive_if_lock(rb_thread_t *th) { - VALUE m = cr->receiving_mutex; + VALUE m = th->ractor_waiting.receiving_mutex; if (m == Qfalse) { - m = cr->receiving_mutex = rb_mutex_new(); + m = th->ractor_waiting.receiving_mutex = rb_mutex_new(); } rb_mutex_lock(m); } @@ -812,7 +857,7 @@ receive_if_body(VALUE ptr) { struct receive_block_data *data = (struct receive_block_data *)ptr; - ractor_receive_if_lock(data->cr); + ractor_receive_if_lock(data->th); VALUE block_result = rb_yield(data->v); rb_ractor_t *cr = data->cr; @@ -847,6 +892,7 @@ receive_if_ensure(VALUE v) { struct receive_block_data *data = (struct receive_block_data *)v; rb_ractor_t *cr = data->cr; + rb_thread_t *cur_th = data->th; if (!data->success) { RACTOR_LOCK_SELF(cr); @@ -859,7 +905,7 @@ receive_if_ensure(VALUE v) RACTOR_UNLOCK_SELF(cr); } - rb_mutex_unlock(cr->receiving_mutex); + rb_mutex_unlock(cur_th->ractor_waiting.receiving_mutex); return Qnil; } @@ -869,6 +915,7 @@ ractor_receive_if(rb_execution_context_t *ec, VALUE crv, VALUE b) if (!RTEST(b)) rb_raise(rb_eArgError, "no block given"); rb_ractor_t *cr = rb_ec_ractor_ptr(ec); + rb_thread_t *cur_th = rb_ec_thread_ptr(ec); unsigned int serial = (unsigned int)-1; int index = 0; struct rb_ractor_queue *rq = &cr->sync.recv_queue; @@ -902,6 +949,7 @@ ractor_receive_if(rb_execution_context_t *ec, VALUE crv, VALUE b) if (!UNDEF_P(v)) { struct receive_block_data data = { .cr = cr, + .th = cur_th, .rq = rq, .v = v, .index = index, @@ -931,7 +979,8 @@ ractor_send_basket(rb_execution_context_t *ec, rb_ractor_t *r, struct rb_ractor_ } else { ractor_queue_enq(r, &r->sync.recv_queue, b); - ractor_wakeup(r, wait_receiving, wakeup_by_send); + // wakeup any receiving thread in `r` + ractor_wakeup(r, NULL, wait_receiving, wakeup_by_send); } } RACTOR_UNLOCK(r); @@ -970,40 +1019,43 @@ ractor_basket_prepare_contents(VALUE obj, VALUE move, volatile VALUE *pobj, enum } static void -ractor_basket_fill_(rb_ractor_t *cr, struct rb_ractor_basket *basket, VALUE obj, bool exc) +ractor_basket_fill_(rb_ractor_t *cr, rb_thread_t *cur_th, struct rb_ractor_basket *basket, VALUE obj, bool exc) { VM_ASSERT(cr == GET_RACTOR()); basket->sender = cr->pub.self; + basket->sending_th = cur_th; basket->p.send.exception = exc; basket->p.send.v = obj; } static void -ractor_basket_fill(rb_ractor_t *cr, struct rb_ractor_basket *basket, VALUE obj, VALUE move, bool exc) +ractor_basket_fill(rb_ractor_t *cr, rb_thread_t *cur_th, struct rb_ractor_basket *basket, VALUE obj, VALUE move, bool exc) { VALUE v; enum rb_ractor_basket_type type; ractor_basket_prepare_contents(obj, move, &v, &type); - ractor_basket_fill_(cr, basket, v, exc); + ractor_basket_fill_(cr, cur_th, basket, v, exc); basket->type.e = type; } static void -ractor_basket_fill_will(rb_ractor_t *cr, struct rb_ractor_basket *basket, VALUE obj, bool exc) +ractor_basket_fill_will(rb_ractor_t *cr, rb_thread_t *cur_th, struct rb_ractor_basket *basket, VALUE obj, bool exc) { - ractor_basket_fill_(cr, basket, obj, exc); + ractor_basket_fill_(cr, cur_th, basket, obj, exc); basket->type.e = basket_type_will; } static VALUE -ractor_send(rb_execution_context_t *ec, rb_ractor_t *r, VALUE obj, VALUE move) +ractor_send(rb_execution_context_t *ec, rb_ractor_t *recv_r, VALUE obj, VALUE move) { struct rb_ractor_basket basket; + rb_ractor_t *cr = rb_ec_ractor_ptr(ec); + rb_thread_t *cur_th = rb_ec_thread_ptr(ec); // TODO: Ractor local GC - ractor_basket_fill(rb_ec_ractor_ptr(ec), &basket, obj, move, false); - ractor_send_basket(ec, r, &basket); - return r->pub.self; + ractor_basket_fill(cr, cur_th, &basket, obj, move, false); + ractor_send_basket(ec, recv_r, &basket); + return recv_r->pub.self; } // Ractor#take @@ -1048,15 +1100,16 @@ ractor_take_will_lock(rb_ractor_t *r, struct rb_ractor_basket *b) } static bool -ractor_register_take(rb_ractor_t *cr, rb_ractor_t *r, struct rb_ractor_basket *take_basket, +ractor_register_take(rb_ractor_t *cr, rb_thread_t *cur_th, rb_ractor_t *r, struct rb_ractor_basket *take_basket, bool is_take, struct rb_ractor_selector_take_config *config, bool ignore_error) { struct rb_ractor_basket b = { .type.e = basket_type_take_basket, .sender = cr->pub.self, + .sending_th = cur_th, .p = { .take = { - .basket = take_basket, + .basket = take_basket, // pointer to our stack value saved in ractor `r` queue .config = config, }, }, @@ -1081,7 +1134,8 @@ ractor_register_take(rb_ractor_t *cr, rb_ractor_t *r, struct rb_ractor_basket *t ractor_queue_enq(r, &r->sync.takers_queue, &b); if (basket_none_p(take_basket)) { - ractor_wakeup(r, wait_yielding, wakeup_by_take); + // wakeup any thread in `r` that has yielded, if there is any. + ractor_wakeup(r, NULL, wait_yielding, wakeup_by_take); } } } @@ -1126,17 +1180,18 @@ ractor_deregister_take(rb_ractor_t *r, struct rb_ractor_basket *take_basket) } static VALUE -ractor_try_take(rb_ractor_t *cr, rb_ractor_t *r, struct rb_ractor_basket *take_basket) +ractor_try_take(rb_ractor_t *cr, rb_thread_t *cur_th, rb_ractor_t *recv_r, struct rb_ractor_basket *take_basket) { bool taken; RACTOR_LOCK_SELF(cr); { + // If it hasn't yielded yet or is currently in the process of yielding, sleep more if (basket_none_p(take_basket) || basket_type_p(take_basket, basket_type_yielding)) { taken = false; } else { - taken = true; + taken = true; // basket type might be, for ex, basket_type_copy if value was copied during yield } } RACTOR_UNLOCK_SELF(cr); @@ -1144,7 +1199,7 @@ ractor_try_take(rb_ractor_t *cr, rb_ractor_t *r, struct rb_ractor_basket *take_b if (taken) { RUBY_DEBUG_LOG("taken"); if (basket_type_p(take_basket, basket_type_deleted)) { - VM_ASSERT(r->sync.outgoing_port_closed); + VM_ASSERT(recv_r->sync.outgoing_port_closed); rb_raise(rb_eRactorClosedError, "The outgoing-port is already closed"); } return ractor_basket_accept(take_basket); @@ -1179,6 +1234,7 @@ ractor_check_specific_take_basket_lock(rb_ractor_t *r, struct rb_ractor_basket * } #endif +// cleanup function, cr is unlocked static void ractor_take_cleanup(rb_ractor_t *cr, rb_ractor_t *r, struct rb_ractor_basket *tb) { @@ -1208,7 +1264,7 @@ ractor_wait_take_cleanup(rb_ractor_t *cr, void *ptr) } static void -ractor_wait_take(rb_execution_context_t *ec, rb_ractor_t *cr, rb_ractor_t *r, struct rb_ractor_basket *take_basket) +ractor_wait_take(rb_execution_context_t *ec, rb_ractor_t *cr, rb_thread_t *cur_th, rb_ractor_t *r, struct rb_ractor_basket *take_basket) { struct take_wait_take_cleanup_data data = { .r = r, @@ -1218,32 +1274,33 @@ ractor_wait_take(rb_execution_context_t *ec, rb_ractor_t *cr, rb_ractor_t *r, st RACTOR_LOCK_SELF(cr); { if (basket_none_p(take_basket) || basket_type_p(take_basket, basket_type_yielding)) { - ractor_sleep_with_cleanup(ec, cr, wait_taking, ractor_wait_take_cleanup, &data); + ractor_sleep_with_cleanup(ec, cr, cur_th, wait_taking, ractor_wait_take_cleanup, &data); } } RACTOR_UNLOCK_SELF(cr); } static VALUE -ractor_take(rb_execution_context_t *ec, rb_ractor_t *r) +ractor_take(rb_execution_context_t *ec, rb_ractor_t *recv_r) { - RUBY_DEBUG_LOG("from r:%u", rb_ractor_id(r)); + RUBY_DEBUG_LOG("from r:%u", rb_ractor_id(recv_r)); VALUE v; rb_ractor_t *cr = rb_ec_ractor_ptr(ec); + rb_thread_t *cur_th = rb_ec_thread_ptr(ec); struct rb_ractor_basket take_basket = { .type.e = basket_type_none, .sender = 0, }; - ractor_register_take(cr, r, &take_basket, true, NULL, false); + ractor_register_take(cr, cur_th, recv_r, &take_basket, true, NULL, false); - while (UNDEF_P(v = ractor_try_take(cr, r, &take_basket))) { - ractor_wait_take(ec, cr, r, &take_basket); + while (UNDEF_P(v = ractor_try_take(cr, cur_th, recv_r, &take_basket))) { + ractor_wait_take(ec, cr, cur_th, recv_r, &take_basket); } - VM_ASSERT(!basket_none_p(&take_basket)); - VM_ASSERT(!ractor_check_specific_take_basket_lock(r, &take_basket)); + VM_ASSERT(!basket_none_p(&take_basket)); // might be, for ex, basket_type_copy + VM_ASSERT(!ractor_check_specific_take_basket_lock(recv_r, &take_basket)); return v; } @@ -1266,6 +1323,7 @@ ractor_check_take_basket(rb_ractor_t *cr, struct rb_ractor_queue *rs) return false; } +// Find another ractor that is taking from this ractor, so we can yield to it static bool ractor_deq_take_basket(rb_ractor_t *cr, struct rb_ractor_queue *rs, struct rb_ractor_basket *b) { @@ -1276,11 +1334,11 @@ ractor_deq_take_basket(rb_ractor_t *cr, struct rb_ractor_queue *rs, struct rb_ra RACTOR_LOCK_SELF(cr); { while (ractor_queue_deq(cr, rs, b)) { - if (basket_type_p(b, basket_type_take_basket)) { + if (basket_type_p(b, basket_type_take_basket)) { // some other ractor is taking struct rb_ractor_basket *tb = b->p.take.basket; if (RUBY_ATOMIC_CAS(tb->type.atomic, basket_type_none, basket_type_yielding) == basket_type_none) { - found = true; + found = true; // payload basket is now "yielding" type break; } else { @@ -1307,25 +1365,30 @@ ractor_deq_take_basket(rb_ractor_t *cr, struct rb_ractor_queue *rs, struct rb_ra return found; } +// Try yielding to a taking ractor static bool ractor_try_yield(rb_execution_context_t *ec, rb_ractor_t *cr, struct rb_ractor_queue *ts, volatile VALUE obj, VALUE move, bool exc, bool is_will) { + // Don't lock yielding ractor at same time as taking ractor. This could deadlock due to timing + // issue because we don't have a lock hierarchy. ASSERT_ractor_unlocking(cr); + rb_thread_t *cur_th = rb_ec_thread_ptr(ec); struct rb_ractor_basket b; - if (ractor_deq_take_basket(cr, ts, &b)) { + if (ractor_deq_take_basket(cr, ts, &b)) { // deq a take basket from takers queue of `cr` into `b` VM_ASSERT(basket_type_p(&b, basket_type_take_basket)); VM_ASSERT(basket_type_p(b.p.take.basket, basket_type_yielding)); - rb_ractor_t *tr = RACTOR_PTR(b.sender); - struct rb_ractor_basket *tb = b.p.take.basket; + rb_ractor_t *tr = RACTOR_PTR(b.sender); // taking ractor + rb_thread_t *tr_th = b.sending_th; // taking thread + struct rb_ractor_basket *tb = b.p.take.basket; // payload basket enum rb_ractor_basket_type type; RUBY_DEBUG_LOG("basket from r:%u", rb_ractor_id(tr)); if (is_will) { - type = basket_type_will; + type = basket_type_will; // last message } else { enum ruby_tag_type state; @@ -1337,7 +1400,7 @@ ractor_try_yield(rb_execution_context_t *ec, rb_ractor_t *cr, struct rb_ractor_q ractor_basket_prepare_contents(obj, move, &obj, &type); } EC_POP_TAG(); - // rescue + // rescue ractor copy/move error, then re-raise if (state) { RACTOR_LOCK_SELF(cr); { @@ -1354,11 +1417,11 @@ ractor_try_yield(rb_execution_context_t *ec, rb_ractor_t *cr, struct rb_ractor_q VM_ASSERT(basket_type_p(tb, basket_type_yielding)); // fill atomic RUBY_DEBUG_LOG("fill %sbasket from r:%u", is_will ? "will " : "", rb_ractor_id(tr)); - ractor_basket_fill_(cr, tb, obj, exc); + ractor_basket_fill_(cr, cur_th, tb, obj, exc); // fill the take basket payload if (RUBY_ATOMIC_CAS(tb->type.atomic, basket_type_yielding, type) != basket_type_yielding) { rb_bug("unreachable"); } - ractor_wakeup(tr, wait_taking, wakeup_by_yield); + ractor_wakeup(tr, tr_th, wait_taking, wakeup_by_yield); } RACTOR_UNLOCK(tr); @@ -1376,15 +1439,17 @@ ractor_try_yield(rb_execution_context_t *ec, rb_ractor_t *cr, struct rb_ractor_q static void ractor_wait_yield(rb_execution_context_t *ec, rb_ractor_t *cr, struct rb_ractor_queue *ts) { + rb_thread_t *cur_th = rb_ec_thread_ptr(ec); RACTOR_LOCK_SELF(cr); { while (!ractor_check_take_basket(cr, ts) && !cr->sync.outgoing_port_closed) { - ractor_sleep(ec, cr, wait_yielding); + ractor_sleep(ec, cr, cur_th, wait_yielding); } } RACTOR_UNLOCK_SELF(cr); } +// In order to yield, we wait until our takers queue has at least one element. Then, we wakeup a taker. static VALUE ractor_yield(rb_execution_context_t *ec, rb_ractor_t *cr, VALUE obj, VALUE move) { @@ -1525,7 +1590,7 @@ ractor_selector_add(VALUE selv, VALUE rv) config->closed = false; config->oneshot = false; - if (ractor_register_take(GET_RACTOR(), r, &s->take_basket, false, config, true)) { + if (ractor_register_take(GET_RACTOR(), GET_THREAD(), r, &s->take_basket, false, config, true)) { st_insert(s->take_ractors, (st_data_t)r, (st_data_t)config); } @@ -1649,7 +1714,7 @@ ractor_selector_wait_i(st_data_t key, st_data_t val, st_data_t dat) } else { RUBY_DEBUG_LOG("wakeup r:%u", rb_ractor_id(r)); - ractor_wakeup(r, wait_yielding, wakeup_by_take); + ractor_wakeup(r, NULL, wait_yielding, wakeup_by_take); ret = ST_CONTINUE; } } @@ -1660,14 +1725,21 @@ ractor_selector_wait_i(st_data_t key, st_data_t val, st_data_t dat) // Ractor::Selector#wait +// cleanup function, cr is unlocked static void -ractor_selector_wait_cleaup(rb_ractor_t *cr, void *ptr) +ractor_selector_wait_cleanup(rb_ractor_t *cr, void *ptr) { struct rb_ractor_basket *tb = (struct rb_ractor_basket *)ptr; RACTOR_LOCK_SELF(cr); { - while (basket_type_p(tb, basket_type_yielding)) rb_thread_sleep(0); + while (basket_type_p(tb, basket_type_yielding)) { + RACTOR_UNLOCK_SELF(cr); + { + rb_thread_sleep(0); + } + RACTOR_LOCK_SELF(cr); + } // if tb->type is not none, taking is succeeded, but interruption ignore it unfortunately. tb->type.e = basket_type_reserved; } @@ -1683,6 +1755,7 @@ ractor_selector__wait(VALUE selv, VALUE do_receivev, VALUE do_yieldv, VALUE yiel struct rb_ractor_basket *tb = &s->take_basket; struct rb_ractor_basket taken_basket; rb_ractor_t *cr = rb_ec_ractor_ptr(ec); + rb_thread_t *cur_th = rb_ec_thread_ptr(ec); bool do_receive = !!RTEST(do_receivev); bool do_yield = !!RTEST(do_yieldv); VALUE ret_v, ret_r; @@ -1744,7 +1817,7 @@ ractor_selector__wait(VALUE selv, VALUE do_receivev, VALUE do_yieldv, VALUE yiel break; } - ractor_sleep_with_cleanup(ec, cr, wait_status, ractor_selector_wait_cleaup, tb); + ractor_sleep_with_cleanup(ec, cr, cur_th, wait_status, ractor_selector_wait_cleanup, tb); } taken_basket = *tb; @@ -1870,13 +1943,17 @@ static VALUE ractor_close_incoming(rb_execution_context_t *ec, rb_ractor_t *r) { VALUE prev; + rb_thread_t *r_th = NULL; + if (r == rb_ec_ractor_ptr(ec)) { + r_th = rb_ec_thread_ptr(ec); + } RACTOR_LOCK(r); { if (!r->sync.incoming_port_closed) { prev = Qfalse; r->sync.incoming_port_closed = true; - if (ractor_wakeup(r, wait_receiving, wakeup_by_close)) { + if (ractor_wakeup(r, r_th, wait_receiving, wakeup_by_close)) { VM_ASSERT(ractor_queue_empty_p(r, &r->sync.recv_queue)); RUBY_DEBUG_LOG("cancel receiving"); } @@ -1915,6 +1992,7 @@ ractor_close_outgoing(rb_execution_context_t *ec, rb_ractor_t *r) while (ractor_queue_deq(r, ts, &b)) { if (basket_type_p(&b, basket_type_take_basket)) { tr = RACTOR_PTR(b.sender); + rb_thread_t *tr_th = b.sending_th; struct rb_ractor_basket *tb = b.p.take.basket; if (RUBY_ATOMIC_CAS(tb->type.atomic, basket_type_none, basket_type_yielding) == basket_type_none) { @@ -1932,14 +2010,14 @@ ractor_close_outgoing(rb_execution_context_t *ec, rb_ractor_t *r) // TODO: deadlock-able? RACTOR_LOCK(tr); { - ractor_wakeup(tr, wait_taking, wakeup_by_close); + ractor_wakeup(tr, tr_th, wait_taking, wakeup_by_close); } RACTOR_UNLOCK(tr); } } // raising yielding Ractor - ractor_wakeup(r, wait_yielding, wakeup_by_close); + ractor_wakeup(r, NULL, wait_yielding, wakeup_by_close); VM_ASSERT(ractor_queue_empty_p(r, ts)); } @@ -2121,9 +2199,9 @@ ractor_init(rb_ractor_t *r, VALUE name, VALUE loc) rb_native_cond_initialize(&r->barrier_wait_cond); #ifdef RUBY_THREAD_WIN32_H - rb_native_cond_initialize(&r->sync.cond); rb_native_cond_initialize(&r->barrier_wait_cond); #endif + ccan_list_head_init(&r->sync.wait.waiting_threads); // thread management rb_thread_sched_init(&r->threads.sched, false); @@ -2193,6 +2271,7 @@ ractor_yield_atexit(rb_execution_context_t *ec, rb_ractor_t *cr, VALUE v, bool e ASSERT_ractor_unlocking(cr); struct rb_ractor_queue *ts = &cr->sync.takers_queue; + rb_thread_t *cur_th = rb_ec_thread_ptr(ec); retry: if (ractor_try_yield(ec, cr, ts, v, Qfalse, exc, true)) { @@ -2203,9 +2282,9 @@ ractor_yield_atexit(rb_execution_context_t *ec, rb_ractor_t *cr, VALUE v, bool e RACTOR_LOCK(cr); { if (!ractor_check_take_basket(cr, ts)) { - VM_ASSERT(cr->sync.wait.status == wait_none); + VM_ASSERT(cur_th->ractor_waiting.wait_status == wait_none); RUBY_DEBUG_LOG("leave a will"); - ractor_basket_fill_will(cr, &cr->sync.will_basket, v, exc); + ractor_basket_fill_will(cr, cur_th, &cr->sync.will_basket, v, exc); } else { RUBY_DEBUG_LOG("rare timing!"); @@ -3598,7 +3677,6 @@ move_leave(VALUE obj, struct obj_traverse_replace_data *data) } // Avoid mutations using bind_call, etc. - // We keep FL_SEEN_OBJ_ID so GC later clean the obj_id_table. MEMZERO((char *)obj + sizeof(struct RBasic), char, size - sizeof(struct RBasic)); RBASIC(obj)->flags = T_OBJECT | FL_FREEZE; RBASIC_SET_CLASS_RAW(obj, rb_cRactorMovedObject); @@ -835,15 +835,21 @@ class Ractor end end - # get a value from ractor-local storage of current Ractor + # get a value from ractor-local storage for current Ractor # Obsolete and use Ractor.[] instead. def [](sym) + if (self != Ractor.current) + raise RuntimeError, "Cannot get ractor local storage for non-current ractor" + end Primitive.ractor_local_value(sym) end - # set a value in ractor-local storage of current Ractor + # set a value in ractor-local storage for current Ractor # Obsolete and use Ractor.[]= instead. def []=(sym, val) + if (self != Ractor.current) + raise RuntimeError, "Cannot set ractor local storage for non-current ractor" + end Primitive.ractor_local_value_set(sym, val) end diff --git a/ractor_core.h b/ractor_core.h index 51fb3246ac..256ecc38e6 100644 --- a/ractor_core.h +++ b/ractor_core.h @@ -43,7 +43,8 @@ struct rb_ractor_basket { enum rb_ractor_basket_type e; rb_atomic_t atomic; } type; - VALUE sender; + VALUE sender; // Ractor object sending message + rb_thread_t *sending_th; union { struct { @@ -117,14 +118,9 @@ struct rb_ractor_sync { struct rb_ractor_basket will_basket; struct ractor_wait { - enum rb_ractor_wait_status status; - enum rb_ractor_wakeup_status wakeup_status; - rb_thread_t *waiting_thread; + struct ccan_list_head waiting_threads; + // each thread has struct ccan_list_node ractor_waiting.waiting_node } wait; - -#ifndef RUBY_THREAD_PTHREAD_H - rb_nativethread_cond_t cond; -#endif }; // created @@ -152,7 +148,6 @@ struct rb_ractor_struct { struct rb_ractor_pub pub; struct rb_ractor_sync sync; - VALUE receiving_mutex; // vm wide barrier synchronization rb_nativethread_cond_t barrier_wait_cond; @@ -118,8 +118,10 @@ typedef struct { typedef struct { short int len; -#if defined(__has_attribute) && __has_attribute(nonstring) +#if defined(__has_attribute) +# if __has_attribute(nonstring) __attribute__((nonstring)) +# endif #endif const UChar name[6]; int ctype; @@ -1139,6 +1139,11 @@ set_i_merge(int argc, VALUE *argv, VALUE set) if (rb_keyword_given_p()) { rb_raise(rb_eArgError, "no keywords accepted"); } + + if (set_iterating_p(set)) { + rb_raise(rb_eRuntimeError, "cannot add to set during iteration"); + } + rb_check_frozen(set); int i; @@ -2184,7 +2189,6 @@ Init_Set(void) rb_define_method(rb_cSet, "superset?", set_i_superset, 1); rb_define_alias(rb_cSet, ">=", "superset?"); rb_define_method(rb_cSet, "to_a", set_i_to_a, 0); - rb_define_method(rb_cSet, "to_h", set_i_to_h, 0); rb_define_method(rb_cSet, "to_set", set_i_to_set, -1); /* :nodoc: */ @@ -745,8 +745,8 @@ rb_shape_has_object_id(rb_shape_t *shape) return shape->flags & SHAPE_FL_HAS_OBJECT_ID; } -rb_shape_t * -rb_shape_object_id_shape(VALUE obj) +shape_id_t +rb_shape_transition_object_id(VALUE obj) { rb_shape_t* shape = rb_obj_shape(obj); RUBY_ASSERT(shape); @@ -755,13 +755,13 @@ rb_shape_object_id_shape(VALUE obj) while (shape->type != SHAPE_OBJ_ID) { shape = RSHAPE(shape->parent_id); } - return shape; } - - bool dont_care; - rb_shape_t* next_shape = get_next_shape_internal(shape, ruby_internal_object_id, SHAPE_OBJ_ID, &dont_care, true); - RUBY_ASSERT(next_shape); - return next_shape; + else { + bool dont_care; + shape = get_next_shape_internal(shape, ruby_internal_object_id, SHAPE_OBJ_ID, &dont_care, true); + } + RUBY_ASSERT(shape); + return rb_shape_id(shape); } /* @@ -168,8 +168,8 @@ shape_id_t rb_shape_transition_complex(VALUE obj); bool rb_shape_transition_remove_ivar(VALUE obj, ID id, VALUE *removed); shape_id_t rb_shape_transition_add_ivar(VALUE obj, ID id); shape_id_t rb_shape_transition_add_ivar_no_warnings(VALUE obj, ID id); +shape_id_t rb_shape_transition_object_id(VALUE obj); -rb_shape_t *rb_shape_object_id_shape(VALUE obj); bool rb_shape_has_object_id(rb_shape_t *shape); void rb_shape_free_all(void); @@ -676,6 +676,7 @@ signal_ignored(int sig) if (sigaction(sig, NULL, &old) < 0) return FALSE; func = old.sa_handler; #else + // TODO: this is not a thread-safe way to do it. Needs lock. sighandler_t old = signal(sig, SIG_DFL); signal(sig, old); func = old; diff --git a/spec/bundler/commands/doctor_spec.rb b/spec/bundler/commands/doctor_spec.rb index 456fb4ec78..5ceaf37f29 100644 --- a/spec/bundler/commands/doctor_spec.rb +++ b/spec/bundler/commands/doctor_spec.rb @@ -62,6 +62,12 @@ RSpec.describe "bundle doctor" do expect(@stdout.string).to include("No issues") end + it "parses otool output correctly" do + doctor = Bundler::CLI::Doctor::Diagnose.new({}) + expect(doctor).to receive(:`).with("/usr/bin/otool -L fake").and_return("/home/gem/ruby/3.4.3/gems/blake3-rb-1.5.4.4/lib/digest/blake3/blake3_ext.bundle:\n\t (compatibility version 0.0.0, current version 0.0.0)\n\t/usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1351.0.0)") + expect(doctor.dylibs_darwin("fake")).to eq(["/usr/lib/libSystem.B.dylib"]) + end + it "exits with a message if one of the linked libraries is missing" do doctor = Bundler::CLI::Doctor::Diagnose.new({}) expect(doctor).to receive(:bundles_for_gem).exactly(2).times.and_return ["/path/to/myrack/myrack.bundle"] diff --git a/spec/ruby/core/set/add_spec.rb b/spec/ruby/core/set/add_spec.rb index 73662acfd1..0fe1a0926c 100644 --- a/spec/ruby/core/set/add_spec.rb +++ b/spec/ruby/core/set/add_spec.rb @@ -23,4 +23,12 @@ describe "Set#add?" do @set.add?("cat") @set.add?("cat").should be_nil end + + it "raises RuntimeError when called during iteration" do + set = Set[:a, :b, :c, :d, :e, :f] + set.each do |_m| + -> { set << 1 }.should raise_error(RuntimeError, /iteration/) + end + set.should == Set[:a, :b, :c, :d, :e, :f] + end end diff --git a/spec/ruby/core/set/merge_spec.rb b/spec/ruby/core/set/merge_spec.rb index 2ee6a4b709..0c6ed27670 100644 --- a/spec/ruby/core/set/merge_spec.rb +++ b/spec/ruby/core/set/merge_spec.rb @@ -16,6 +16,13 @@ describe "Set#merge" do -> { Set[1, 2].merge(Object.new) }.should raise_error(ArgumentError) end + it "raises RuntimeError when called during iteration" do + set = Set[:a, :b] + set.each do |_m| + -> { set.merge([1, 2]) }.should raise_error(RuntimeError, /iteration/) + end + end + ruby_version_is ""..."3.3" do it "accepts only a single argument" do -> { Set[].merge([], []) }.should raise_error(ArgumentError, "wrong number of arguments (given 2, expected 1)") diff --git a/spec/ruby/core/set/replace_spec.rb b/spec/ruby/core/set/replace_spec.rb index 60b33eb0c0..c66a2d0ec3 100644 --- a/spec/ruby/core/set/replace_spec.rb +++ b/spec/ruby/core/set/replace_spec.rb @@ -10,6 +10,14 @@ describe "Set#replace" do @set.should == Set[1, 2, 3] end + it "raises RuntimeError when called during iteration" do + set = Set[:a, :b, :c, :d, :e, :f] + set.each do |_m| + -> { set.replace(Set[1, 2, 3]) }.should raise_error(RuntimeError, /iteration/) + end + set.should == Set[:a, :b, :c, :d, :e, :f] + end + it "accepts any enumerable as other" do @set.replace([1, 2, 3]).should == Set[1, 2, 3] end @@ -3601,6 +3601,8 @@ rb_str_freeze(VALUE str) * without warning issuance. * * Otherwise returns <tt>self.dup</tt>, which is not frozen. + * + * Related: see {Freezing/Unfreezing}[rdoc-ref:String@Freezing-2FUnfreezing]. */ static VALUE str_uplus(VALUE str) @@ -4608,7 +4610,7 @@ rb_str_eql(VALUE str1, VALUE str2) /* * call-seq: - * string <=> other_string -> -1, 0, 1, or nil + * self <=> other_string -> -1, 0, 1, or nil * * Compares +self+ and +other_string+, returning: * @@ -4619,13 +4621,14 @@ rb_str_eql(VALUE str1, VALUE str2) * * Examples: * - * 'foo' <=> 'foo' # => 0 + * 'foo' <=> 'foo' # => 0 * 'foo' <=> 'food' # => -1 * 'food' <=> 'foo' # => 1 - * 'FOO' <=> 'foo' # => -1 - * 'foo' <=> 'FOO' # => 1 - * 'foo' <=> 1 # => nil + * 'FOO' <=> 'foo' # => -1 + * 'foo' <=> 'FOO' # => 1 + * 'foo' <=> 1 # => nil * + * Related: see {Comparing}[rdoc-ref:String@Comparing]. */ static VALUE diff --git a/test/-ext-/debug/test_debug.rb b/test/-ext-/debug/test_debug.rb index 2229859801..c9263d76fa 100644 --- a/test/-ext-/debug/test_debug.rb +++ b/test/-ext-/debug/test_debug.rb @@ -75,19 +75,58 @@ class TestDebug < Test::Unit::TestCase end assert_equal true, x, '[Bug #15105]' end +end + +# This is a YJIT test, but we can't test this without a C extension that calls +# rb_debug_inspector_open(), so we're testing it using "-test-/debug" here. +class TestDebugWithYJIT < Test::Unit::TestCase + class LocalSetArray + def to_a + Bug::Debug.inspector.each do |_, binding,| + binding.local_variable_set(:local, :ok) if binding + end + [:ok] + end + end + + class DebugArray + def to_a + Bug::Debug.inspector + [:ok] + end + end + + def test_yjit_invalidates_getlocal_after_splatarray + val = getlocal_after_splatarray(LocalSetArray.new) + assert_equal [:ok, :ok], val + end - # This is a YJIT test, but we can't test this without a C extension that calls - # rb_debug_inspector_open(), so we're testing it using "-test-/debug" here. - def test_yjit_invalidates_setlocal_after_inspector_call + def test_yjit_invalidates_setlocal_after_splatarray + val = setlocal_after_splatarray(DebugArray.new) + assert_equal [:ok], val + end + + def test_yjit_invalidates_setlocal_after_proc_call val = setlocal_after_proc_call(proc { Bug::Debug.inspector; :ok }) assert_equal :ok, val - end if defined?(RubyVM::YJIT) && RubyVM::YJIT.enabled? + end private + def getlocal_after_splatarray(array) + local = 1 + [*array, local] + end + + def setlocal_after_splatarray(array) + local = *array # setlocal followed by splatarray + itself # split a block using a C call + local # getlocal + end + def setlocal_after_proc_call(block) local = block.call # setlocal followed by OPTIMIZED_METHOD_TYPE_CALL itself # split a block using a C call local # getlocal end -end +end if defined?(RubyVM::YJIT) && RubyVM::YJIT.enabled? diff --git a/test/-ext-/thread_fd/test_thread_fd_close.rb b/test/-ext-/thread_fd/test_thread_fd_close.rb deleted file mode 100644 index 1d2ef63635..0000000000 --- a/test/-ext-/thread_fd/test_thread_fd_close.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true -require 'test/unit' -require '-test-/thread_fd' - -class TestThreadFdClose < Test::Unit::TestCase - - def test_thread_fd_close - IO.pipe do |r, w| - th = Thread.new do - begin - assert_raise(IOError) { - r.read(4) - } - ensure - w.syswrite('done') - end - end - Thread.pass until th.stop? - IO.thread_fd_close(r.fileno) - assert_equal 'done', r.read(4) - th.join - end - end -end diff --git a/test/json/json_ext_parser_test.rb b/test/json/json_ext_parser_test.rb index 8aa626257e..e610f642f1 100644 --- a/test/json/json_ext_parser_test.rb +++ b/test/json/json_ext_parser_test.rb @@ -14,16 +14,35 @@ class JSONExtParserTest < Test::Unit::TestCase end def test_error_messages - ex = assert_raise(ParserError) { parse('Infinity') } - assert_equal "unexpected token at 'Infinity'", ex.message + ex = assert_raise(ParserError) { parse('Infinity something') } + unless RUBY_PLATFORM =~ /java/ + assert_equal "unexpected token 'Infinity' at line 1 column 1", ex.message + end + ex = assert_raise(ParserError) { parse('foo bar') } unless RUBY_PLATFORM =~ /java/ - ex = assert_raise(ParserError) { parse('-Infinity') } - assert_equal "unexpected token at '-Infinity'", ex.message + assert_equal "unexpected token 'foo' at line 1 column 1", ex.message end - ex = assert_raise(ParserError) { parse('NaN') } - assert_equal "unexpected token at 'NaN'", ex.message + ex = assert_raise(ParserError) { parse('-Infinity something') } + unless RUBY_PLATFORM =~ /java/ + assert_equal "unexpected token '-Infinity' at line 1 column 1", ex.message + end + + ex = assert_raise(ParserError) { parse('NaN something') } + unless RUBY_PLATFORM =~ /java/ + assert_equal "unexpected token 'NaN' at line 1 column 1", ex.message + end + + ex = assert_raise(ParserError) { parse(' ') } + unless RUBY_PLATFORM =~ /java/ + assert_equal "unexpected end of input at line 1 column 4", ex.message + end + + ex = assert_raise(ParserError) { parse('{ ') } + unless RUBY_PLATFORM =~ /java/ + assert_equal "expected object key, got EOF at line 1 column 5", ex.message + end end if GC.respond_to?(:stress=) diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 3192b555a3..55a3065ae5 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -771,6 +771,14 @@ class JSONGeneratorTest < Test::Unit::TestCase values = [-1.0, 1.0, 0.0, 12.2, 7.5 / 3.2, 12.0, 100.0, 1000.0] expecteds = ["-1.0", "1.0", "0.0", "12.2", "2.34375", "12.0", "100.0", "1000.0"] + if RUBY_ENGINE == "jruby" + values << 1746861937.7842371 + expecteds << "1.7468619377842371E9" + else + values << 1746861937.7842371 + expecteds << "1746861937.7842371" + end + values.zip(expecteds).each do |value, expected| assert_equal expected, value.to_json end diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 87b78fb0ca..befc80c958 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -638,7 +638,7 @@ class JSONParserTest < Test::Unit::TestCase error = assert_raise(JSON::ParserError) do JSON.parse('{"foo": ' + ('A' * 500) + '}') end - assert_operator 60, :>, error.message.bytesize + assert_operator 80, :>, error.message.bytesize end def test_parse_error_incomplete_hash @@ -646,7 +646,7 @@ class JSONParserTest < Test::Unit::TestCase JSON.parse('{"input":{"firstName":"Bob","lastName":"Mob","email":"[email protected]"}') end if RUBY_ENGINE == "ruby" - assert_equal %(expected ',' or '}' after object value, got: ''), error.message + assert_equal %(expected ',' or '}' after object value, got: EOF at line 1 column 72), error.message end end @@ -654,16 +654,16 @@ class JSONParserTest < Test::Unit::TestCase omit "C ext only test" unless RUBY_ENGINE == "ruby" error = assert_raise(JSON::ParserError) { JSON.parse("あああああああああああああああああああああああ") } - assert_equal "unexpected character: 'ああああああああああ'", error.message + assert_equal "unexpected character: 'ああああああああああ' at line 1 column 1", error.message error = assert_raise(JSON::ParserError) { JSON.parse("aあああああああああああああああああああああああ") } - assert_equal "unexpected character: 'aああああああああああ'", error.message + assert_equal "unexpected character: 'aああああああああああ' at line 1 column 1", error.message error = assert_raise(JSON::ParserError) { JSON.parse("abあああああああああああああああああああああああ") } - assert_equal "unexpected character: 'abあああああああああ'", error.message + assert_equal "unexpected character: 'abあああああああああ' at line 1 column 1", error.message error = assert_raise(JSON::ParserError) { JSON.parse("abcあああああああああああああああああああああああ") } - assert_equal "unexpected character: 'abcあああああああああ'", error.message + assert_equal "unexpected character: 'abcあああああああああ' at line 1 column 1", error.message end def test_parse_leading_slash diff --git a/test/psych/test_stringio.rb b/test/psych/test_stringio.rb new file mode 100644 index 0000000000..7fef1402a0 --- /dev/null +++ b/test/psych/test_stringio.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true +require_relative 'helper' + +module Psych + class TestStringIO < TestCase + # The superclass of StringIO before Ruby 3.0 was `Data`, + # which can interfere with the Ruby 3.2+ `Data` dumping. + def test_stringio + assert_nothing_raised do + Psych.dump(StringIO.new("foo")) + end + end + end +end diff --git a/test/ruby/test_class.rb b/test/ruby/test_class.rb index 6fc461ed08..9a34a81334 100644 --- a/test/ruby/test_class.rb +++ b/test/ruby/test_class.rb @@ -283,12 +283,8 @@ class TestClass < Test::Unit::TestCase assert_raise(TypeError, bug6863) { Class.new(Class.allocate) } allocator = Class.instance_method(:allocate) - assert_raise_with_message(TypeError, /prohibited/) { - allocator.bind(Rational).call - } - assert_raise_with_message(TypeError, /prohibited/) { - allocator.bind_call(Rational) - } + assert_nothing_raised { allocator.bind(Rational).call } + assert_nothing_raised { allocator.bind_call(Rational) } end def test_nonascii_name diff --git a/test/ruby/test_hash.rb b/test/ruby/test_hash.rb index b6c18ea958..76af5b6183 100644 --- a/test/ruby/test_hash.rb +++ b/test/ruby/test_hash.rb @@ -1853,6 +1853,14 @@ class TestHash < Test::Unit::TestCase end end assert_equal(@cls[a: 2, b: 2, c: 3, d: 4, e: 5, f: 6, g: 7, h: 8, i: 9, j: 10], x) + + x = (1..1337).to_h {|k| [k, k]} + assert_raise_with_message(RuntimeError, /rehash during iteration/) do + x.transform_values! {|v| + x.rehash if v == 1337 + v * 2 + } + end end def hrec h, n, &b diff --git a/test/ruby/test_io.rb b/test/ruby/test_io.rb index 32d7519bdf..a81d689355 100644 --- a/test/ruby/test_io.rb +++ b/test/ruby/test_io.rb @@ -3826,7 +3826,7 @@ __END__ end tempfiles = [] - (0..fd_setsize+1).map {|i| + (0...fd_setsize).map {|i| tempfiles << Tempfile.create("test_io_select_with_many_files") } diff --git a/test/ruby/test_object_id.rb b/test/ruby/test_object_id.rb index 2277bba634..97ed70d839 100644 --- a/test/ruby/test_object_id.rb +++ b/test/ruby/test_object_id.rb @@ -1,4 +1,5 @@ require 'test/unit' +require "securerandom" class TestObjectId < Test::Unit::TestCase def setup @@ -159,14 +160,14 @@ class TestObjectIdTooComplexClass < TestObjectId @obj = TooComplex.new - @obj.instance_variable_set("@___#{rand(100_000)}", 1) + @obj.instance_variable_set("@___#{SecureRandom.hex}", 1) 8.times do |i| @obj.instance_variable_set("@TestObjectIdTooComplexClass#{i}", 1) @obj.remove_instance_variable("@TestObjectIdTooComplexClass#{i}") end - @obj.instance_variable_set("@___#{rand(100_000)}", 1) + @obj.instance_variable_set("@test", 1) if defined?(RubyVM::Shape) assert_predicate(RubyVM::Shape.of(@obj), :too_complex?) diff --git a/test/ruby/test_shapes.rb b/test/ruby/test_shapes.rb index 94670a4055..0458b3235b 100644 --- a/test/ruby/test_shapes.rb +++ b/test/ruby/test_shapes.rb @@ -681,8 +681,8 @@ class TestShapes < Test::Unit::TestCase id_shape = RubyVM::Shape.of(tc) refute_equal frozen_shape.id, id_shape.id assert_predicate id_shape, :too_complex? - assert_predicate id_shape, :shape_frozen? assert_predicate id_shape, :has_object_id? + assert_predicate id_shape, :shape_frozen? assert_equal 3, tc.very_unique assert_equal 3, Ractor.make_shareable(tc).very_unique diff --git a/test/rubygems/test_gem_safe_marshal.rb b/test/rubygems/test_gem_safe_marshal.rb index deeb8205bc..bd15f4f796 100644 --- a/test/rubygems/test_gem_safe_marshal.rb +++ b/test/rubygems/test_gem_safe_marshal.rb @@ -234,8 +234,6 @@ class TestGemSafeMarshal < Gem::TestCase end def test_link_after_float - pend "Marshal.load of links and floats is broken on truffleruby, see https://github.com/oracle/truffleruby/issues/3747" if RUBY_ENGINE == "truffleruby" - a = [] a << a assert_safe_load_as [0.0, a, 1.0, a] diff --git a/test/rubygems/test_gem_specification.rb b/test/rubygems/test_gem_specification.rb index 3f8d921467..7cb7ee1605 100644 --- a/test/rubygems/test_gem_specification.rb +++ b/test/rubygems/test_gem_specification.rb @@ -3664,8 +3664,6 @@ Did you mean 'Ruby'? end def test__load_fixes_Date_objects - pend "Marshal.load of links and floats is broken on truffleruby, see https://github.com/oracle/truffleruby/issues/3747" if RUBY_ENGINE == "truffleruby" - spec = util_spec "a", 1 spec.instance_variable_set :@date, Date.today diff --git a/test/stringio/test_stringio.rb b/test/stringio/test_stringio.rb index 570b3d7ad6..002b946b6f 100644 --- a/test/stringio/test_stringio.rb +++ b/test/stringio/test_stringio.rb @@ -1057,6 +1057,13 @@ class TestStringIO < Test::Unit::TestCase assert_equal("test", io.string) assert_same(chilled_string, io.string) end + + def test_chilled_string_set_enocoding + chilled_string = eval(%{""}) + io = StringIO.new(chilled_string) + assert_warning("") { io.set_encoding(Encoding::BINARY) } + assert_same(chilled_string, io.string) + end end private @@ -99,6 +99,8 @@ #include "vm_debug.h" #include "vm_sync.h" +#include "ccan/list/list.h" + #ifndef USE_NATIVE_THREAD_PRIORITY #define USE_NATIVE_THREAD_PRIORITY 0 #define RUBY_THREAD_PRIORITY_MAX 3 @@ -149,13 +151,6 @@ MAYBE_UNUSED(static int consume_communication_pipe(int fd)); static volatile int system_working = 1; static rb_internal_thread_specific_key_t specific_key_count; -struct waiting_fd { - struct ccan_list_node wfd_node; /* <=> vm.waiting_fds */ - rb_thread_t *th; - int fd; - struct rb_io_close_wait_list *busy; -}; - /********************************************************************************/ #define THREAD_SYSTEM_DEPENDENT_IMPLEMENTATION @@ -340,7 +335,7 @@ unblock_function_clear(rb_thread_t *th) } static void -threadptr_interrupt_locked(rb_thread_t *th, bool trap) +threadptr_set_interrupt_locked(rb_thread_t *th, bool trap) { // th->interrupt_lock should be acquired here @@ -362,26 +357,27 @@ threadptr_interrupt_locked(rb_thread_t *th, bool trap) } static void -threadptr_interrupt(rb_thread_t *th, int trap) +threadptr_set_interrupt(rb_thread_t *th, int trap) { rb_native_mutex_lock(&th->interrupt_lock); { - threadptr_interrupt_locked(th, trap); + threadptr_set_interrupt_locked(th, trap); } rb_native_mutex_unlock(&th->interrupt_lock); } +/* Set interrupt flag on another thread or current thread, and call its UBF if it has one set */ void rb_threadptr_interrupt(rb_thread_t *th) { RUBY_DEBUG_LOG("th:%u", rb_th_serial(th)); - threadptr_interrupt(th, false); + threadptr_set_interrupt(th, false); } static void threadptr_trap_interrupt(rb_thread_t *th) { - threadptr_interrupt(th, true); + threadptr_set_interrupt(th, true); } static void @@ -530,6 +526,9 @@ thread_cleanup_func(void *th_ptr, int atfork) } rb_native_mutex_destroy(&th->interrupt_lock); +#ifndef RUBY_THREAD_PTHREAD_H + rb_native_cond_destroy(&th->ractor_waiting.cond); +#endif } static VALUE rb_threadptr_raise(rb_thread_t *, int, VALUE *); @@ -1694,44 +1693,45 @@ waitfd_to_waiting_flag(int wfd_event) return wfd_event << 1; } -static void -thread_io_setup_wfd(rb_thread_t *th, int fd, struct waiting_fd *wfd) -{ - wfd->fd = fd; - wfd->th = th; - wfd->busy = NULL; +struct io_blocking_operation_arguments { + struct rb_io *io; + struct rb_io_blocking_operation *blocking_operation; +}; - RB_VM_LOCK_ENTER(); - { - ccan_list_add(&th->vm->waiting_fds, &wfd->wfd_node); +static VALUE +io_blocking_operation_release(VALUE _arguments) { + struct io_blocking_operation_arguments *arguments = (void*)_arguments; + struct rb_io_blocking_operation *blocking_operation = arguments->blocking_operation; + + ccan_list_del(&blocking_operation->list); + + rb_io_t *io = arguments->io; + rb_thread_t *thread = io->closing_ec->thread_ptr; + rb_fiber_t *fiber = io->closing_ec->fiber_ptr; + + if (thread->scheduler != Qnil) { + rb_fiber_scheduler_unblock(thread->scheduler, io->self, rb_fiberptr_self(fiber)); + } else { + rb_thread_wakeup(thread->self); } - RB_VM_LOCK_LEAVE(); + + return Qnil; } static void -thread_io_wake_pending_closer(struct waiting_fd *wfd) +rb_io_blocking_operation_release(struct rb_io *io, struct rb_io_blocking_operation *blocking_operation) { - bool has_waiter = wfd->busy && RB_TEST(wfd->busy->wakeup_mutex); - if (has_waiter) { - rb_mutex_lock(wfd->busy->wakeup_mutex); - } + VALUE wakeup_mutex = io->wakeup_mutex; - /* Needs to be protected with RB_VM_LOCK because we don't know if - wfd is on the global list of pending FD ops or if it's on a - struct rb_io_close_wait_list close-waiter. */ - RB_VM_LOCK_ENTER(); - ccan_list_del(&wfd->wfd_node); - RB_VM_LOCK_LEAVE(); + if (RB_TEST(wakeup_mutex)) { + struct io_blocking_operation_arguments arguments = { + .io = io, + .blocking_operation = blocking_operation + }; - if (has_waiter) { - rb_thread_t *th = rb_thread_ptr(wfd->busy->closing_thread); - if (th->scheduler != Qnil) { - rb_fiber_scheduler_unblock(th->scheduler, wfd->busy->closing_thread, wfd->busy->closing_fiber); - } - else { - rb_thread_wakeup(wfd->busy->closing_thread); - } - rb_mutex_unlock(wfd->busy->wakeup_mutex); + rb_mutex_synchronize(wakeup_mutex, io_blocking_operation_release, (VALUE)&arguments); + } else { + ccan_list_del(&blocking_operation->list); } } @@ -1802,12 +1802,11 @@ rb_thread_mn_schedulable(VALUE thval) VALUE rb_thread_io_blocking_call(struct rb_io* io, rb_blocking_function_t *func, void *data1, int events) { - rb_execution_context_t *volatile ec = GET_EC(); - rb_thread_t *volatile th = rb_ec_thread_ptr(ec); + rb_execution_context_t * ec = GET_EC(); + rb_thread_t *th = rb_ec_thread_ptr(ec); RUBY_DEBUG_LOG("th:%u fd:%d ev:%d", rb_th_serial(th), io->fd, events); - struct waiting_fd waiting_fd; volatile VALUE val = Qundef; /* shouldn't be used */ volatile int saved_errno = 0; enum ruby_tag_type state; @@ -1822,7 +1821,11 @@ rb_thread_io_blocking_call(struct rb_io* io, rb_blocking_function_t *func, void // `func` or not (as opposed to some previously set value). errno = 0; - thread_io_setup_wfd(th, fd, &waiting_fd); + struct rb_io_blocking_operation blocking_operation = { + .ec = ec, + }; + ccan_list_add(&io->blocking_operations, &blocking_operation.list); + { EC_PUSH_TAG(ec); if ((state = EC_EXEC_TAG()) == TAG_NONE) { @@ -1847,15 +1850,13 @@ rb_thread_io_blocking_call(struct rb_io* io, rb_blocking_function_t *func, void th = rb_ec_thread_ptr(ec); th->mn_schedulable = prev_mn_schedulable; } - /* - * must be deleted before jump - * this will delete either from waiting_fds or on-stack struct rb_io_close_wait_list - */ - thread_io_wake_pending_closer(&waiting_fd); + + rb_io_blocking_operation_release(io, &blocking_operation); if (state) { EC_JUMP_TAG(ec, state); } + /* TODO: check func() */ RUBY_VM_CHECK_INTS_BLOCKING(ec); @@ -2426,6 +2427,7 @@ NORETURN(static void rb_threadptr_to_kill(rb_thread_t *th)); static void rb_threadptr_to_kill(rb_thread_t *th) { + VM_ASSERT(GET_THREAD() == th); rb_threadptr_pending_interrupt_clear(th); th->status = THREAD_RUNNABLE; th->to_kill = 1; @@ -2449,6 +2451,11 @@ threadptr_get_interrupts(rb_thread_t *th) static void threadptr_interrupt_exec_exec(rb_thread_t *th); +// Execute interrupts on currently running thread +// In certain situations, calling this function will raise an exception. Some examples are: +// * during VM shutdown (`rb_ractor_terminate_all`) +// * Call to Thread#exit for current thread (`rb_thread_kill`) +// * Call to Thread#raise for current thread int rb_threadptr_execute_interrupts(rb_thread_t *th, int blocking_timing) { @@ -2456,6 +2463,8 @@ rb_threadptr_execute_interrupts(rb_thread_t *th, int blocking_timing) int postponed_job_interrupt = 0; int ret = FALSE; + VM_ASSERT(GET_THREAD() == th); + if (th->ec->raised_flag) return ret; while ((interrupt = threadptr_get_interrupts(th)) != 0) { @@ -2639,76 +2648,81 @@ rb_ec_reset_raised(rb_execution_context_t *ec) return 1; } -int -rb_notify_fd_close(int fd, struct rb_io_close_wait_list *busy) +static size_t +thread_io_close_notify_all(struct rb_io *io) { - rb_vm_t *vm = GET_THREAD()->vm; - struct waiting_fd *wfd = 0, *next; - ccan_list_head_init(&busy->pending_fd_users); - int has_any; - VALUE wakeup_mutex; + RUBY_ASSERT_CRITICAL_SECTION_ENTER(); - RB_VM_LOCK_ENTER(); - { - ccan_list_for_each_safe(&vm->waiting_fds, wfd, next, wfd_node) { - if (wfd->fd == fd) { - rb_thread_t *th = wfd->th; - VALUE err; - - ccan_list_del(&wfd->wfd_node); - ccan_list_add(&busy->pending_fd_users, &wfd->wfd_node); - - wfd->busy = busy; - err = th->vm->special_exceptions[ruby_error_stream_closed]; - rb_threadptr_pending_interrupt_enque(th, err); - rb_threadptr_interrupt(th); - } - } + size_t count = 0; + rb_vm_t *vm = io->closing_ec->thread_ptr->vm; + VALUE error = vm->special_exceptions[ruby_error_stream_closed]; + + struct rb_io_blocking_operation *blocking_operation; + ccan_list_for_each(&io->blocking_operations, blocking_operation, list) { + rb_execution_context_t *ec = blocking_operation->ec; + + rb_thread_t *thread = ec->thread_ptr; + rb_threadptr_pending_interrupt_enque(thread, error); + + // This operation is slow: + rb_threadptr_interrupt(thread); + + count += 1; + } + + RUBY_ASSERT_CRITICAL_SECTION_LEAVE(); + + return count; +} + +size_t +rb_thread_io_close_interrupt(struct rb_io *io) +{ + // We guard this operation based on `io->closing_ec` -> only one thread will ever enter this function. + if (io->closing_ec) { + return 0; } - has_any = !ccan_list_empty(&busy->pending_fd_users); - busy->closing_thread = rb_thread_current(); - busy->closing_fiber = rb_fiber_current(); - wakeup_mutex = Qnil; - if (has_any) { - wakeup_mutex = rb_mutex_new(); - RBASIC_CLEAR_CLASS(wakeup_mutex); /* hide from ObjectSpace */ + // If there are no blocking operations, we are done: + if (ccan_list_empty(&io->blocking_operations)) { + return 0; } - busy->wakeup_mutex = wakeup_mutex; - RB_VM_LOCK_LEAVE(); + // Otherwise, we are now closing the IO: + rb_execution_context_t *ec = GET_EC(); + io->closing_ec = ec; + + // This is used to ensure the correct execution context is woken up after the blocking operation is interrupted: + io->wakeup_mutex = rb_mutex_new(); - /* If the caller didn't pass *busy as a pointer to something on the stack, - we need to guard this mutex object on _our_ C stack for the duration - of this function. */ - RB_GC_GUARD(wakeup_mutex); - return has_any; + return thread_io_close_notify_all(io); } void -rb_notify_fd_close_wait(struct rb_io_close_wait_list *busy) +rb_thread_io_close_wait(struct rb_io* io) { - if (!RB_TEST(busy->wakeup_mutex)) { - /* There was nobody else using this file when we closed it, so we - never bothered to allocate a mutex*/ + VALUE wakeup_mutex = io->wakeup_mutex; + + if (!RB_TEST(wakeup_mutex)) { + // There was nobody else using this file when we closed it, so we never bothered to allocate a mutex: return; } - rb_mutex_lock(busy->wakeup_mutex); - while (!ccan_list_empty(&busy->pending_fd_users)) { - rb_mutex_sleep(busy->wakeup_mutex, Qnil); + rb_mutex_lock(wakeup_mutex); + while (!ccan_list_empty(&io->blocking_operations)) { + rb_mutex_sleep(wakeup_mutex, Qnil); } - rb_mutex_unlock(busy->wakeup_mutex); + rb_mutex_unlock(wakeup_mutex); + + // We are done closing: + io->wakeup_mutex = Qnil; + io->closing_ec = NULL; } void rb_thread_fd_close(int fd) { - struct rb_io_close_wait_list busy; - - if (rb_notify_fd_close(fd, &busy)) { - rb_notify_fd_close_wait(&busy); - } + rb_warn("rb_thread_fd_close is deprecated (and is now a no-op)."); } /* @@ -4412,14 +4426,17 @@ thread_io_wait(struct rb_io *io, int fd, int events, struct timeval *timeout) }}; volatile int result = 0; nfds_t nfds; - struct waiting_fd wfd; + struct rb_io_blocking_operation blocking_operation; enum ruby_tag_type state; volatile int lerrno; rb_execution_context_t *ec = GET_EC(); rb_thread_t *th = rb_ec_thread_ptr(ec); - thread_io_setup_wfd(th, fd, &wfd); + if (io) { + blocking_operation.ec = ec; + ccan_list_add(&io->blocking_operations, &blocking_operation.list); + } if (timeout == NULL && thread_io_wait_events(th, fd, events, NULL)) { // fd is readable @@ -4428,25 +4445,27 @@ thread_io_wait(struct rb_io *io, int fd, int events, struct timeval *timeout) errno = 0; } else { - EC_PUSH_TAG(wfd.th->ec); + EC_PUSH_TAG(ec); if ((state = EC_EXEC_TAG()) == TAG_NONE) { rb_hrtime_t *to, rel, end = 0; - RUBY_VM_CHECK_INTS_BLOCKING(wfd.th->ec); + RUBY_VM_CHECK_INTS_BLOCKING(ec); timeout_prepare(&to, &rel, &end, timeout); do { nfds = numberof(fds); - result = wait_for_single_fd_blocking_region(wfd.th, fds, nfds, to, &lerrno); + result = wait_for_single_fd_blocking_region(th, fds, nfds, to, &lerrno); - RUBY_VM_CHECK_INTS_BLOCKING(wfd.th->ec); + RUBY_VM_CHECK_INTS_BLOCKING(ec); } while (wait_retryable(&result, lerrno, to, end)); } EC_POP_TAG(); } - thread_io_wake_pending_closer(&wfd); + if (io) { + rb_io_blocking_operation_release(io, &blocking_operation); + } if (state) { - EC_JUMP_TAG(wfd.th->ec, state); + EC_JUMP_TAG(ec, state); } if (result < 0) { @@ -4479,6 +4498,9 @@ thread_io_wait(struct rb_io *io, int fd, int events, struct timeval *timeout) } #else /* ! USE_POLL - implement rb_io_poll_fd() using select() */ struct select_args { + struct rb_io *io; + struct rb_io_blocking_operation *blocking_operation; + union { int fd; int error; @@ -4486,7 +4508,6 @@ struct select_args { rb_fdset_t *read; rb_fdset_t *write; rb_fdset_t *except; - struct waiting_fd wfd; struct timeval *tv; }; @@ -4517,7 +4538,10 @@ select_single_cleanup(VALUE ptr) { struct select_args *args = (struct select_args *)ptr; - thread_io_wake_pending_closer(&args->wfd); + if (args->blocking_operation) { + rb_io_blocking_operation_release(args->io, args->blocking_operation); + } + if (args->read) rb_fd_term(args->read); if (args->write) rb_fd_term(args->write); if (args->except) rb_fd_term(args->except); @@ -4542,22 +4566,31 @@ thread_io_wait(struct rb_io *io, int fd, int events, struct timeval *timeout) { rb_fdset_t rfds, wfds, efds; struct select_args args; - int r; VALUE ptr = (VALUE)&args; - rb_thread_t *th = GET_THREAD(); + + struct rb_io_blocking_operation blocking_operation; + if (io) { + args.io = io; + blocking_operation.ec = GET_EC(); + ccan_list_add(&io->blocking_operations, &blocking_operation.list); + args.blocking_operation = &blocking_operation; + } else { + args.io = NULL; + blocking_operation.ec = NULL; + args.blocking_operation = NULL; + } args.as.fd = fd; args.read = (events & RB_WAITFD_IN) ? init_set_fd(fd, &rfds) : NULL; args.write = (events & RB_WAITFD_OUT) ? init_set_fd(fd, &wfds) : NULL; args.except = (events & RB_WAITFD_PRI) ? init_set_fd(fd, &efds) : NULL; args.tv = timeout; - thread_io_setup_wfd(th, fd, &args.wfd); - r = (int)rb_ensure(select_single, ptr, select_single_cleanup, ptr); - if (r == -1) + int result = (int)rb_ensure(select_single, ptr, select_single_cleanup, ptr); + if (result == -1) errno = args.as.error; - return r; + return result; } #endif /* ! USE_POLL */ @@ -5651,21 +5684,6 @@ rb_check_deadlock(rb_ractor_t *r) } } -// Used for VM memsize reporting. Returns the size of a list of waiting_fd -// structs. Defined here because the struct definition lives here as well. -size_t -rb_vm_memsize_waiting_fds(struct ccan_list_head *waiting_fds) -{ - struct waiting_fd *waitfd = 0; - size_t size = 0; - - ccan_list_for_each(waiting_fds, waitfd, wfd_node) { - size += sizeof(struct waiting_fd); - } - - return size; -} - static void update_line_coverage(VALUE data, const rb_trace_arg_t *trace_arg) { @@ -6027,7 +6045,7 @@ rb_threadptr_interrupt_exec(rb_thread_t *th, rb_interrupt_exec_func_t *func, voi rb_native_mutex_lock(&th->interrupt_lock); { ccan_list_add_tail(&th->interrupt_exec_tasks, &task->node); - threadptr_interrupt_locked(th, true); + threadptr_set_interrupt_locked(th, true); } rb_native_mutex_unlock(&th->interrupt_lock); } diff --git a/thread_pthread.c b/thread_pthread.c index fc96622592..7811d5afbf 100644 --- a/thread_pthread.c +++ b/thread_pthread.c @@ -1309,16 +1309,20 @@ ractor_sched_deq(rb_vm_t *vm, rb_ractor_t *cr) void rb_ractor_lock_self(rb_ractor_t *r); void rb_ractor_unlock_self(rb_ractor_t *r); +// The current thread for a ractor is put to "sleep" (descheduled in the STOPPED_FOREVER state) waiting for +// a ractor action to wake it up. See docs for `ractor_sched_sleep_with_cleanup` for more info. void -rb_ractor_sched_sleep(rb_execution_context_t *ec, rb_ractor_t *cr, rb_unblock_function_t *ubf) +rb_ractor_sched_sleep(rb_execution_context_t *ec, rb_ractor_t *cr, rb_unblock_function_t *ubf_schedule_ractor_th) { // ractor lock of cr is acquired // r is sleeping status rb_thread_t * volatile th = rb_ec_thread_ptr(ec); struct rb_thread_sched *sched = TH_SCHED(th); - cr->sync.wait.waiting_thread = th; // TODO: multi-thread + struct ccan_list_node *waitn = &th->ractor_waiting.waiting_node; + VM_ASSERT(waitn->next == waitn->prev && waitn->next == waitn); // it should be unlinked + ccan_list_add(&cr->sync.wait.waiting_threads, waitn); - setup_ubf(th, ubf, (void *)cr); + setup_ubf(th, ubf_schedule_ractor_th, (void *)ec); thread_sched_lock(sched, th); { @@ -1327,8 +1331,8 @@ rb_ractor_sched_sleep(rb_execution_context_t *ec, rb_ractor_t *cr, rb_unblock_fu if (RUBY_VM_INTERRUPTED(th->ec)) { RUBY_DEBUG_LOG("interrupted"); } - else if (cr->sync.wait.wakeup_status != wakeup_none) { - RUBY_DEBUG_LOG("awaken:%d", (int)cr->sync.wait.wakeup_status); + else if (th->ractor_waiting.wakeup_status != wakeup_none) { + RUBY_DEBUG_LOG("awaken:%d", (int)th->ractor_waiting.wakeup_status); } else { // sleep @@ -1350,25 +1354,24 @@ rb_ractor_sched_sleep(rb_execution_context_t *ec, rb_ractor_t *cr, rb_unblock_fu setup_ubf(th, NULL, NULL); rb_ractor_lock_self(cr); - cr->sync.wait.waiting_thread = NULL; + ccan_list_del_init(waitn); } void -rb_ractor_sched_wakeup(rb_ractor_t *r) +rb_ractor_sched_wakeup(rb_ractor_t *r, rb_thread_t *th) { - rb_thread_t *r_th = r->sync.wait.waiting_thread; // ractor lock of r is acquired - struct rb_thread_sched *sched = TH_SCHED(r_th); + struct rb_thread_sched *sched = TH_SCHED(th); - VM_ASSERT(r->sync.wait.wakeup_status != 0); + VM_ASSERT(th->ractor_waiting.wakeup_status != 0); - thread_sched_lock(sched, r_th); + thread_sched_lock(sched, th); { - if (r_th->status == THREAD_STOPPED_FOREVER) { - thread_sched_to_ready_common(sched, r_th, true, false); + if (th->status == THREAD_STOPPED_FOREVER) { + thread_sched_to_ready_common(sched, th, true, false); } } - thread_sched_unlock(sched, r_th); + thread_sched_unlock(sched, th); } static bool diff --git a/tool/bundler/test_gems.rb b/tool/bundler/test_gems.rb index 80e4049ad6..7cbea1c83c 100644 --- a/tool/bundler/test_gems.rb +++ b/tool/bundler/test_gems.rb @@ -3,7 +3,7 @@ source "https://rubygems.org" gem "rack", "~> 3.0" -gem "cgi", "~> 0.5.0.beta1" +gem "cgi", "~> 0.5.0.beta2" gem "rackup", "~> 2.1" gem "webrick", "~> 1.9" gem "rack-test", "~> 2.1" diff --git a/tool/bundler/test_gems.rb.lock b/tool/bundler/test_gems.rb.lock index db21fe0c4a..cb3b67d5dd 100644 --- a/tool/bundler/test_gems.rb.lock +++ b/tool/bundler/test_gems.rb.lock @@ -3,8 +3,8 @@ GEM specs: base64 (0.2.0) builder (3.3.0) - cgi (0.5.0.beta1) - cgi (0.5.0.beta1-java) + cgi (0.5.0.beta2) + cgi (0.5.0.beta2-java) compact_index (0.15.0) fiddle (1.1.6) logger (1.7.0) @@ -49,7 +49,7 @@ PLATFORMS DEPENDENCIES builder (~> 3.2) - cgi (~> 0.5.0.beta1) + cgi (~> 0.5.0.beta2) compact_index (~> 0.15.0) fiddle rack (~> 3.0) @@ -64,8 +64,8 @@ DEPENDENCIES CHECKSUMS base64 (0.2.0) sha256=0f25e9b21a02a0cc0cea8ef92b2041035d39350946e8789c562b2d1a3da01507 builder (3.3.0) sha256=497918d2f9dca528fdca4b88d84e4ef4387256d984b8154e9d5d3fe5a9c8835f - cgi (0.5.0.beta1) sha256=e89d89608613f9bd053b82268cf202a64a589816d5b0f4f0a313945305d83b6d - cgi (0.5.0.beta1-java) sha256=1f638a6a14879ca2c78d7d42835efe7a27be8f39d4fad44d6e226fb877bbb8e2 + cgi (0.5.0.beta2) sha256=0721a87b0fe40fc403af3c5569f117c1133f6d6cf6a0b88a8248af7ae5209129 + cgi (0.5.0.beta2-java) sha256=05c61b1c58c3ee9c7e0b0efcd5b2b85a9e97fd5a4504a76ecf71fc1606e19328 compact_index (0.15.0) sha256=5c6c404afca8928a7d9f4dde9524f6e1610db17e675330803055db282da84a8b fiddle (1.1.6) sha256=79e8d909e602d979434cf9fccfa6e729cb16432bb00e39c7596abe6bee1249ab logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203 diff --git a/tool/bundler/vendor_gems.rb b/tool/bundler/vendor_gems.rb index 467bbf33b3..2e9fb7f18f 100644 --- a/tool/bundler/vendor_gems.rb +++ b/tool/bundler/vendor_gems.rb @@ -4,8 +4,8 @@ source "https://rubygems.org" gem "fileutils", "1.7.3" gem "molinillo", github: "cocoapods/molinillo" -gem "net-http", "0.6.0" -gem "net-http-persistent", "4.0.4" +gem "net-http", github: "ruby/net-http", ref: "d8fd39c589279b1aaec85a7c8de9b3e199c72efe" +gem "net-http-persistent", github: "hsbt/net-http-persistent", ref: "9b6fbd733cf35596dfe7f80c0c154f9f3d17dbdb" gem "net-protocol", "0.2.2" gem "optparse", "0.6.0" gem "pub_grub", github: "jhawthorn/pub_grub", ref: "df6add45d1b4d122daff2f959c9bd1ca93d14261" diff --git a/tool/bundler/vendor_gems.rb.lock b/tool/bundler/vendor_gems.rb.lock index 55ac15bd03..0fb79079b2 100644 --- a/tool/bundler/vendor_gems.rb.lock +++ b/tool/bundler/vendor_gems.rb.lock @@ -5,21 +5,33 @@ GIT molinillo (0.8.0) GIT + remote: https://github.com/hsbt/net-http-persistent.git + revision: 9b6fbd733cf35596dfe7f80c0c154f9f3d17dbdb + ref: 9b6fbd733cf35596dfe7f80c0c154f9f3d17dbdb + specs: + net-http-persistent (4.0.6) + connection_pool (~> 2.2, >= 2.2.4) + +GIT remote: https://github.com/jhawthorn/pub_grub.git revision: df6add45d1b4d122daff2f959c9bd1ca93d14261 ref: df6add45d1b4d122daff2f959c9bd1ca93d14261 specs: pub_grub (0.5.0) +GIT + remote: https://github.com/ruby/net-http.git + revision: d8fd39c589279b1aaec85a7c8de9b3e199c72efe + ref: d8fd39c589279b1aaec85a7c8de9b3e199c72efe + specs: + net-http (0.6.0) + uri + GEM remote: https://rubygems.org/ specs: connection_pool (2.4.1) fileutils (1.7.3) - net-http (0.6.0) - uri - net-http-persistent (4.0.4) - connection_pool (~> 2.2) net-protocol (0.2.2) timeout optparse (0.6.0) @@ -41,8 +53,8 @@ PLATFORMS DEPENDENCIES fileutils (= 1.7.3) molinillo! - net-http (= 0.6.0) - net-http-persistent (= 4.0.4) + net-http! + net-http-persistent! net-protocol (= 0.2.2) optparse (= 0.6.0) pub_grub! @@ -57,8 +69,8 @@ CHECKSUMS connection_pool (2.4.1) sha256=0f40cf997091f1f04ff66da67eabd61a9fe0d4928b9a3645228532512fab62f4 fileutils (1.7.3) sha256=57271e854b694a87755d76f836f5c57b2c9538ebbaf4b2154bb66addf15eb5da molinillo (0.8.0) - net-http (0.6.0) sha256=9621b20c137898af9d890556848c93603716cab516dc2c89b01a38b894e259fb - net-http-persistent (4.0.4) sha256=b62b7e528f72890f5cd67e7f687eeae55cf9f2cda22fd659494363da2fa7f4b2 + net-http (0.6.0) + net-http-persistent (4.0.6) net-protocol (0.2.2) sha256=aa73e0cba6a125369de9837b8d8ef82a61849360eba0521900e2c3713aa162a8 optparse (0.6.0) sha256=25e90469c1cd44048a89dc01c1dde9d5f0bdf717851055fb18237780779b068c pub_grub (0.5.0) diff --git a/tool/missing-baseruby.bat b/tool/missing-baseruby.bat index fcc75ea902..9274482a4c 100755 --- a/tool/missing-baseruby.bat +++ b/tool/missing-baseruby.bat @@ -18,6 +18,6 @@ : ; abort () { exit 1; } call :warn "executable host ruby is required. use --with-baseruby option." -call :warn "Note that BASERUBY must be Ruby 3.0.0 or later." +call :warn "Note that BASERUBY must be Ruby 3.1.0 or later." call :abort : || (:^; abort if RUBY_VERSION < s[%r"warn .*Ruby ([\d.]+)(?:\.0)?",1]) diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index 5b0bca72b7..932f37b77c 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -236,12 +236,13 @@ module SyncDefaultGems when "cgi" rm_rf(%w[lib/cgi.rb lib/cgi ext/cgi test/cgi]) cp_r("#{upstream}/ext/cgi", "ext") - cp_r("#{upstream}/lib/cgi", "lib") - cp_r("#{upstream}/lib/cgi.rb", "lib") + mkdir_p("lib/cgi") + cp_r("#{upstream}/lib/cgi/escape.rb", "lib/cgi") + mkdir_p("test/cgi") + cp_r("#{upstream}/test/cgi/test_cgi_escape.rb", "test/cgi") + cp_r("#{upstream}/test/cgi/update_env.rb", "test/cgi") rm_rf("lib/cgi/escape.jar") - cp_r("#{upstream}/test/cgi", "test") - cp_r("#{upstream}/cgi.gemspec", "lib/cgi") - `git checkout ext/cgi/escape/depend` + `git checkout lib/cgi.rb lib/cgi/util.rb ext/cgi/escape/depend` when "openssl" rm_rf(%w[ext/openssl test/openssl]) cp_r("#{upstream}/ext/openssl", "ext") @@ -364,8 +365,7 @@ module SyncDefaultGems end def check_prerelease_version(gem) - return if gem == "rubygems" - return if gem == "mmtk" + return if ["rubygems", "mmtk", "cgi"].include?(gem) gem = gem.downcase diff --git a/variable.c b/variable.c index b6f1abdc93..d7f9579d9c 100644 --- a/variable.c +++ b/variable.c @@ -1345,12 +1345,12 @@ gen_fields_tbl_count(VALUE obj, const struct gen_fields_tbl *fields_tbl) } VALUE -rb_obj_field_get(VALUE obj, rb_shape_t *target_shape) +rb_obj_field_get(VALUE obj, shape_id_t target_shape_id) { RUBY_ASSERT(!SPECIAL_CONST_P(obj)); - RUBY_ASSERT(target_shape->type == SHAPE_IVAR || target_shape->type == SHAPE_OBJ_ID); + RUBY_ASSERT(RSHAPE(target_shape_id)->type == SHAPE_IVAR || RSHAPE(target_shape_id)->type == SHAPE_OBJ_ID); - if (rb_shape_too_complex_p(target_shape)) { + if (rb_shape_id_too_complex_p(target_shape_id)) { st_table *fields_hash; switch (BUILTIN_TYPE(obj)) { case T_CLASS: @@ -1370,12 +1370,12 @@ rb_obj_field_get(VALUE obj, rb_shape_t *target_shape) break; } VALUE value = Qundef; - st_lookup(fields_hash, target_shape->edge_name, &value); + st_lookup(fields_hash, RSHAPE(target_shape_id)->edge_name, &value); RUBY_ASSERT(!UNDEF_P(value)); return value; } - attr_index_t attr_index = target_shape->next_field_index - 1; + attr_index_t attr_index = RSHAPE(target_shape_id)->next_field_index - 1; VALUE *fields; switch (BUILTIN_TYPE(obj)) { case T_CLASS: @@ -1689,7 +1689,7 @@ static struct general_ivar_set_result general_ivar_set(VALUE obj, ID id, VALUE val, void *data, VALUE *(*shape_fields_func)(VALUE, void *), void (*shape_resize_fields_func)(VALUE, attr_index_t, attr_index_t, void *), - void (*set_shape_func)(VALUE, rb_shape_t *, void *), + void (*set_shape_id_func)(VALUE, shape_id_t, void *), void (*transition_too_complex_func)(VALUE, void *), st_table *(*too_complex_table_func)(VALUE, void *)) { @@ -1726,7 +1726,7 @@ general_ivar_set(VALUE obj, ID id, VALUE val, void *data, RUBY_ASSERT(next_shape->type == SHAPE_IVAR); RUBY_ASSERT(index == (next_shape->next_field_index - 1)); - set_shape_func(obj, next_shape, data); + set_shape_id_func(obj, next_shape_id, data); } VALUE *table = shape_fields_func(obj, data); @@ -1748,34 +1748,36 @@ too_complex: } static void -general_field_set(VALUE obj, rb_shape_t *target_shape, VALUE val, void *data, +general_field_set(VALUE obj, shape_id_t target_shape_id, VALUE val, void *data, VALUE *(*shape_fields_func)(VALUE, void *), void (*shape_resize_fields_func)(VALUE, attr_index_t, attr_index_t, void *), - void (*set_shape_func)(VALUE, rb_shape_t *, void *), + void (*set_shape_id_func)(VALUE, shape_id_t, void *), void (*transition_too_complex_func)(VALUE, void *), st_table *(*too_complex_table_func)(VALUE, void *)) { rb_shape_t *current_shape = rb_obj_shape(obj); - if (UNLIKELY(rb_shape_too_complex_p(target_shape))) { + if (UNLIKELY(rb_shape_id_too_complex_p(target_shape_id))) { if (UNLIKELY(!rb_shape_too_complex_p(current_shape))) { transition_too_complex_func(obj, data); } - set_shape_func(obj, target_shape, data); - st_table *table = too_complex_table_func(obj, data); - st_insert(table, (st_data_t)target_shape->edge_name, (st_data_t)val); + if (RSHAPE(target_shape_id)->next_field_index > current_shape->next_field_index) { + set_shape_id_func(obj, target_shape_id, data); + } + + st_insert(table, (st_data_t)RSHAPE(target_shape_id)->edge_name, (st_data_t)val); RB_OBJ_WRITTEN(obj, Qundef, val); } else { - attr_index_t index = target_shape->next_field_index - 1; + attr_index_t index = RSHAPE(target_shape_id)->next_field_index - 1; if (index >= current_shape->capacity) { - shape_resize_fields_func(obj, current_shape->capacity, target_shape->capacity, data); + shape_resize_fields_func(obj, current_shape->capacity, RSHAPE(target_shape_id)->capacity, data); } - if (target_shape->next_field_index > current_shape->next_field_index) { - set_shape_func(obj, target_shape, data); + if (RSHAPE(target_shape_id)->next_field_index > current_shape->next_field_index) { + set_shape_id_func(obj, target_shape_id, data); } VALUE *table = shape_fields_func(obj, data); @@ -1787,7 +1789,7 @@ struct gen_fields_lookup_ensure_size { VALUE obj; ID id; struct gen_fields_tbl *fields_tbl; - rb_shape_t *shape; + shape_id_t shape_id; bool resize; }; @@ -1801,25 +1803,25 @@ generic_fields_lookup_ensure_size(st_data_t *k, st_data_t *v, st_data_t u, int e if (!existing || fields_lookup->resize) { if (existing) { - RUBY_ASSERT(fields_lookup->shape->type == SHAPE_IVAR || fields_lookup->shape->type == SHAPE_OBJ_ID); - RUBY_ASSERT(RSHAPE(fields_lookup->shape->parent_id)->capacity < fields_lookup->shape->capacity); + RUBY_ASSERT(RSHAPE(fields_lookup->shape_id)->type == SHAPE_IVAR || RSHAPE(fields_lookup->shape_id)->type == SHAPE_OBJ_ID); + RUBY_ASSERT(RSHAPE(RSHAPE(fields_lookup->shape_id)->parent_id)->capacity < RSHAPE(fields_lookup->shape_id)->capacity); } else { FL_SET_RAW((VALUE)*k, FL_EXIVAR); } - fields_tbl = gen_fields_tbl_resize(fields_tbl, fields_lookup->shape->capacity); + fields_tbl = gen_fields_tbl_resize(fields_tbl, RSHAPE(fields_lookup->shape_id)->capacity); *v = (st_data_t)fields_tbl; } RUBY_ASSERT(FL_TEST((VALUE)*k, FL_EXIVAR)); fields_lookup->fields_tbl = fields_tbl; - if (fields_lookup->shape) { + if (fields_lookup->shape_id) { #if SHAPE_IN_BASIC_FLAGS - rb_shape_set_shape(fields_lookup->obj, fields_lookup->shape); + rb_shape_set_shape_id(fields_lookup->obj, fields_lookup->shape_id); #else - fields_tbl->shape_id = rb_shape_id(fields_lookup->shape); + fields_tbl->shape_id = fields_lookup->shape_id; #endif } @@ -1853,11 +1855,11 @@ generic_ivar_set_shape_resize_fields(VALUE obj, attr_index_t _old_capa, attr_ind } static void -generic_ivar_set_set_shape(VALUE obj, rb_shape_t *shape, void *data) +generic_ivar_set_set_shape_id(VALUE obj, shape_id_t shape_id, void *data) { struct gen_fields_lookup_ensure_size *fields_lookup = data; - fields_lookup->shape = shape; + fields_lookup->shape_id = shape_id; } static void @@ -1901,30 +1903,28 @@ generic_ivar_set(VALUE obj, ID id, VALUE val) .obj = obj, .id = id, .resize = false, - .shape = NULL, }; general_ivar_set(obj, id, val, &fields_lookup, generic_ivar_set_shape_fields, generic_ivar_set_shape_resize_fields, - generic_ivar_set_set_shape, + generic_ivar_set_set_shape_id, generic_ivar_set_transition_too_complex, generic_ivar_set_too_complex_table); } static void -generic_field_set(VALUE obj, rb_shape_t *target_shape, VALUE val) +generic_field_set(VALUE obj, shape_id_t target_shape_id, VALUE val) { struct gen_fields_lookup_ensure_size fields_lookup = { .obj = obj, .resize = false, - .shape = NULL, }; - general_field_set(obj, target_shape, val, &fields_lookup, + general_field_set(obj, target_shape_id, val, &fields_lookup, generic_ivar_set_shape_fields, generic_ivar_set_shape_resize_fields, - generic_ivar_set_set_shape, + generic_ivar_set_set_shape_id, generic_ivar_set_transition_too_complex, generic_ivar_set_too_complex_table); } @@ -1982,9 +1982,9 @@ obj_ivar_set_shape_resize_fields(VALUE obj, attr_index_t old_capa, attr_index_t } static void -obj_ivar_set_set_shape(VALUE obj, rb_shape_t *shape, void *_data) +obj_ivar_set_set_shape_id(VALUE obj, shape_id_t shape_id, void *_data) { - rb_shape_set_shape(obj, shape); + rb_shape_set_shape_id(obj, shape_id); } static void @@ -2007,18 +2007,18 @@ rb_obj_ivar_set(VALUE obj, ID id, VALUE val) return general_ivar_set(obj, id, val, NULL, obj_ivar_set_shape_fields, obj_ivar_set_shape_resize_fields, - obj_ivar_set_set_shape, + obj_ivar_set_set_shape_id, obj_ivar_set_transition_too_complex, obj_ivar_set_too_complex_table).index; } static void -obj_field_set(VALUE obj, rb_shape_t *target_shape, VALUE val) +obj_field_set(VALUE obj, shape_id_t target_shape_id, VALUE val) { - general_field_set(obj, target_shape, val, NULL, + general_field_set(obj, target_shape_id, val, NULL, obj_ivar_set_shape_fields, obj_ivar_set_shape_resize_fields, - obj_ivar_set_set_shape, + obj_ivar_set_set_shape_id, obj_ivar_set_transition_too_complex, obj_ivar_set_too_complex_table); } @@ -2138,22 +2138,22 @@ rb_ivar_set_internal(VALUE obj, ID id, VALUE val) ivar_set(obj, id, val); } -static void class_field_set(VALUE obj, rb_shape_t *target_shape, VALUE val); +static void class_field_set(VALUE obj, shape_id_t target_shape_id, VALUE val); void -rb_obj_field_set(VALUE obj, rb_shape_t *target_shape, VALUE val) +rb_obj_field_set(VALUE obj, shape_id_t target_shape_id, VALUE val) { switch (BUILTIN_TYPE(obj)) { case T_OBJECT: - obj_field_set(obj, target_shape, val); + obj_field_set(obj, target_shape_id, val); break; case T_CLASS: case T_MODULE: ASSERT_vm_locking(); - class_field_set(obj, target_shape, val); + class_field_set(obj, target_shape_id, val); break; default: - generic_field_set(obj, target_shape, val); + generic_field_set(obj, target_shape_id, val); break; } } @@ -4739,9 +4739,9 @@ class_ivar_set_shape_resize_fields(VALUE obj, attr_index_t _old_capa, attr_index } static void -class_ivar_set_set_shape(VALUE obj, rb_shape_t *shape, void *_data) +class_ivar_set_set_shape_id(VALUE obj, shape_id_t shape_id, void *_data) { - rb_shape_set_shape(obj, shape); + rb_shape_set_shape_id(obj, shape_id); } static void @@ -4772,7 +4772,7 @@ rb_class_ivar_set(VALUE obj, ID id, VALUE val) existing = general_ivar_set(obj, id, val, NULL, class_ivar_set_shape_fields, class_ivar_set_shape_resize_fields, - class_ivar_set_set_shape, + class_ivar_set_set_shape_id, class_ivar_set_transition_too_complex, class_ivar_set_too_complex_table).existing; } @@ -4782,13 +4782,13 @@ rb_class_ivar_set(VALUE obj, ID id, VALUE val) } static void -class_field_set(VALUE obj, rb_shape_t *target_shape, VALUE val) +class_field_set(VALUE obj, shape_id_t target_shape_id, VALUE val) { RUBY_ASSERT(RB_TYPE_P(obj, T_CLASS) || RB_TYPE_P(obj, T_MODULE)); - general_field_set(obj, target_shape, val, NULL, + general_field_set(obj, target_shape_id, val, NULL, class_ivar_set_shape_fields, class_ivar_set_shape_resize_fields, - class_ivar_set_set_shape, + class_ivar_set_set_shape_id, class_ivar_set_transition_too_complex, class_ivar_set_too_complex_table); } @@ -3230,7 +3230,6 @@ ruby_vm_destruct(rb_vm_t *vm) return 0; } -size_t rb_vm_memsize_waiting_fds(struct ccan_list_head *waiting_fds); // thread.c size_t rb_vm_memsize_workqueue(struct ccan_list_head *workqueue); // vm_trace.c // Used for VM memsize reporting. Returns the size of the at_exit list by @@ -3285,7 +3284,6 @@ vm_memsize(const void *ptr) return ( sizeof(rb_vm_t) + - rb_vm_memsize_waiting_fds(&vm->waiting_fds) + rb_st_memsize(vm->loaded_features_index) + rb_st_memsize(vm->loading_table) + rb_vm_memsize_postponed_job_queue() + @@ -3558,6 +3556,7 @@ thread_mark(void *ptr) rb_gc_mark(th->last_status); rb_gc_mark(th->locking_mutex); rb_gc_mark(th->name); + rb_gc_mark(th->ractor_waiting.receiving_mutex); rb_gc_mark(th->scheduler); @@ -3719,6 +3718,10 @@ th_init(rb_thread_t *th, VALUE self, rb_vm_t *vm) th->ext_config.ractor_safe = true; ccan_list_head_init(&th->interrupt_exec_tasks); + ccan_list_node_init(&th->ractor_waiting.waiting_node); +#ifndef RUBY_THREAD_PTHREAD_H + rb_native_cond_initialize(&th->ractor_waiting.cond); +#endif #if USE_RUBY_DEBUG_LOG static rb_atomic_t thread_serial = 1; @@ -1106,8 +1106,20 @@ typedef struct rb_ractor_struct rb_ractor_t; struct rb_native_thread; +struct rb_thread_ractor_waiting { + //enum rb_ractor_wait_status wait_status; + int wait_status; + //enum rb_ractor_wakeup_status wakeup_status; + int wakeup_status; + struct ccan_list_node waiting_node; // the rb_thread_t + VALUE receiving_mutex; // protects Ractor.receive_if +#ifndef RUBY_THREAD_PTHREAD_H + rb_nativethread_cond_t cond; +#endif +}; + typedef struct rb_thread_struct { - struct ccan_list_node lt_node; // managed by a ractor + struct ccan_list_node lt_node; // managed by a ractor (r->threads.set) VALUE self; rb_ractor_t *ractor; rb_vm_t *vm; @@ -1118,6 +1130,8 @@ typedef struct rb_thread_struct { bool mn_schedulable; rb_atomic_t serial; // only for RUBY_DEBUG_LOG() + struct rb_thread_ractor_waiting ractor_waiting; + VALUE last_status; /* $? */ /* for cfunc */ @@ -1888,7 +1902,6 @@ void rb_thread_wakeup_timer_thread(int); static inline void rb_vm_living_threads_init(rb_vm_t *vm) { - ccan_list_head_init(&vm->waiting_fds); ccan_list_head_init(&vm->workqueue); ccan_list_head_init(&vm->ractor.set); ccan_list_head_init(&vm->ractor.sched.zombie_threads); diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index 47281c34f8..b608f98462 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -2445,6 +2445,11 @@ fn gen_getlocal_generic( ep_offset: u32, level: u32, ) -> Option<CodegenStatus> { + // Split the block if we need to invalidate this instruction when EP escapes + if level == 0 && !jit.escapes_ep() && !jit.at_compile_target() { + return jit.defer_compilation(asm); + } + let local_opnd = if level == 0 && jit.assume_no_ep_escape(asm) { // Load the local using SP register asm.local_opnd(ep_offset) @@ -2535,6 +2540,11 @@ fn gen_setlocal_generic( return Some(KeepCompiling); } + // Split the block if we need to invalidate this instruction when EP escapes + if level == 0 && !jit.escapes_ep() && !jit.at_compile_target() { + return jit.defer_compilation(asm); + } + let (flags_opnd, local_opnd) = if level == 0 && jit.assume_no_ep_escape(asm) { // Load flags and the local using SP register let flags_opnd = asm.ctx.ep_opnd(VM_ENV_DATA_INDEX_FLAGS as i32); diff --git a/zjit/src/backend/lir.rs b/zjit/src/backend/lir.rs index 2fb2dca674..5bca786d13 100644 --- a/zjit/src/backend/lir.rs +++ b/zjit/src/backend/lir.rs @@ -1678,7 +1678,7 @@ impl Assembler asm_comment!(asm, "side exit: {state}"); asm.ccall(Self::rb_zjit_side_exit as *const u8, vec![]); asm.compile(cb)?; - *target = Target::CodePtr(side_exit_ptr); + *target = Target::SideExitPtr(side_exit_ptr); } } } |