Fix infinite loop in ensure after NoMemoryError

VM patch from wanabe.

Test based on example from buzztaiki (Taiki Sugawara).

Test fails when compiles with -DRUBY_DEBUG, as that can
can use rb_bug instead of NoMemoryError, which doesn't
allow testing this case. Test also fails on MingW, as
RangeError is used instead of NoMemoryError. Skip the
test in either case.

Fixes [Bug #15779]

2 files changed, 26 insertions, 1 deletions

diff --git a/test/ruby/test_exception.rb b/test/ruby/test_exception.rb
index 62da13d1b9..0e1d280a9b 100644
--- a/test/ruby/test_exception.rb
+++ b/test/ruby/test_exception.rb
@@ -560,6 +560,31 @@ end.join
     end;
   end
 
+  def test_ensure_after_nomemoryerror
+    assert_separately([], "'a' * 1_000_000_000_000_000_000")
+  rescue NoMemoryError
+    assert_raise(NoMemoryError) do
+      assert_separately([], "#{<<~"begin;"}\n#{<<~'end;'}")
+      bug15779 = bug15779 = '[ruby-core:92342]'
+      begin;
+        require 'open-uri'
+
+        begin
+          'a' * 1_000_000_000_000_000_000
+        ensure
+          URI.open('http://www.ruby-lang.org/')
+        end
+      end;
+    end
+  rescue Minitest::Assertion
+    # Possibly compiled with -DRUBY_DEBUG, in which
+    # case rb_bug is used instead of NoMemoryError,
+    # and we cannot test ensure after NoMemoryError.
+  rescue RangeError
+    # MingW can raise RangeError instead of NoMemoryError,
+    # so we cannot test this case.
+  end
+
   def test_equal
     bug5865 = '[ruby-core:41979]'
     assert_equal(RuntimeError.new("a"), RuntimeError.new("a"), bug5865)
diff --git a/vm.c b/vm.c
index 75cd371c97..4e458b344d 100644
--- a/vm.c
+++ b/vm.c
@@ -2168,7 +2168,7 @@ vm_exec(rb_execution_context_t *ec, bool mjit_enable_p)
     }
     else {
 	result = ec->errinfo;
-        rb_ec_raised_reset(ec, RAISED_STACKOVERFLOW);
+        rb_ec_raised_reset(ec, RAISED_STACKOVERFLOW | RAISED_NOMEMORY);
         while ((result = vm_exec_handle_exception(ec, state, result, &initial)) == Qundef) {
             /* caught a jump, exec the handler */
             result = vm_exec_core(ec, initial);