ripper: fix a bug of Ripper::Lexer with syntax error and heredoc [Bug #17644]

2 files changed, 13 insertions, 1 deletions

diff --git a/ext/ripper/lib/ripper/lexer.rb b/ext/ripper/lib/ripper/lexer.rb
index 8222684b65..f6051c6341 100644
--- a/ext/ripper/lib/ripper/lexer.rb
+++ b/ext/ripper/lib/ripper/lexer.rb
@@ -136,7 +136,7 @@ class Ripper
       end
       @buf.flatten!
       unless (result = @buf).empty?
-        result.concat(@buf) until (@buf = []; super(); @buf.empty?)
+        result.concat(@buf) until (@buf = []; super(); @buf.flatten!; @buf.empty?)
       end
       result
     end
diff --git a/test/ripper/test_lexer.rb b/test/ripper/test_lexer.rb
index 79cb194e5b..d0389f7c96 100644
--- a/test/ripper/test_lexer.rb
+++ b/test/ripper/test_lexer.rb
@@ -216,4 +216,16 @@ class TestRipper::Lexer < Test::Unit::TestCase
       end
     end
   end
+
+  def test_lex_with_syntax_error_and_heredo
+    bug = '[Bug #17644]'
+    s = <<~EOF
+        foo
+      end
+      <<~EOS
+        bar
+      EOS
+    EOF
+    assert_equal([[5, 0], :on_heredoc_end, "EOS\n", state(:EXPR_BEG)], Ripper.lex(s).last, bug)
+  end
 end