[ruby/irb] workaround for lack of tokens from `Ripper.lex`

* Fixes #38

https://github.com/ruby/irb/commit/905fb8e52e

1 files changed, 5 insertions, 2 deletions

diff --git a/lib/irb/ruby-lex.rb b/lib/irb/ruby-lex.rb
index 51efc17433..03022822bb 100644
--- a/lib/irb/ruby-lex.rb
+++ b/lib/irb/ruby-lex.rb
@@ -88,9 +88,12 @@ class RubyLex
 
   def ripper_lex_without_warning(code)
     verbose, $VERBOSE = $VERBOSE, nil
-    tokens = nil
+    tokens = []
     self.class.compile_with_errors_suppressed(code) do |inner_code, line_no|
-      tokens = Ripper.lex(inner_code, '-', line_no)
+      lexer = Ripper::Lexer.new(inner_code, '-', line_no)
+      until (ts = lexer.lex).empty?
+        tokens.concat(ts)
+      end
     end
     $VERBOSE = verbose
     tokens