[Bug #19291] Rewind to the previous line

When rewinding looking ahead after newline token, also reset the last
line string, the pointers to it, and the location, not only the line
number.

3 files changed, 20 insertions, 4 deletions

diff --git a/parse.y b/parse.y
index a6ed829b84..301266c36f 100644
--- a/parse.y
+++ b/parse.y
@@ -7028,6 +7028,14 @@ add_delayed_token(struct parser_params *p, const char *tok, const char *end, int
     }
 }
 
+static void
+set_lastline(struct parser_params *p, VALUE v)
+{
+    p->lex.pbeg = p->lex.pcur = RSTRING_PTR(v);
+    p->lex.pend = p->lex.pcur + RSTRING_LEN(v);
+    p->lex.lastline = v;
+}
+
 static int
 nextline(struct parser_params *p, int set_encoding)
 {
@@ -7065,10 +7073,8 @@ nextline(struct parser_params *p, int set_encoding)
 	p->heredoc_end = 0;
     }
     p->ruby_sourceline++;
-    p->lex.pbeg = p->lex.pcur = RSTRING_PTR(v);
-    p->lex.pend = p->lex.pcur + RSTRING_LEN(v);
+    set_lastline(p, v);
     token_flush(p);
-    p->lex.lastline = v;
     return 0;
 }
 
@@ -9850,6 +9856,7 @@ parser_yylex(struct parser_params *p)
 	/* fall through */
       case '\n':
 	p->token_seen = token_seen;
+	VALUE prevline = p->lex.lastline;
 	c = (IS_lex_state(EXPR_BEG|EXPR_CLASS|EXPR_FNAME|EXPR_DOT) &&
 	     !IS_lex_state(EXPR_LABELED));
 	if (c || IS_lex_state_all(EXPR_ARG|EXPR_LABELED)) {
@@ -9887,10 +9894,12 @@ parser_yylex(struct parser_params *p)
 	      default:
 		p->ruby_sourceline--;
 		p->lex.nextline = p->lex.lastline;
+		set_lastline(p, prevline);
 	      case -1:		/* EOF no decrement*/
 		lex_goto_eol(p);
 		if (c != -1) {
-		    p->lex.ptok = p->lex.pcur;
+		    token_flush(p);
+		    RUBY_SET_YYLLOC(*p->yylloc);
 		}
 		goto normal_newline;
 	    }
diff --git a/test/ripper/test_scanner_events.rb b/test/ripper/test_scanner_events.rb
index 5d4b6ff5de..da3dbfb66c 100644
--- a/test/ripper/test_scanner_events.rb
+++ b/test/ripper/test_scanner_events.rb
@@ -995,4 +995,10 @@ class TestRipper::ScannerEvents < Test::Unit::TestCase
     assert_equal ['U'], scan('tstring_content', '/\\xU/') {|*e| err = e}
     assert_equal [:on_parse_error, "invalid hex escape", "\\x"], err
   end
+
+  def test_error_token
+    src = "{a:,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n""hello}"
+    err = scan('parse_error', src) {|*e| break e}
+    assert_equal "", err[2]
+  end
 end if ripper_test
diff --git a/test/ruby/test_parse.rb b/test/ruby/test_parse.rb
index 4488ea620e..2f2c062ceb 100644
--- a/test/ruby/test_parse.rb
+++ b/test/ruby/test_parse.rb
@@ -14,6 +14,7 @@ class TestParse < Test::Unit::TestCase
 
   def test_error_line
     assert_syntax_error('------,,', /\n\z/, 'Message to pipe should end with a newline')
+    assert_syntax_error("{hello\n  world}", /hello/)
   end
 
   def test_else_without_rescue