Merge rdoc-6.0.0.beta1.

  This version fixed strange behavior of ruby code parser.
  We will list all of impromovement to Changelog when 6.0.0 releasing.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@59686 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

3 files changed, 16 insertions, 52 deletions

diff --git a/lib/rdoc/parser/c.rb b/lib/rdoc/parser/c.rb
index 087d56f7b9..0e72df2ecd 100644
--- a/lib/rdoc/parser/c.rb
+++ b/lib/rdoc/parser/c.rb
@@ -670,7 +670,6 @@ class RDoc::Parser::C < RDoc::Parser
       tk.set_text body
       meth_obj.add_token tk
       meth_obj.comment = comment
-      meth_obj.offset  = offset
       meth_obj.line    = file_content[0, offset].count("\n") + 1
 
       body
@@ -689,7 +688,6 @@ class RDoc::Parser::C < RDoc::Parser
       tk.set_text body
       meth_obj.add_token tk
       meth_obj.comment = comment
-      meth_obj.offset  = offset
       meth_obj.line    = file_content[0, offset].count("\n") + 1
 
       body
diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb
index 9562ddbd7a..e7900c0807 100644
--- a/lib/rdoc/parser/ruby.rb
+++ b/lib/rdoc/parser/ruby.rb
@@ -589,7 +589,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # +comment+.
 
   def parse_attr(context, single, tk, comment)
-    offset  = tk.seek
     line_no = tk.line_no
 
     args = parse_symbol_arg 1
@@ -606,7 +605,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
       end
 
       att = create_attr context, single, name, rw, comment
-      att.offset = offset
       att.line   = line_no
 
       read_documentation_modifiers att, RDoc::ATTR_MODIFIERS
@@ -620,7 +618,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # comment for each to +comment+.
 
   def parse_attr_accessor(context, single, tk, comment)
-    offset  = tk.seek
     line_no = tk.line_no
 
     args = parse_symbol_arg
@@ -642,7 +639,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     for name in args
       att = create_attr context, single, name, rw, comment
-      att.offset = offset
       att.line   = line_no
     end
   end
@@ -651,7 +647,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Parses an +alias+ in +context+ with +comment+
 
   def parse_alias(context, single, tk, comment)
-    offset  = tk.seek
     line_no = tk.line_no
 
     skip_tkspace
@@ -680,7 +675,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
     al = RDoc::Alias.new(get_tkread, old_name, new_name, comment,
                          single == SINGLE)
     record_location al
-    al.offset = offset
     al.line   = line_no
 
     read_documentation_modifiers al, RDoc::ATTR_MODIFIERS
@@ -733,7 +727,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Parses a class in +context+ with +comment+
 
   def parse_class container, single, tk, comment
-    offset  = tk.seek
     line_no = tk.line_no
 
     declaration_context = container
@@ -748,7 +741,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
         case name = get_class_specification
         when 'self', container.name
           parse_statements container, SINGLE
-          return # don't update offset or line
+          return # don't update line
         else
           parse_class_singleton container, name, comment
         end
@@ -757,9 +750,11 @@ class RDoc::Parser::Ruby < RDoc::Parser
         return
       end
 
-    cls.offset = offset
     cls.line   = line_no
 
+    # after end modifiers
+    read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS
+
     cls
   end
 
@@ -845,7 +840,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # true, no found constants will be added to RDoc.
 
   def parse_constant container, tk, comment, ignore_constants = false
-    offset  = tk.seek
     line_no = tk.line_no
 
     name = tk.name
@@ -885,7 +879,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     value.replace body
     record_location con
-    con.offset = offset
     con.line   = line_no
     read_documentation_modifiers con, RDoc::CONSTANT_MODIFIERS
 
@@ -950,7 +943,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
   def parse_comment container, tk, comment
     return parse_comment_tomdoc container, tk, comment if @markup == 'tomdoc'
     column  = tk.char_no
-    offset  = tk.seek
     line_no = tk.line_no
 
     text = comment.text
@@ -966,7 +958,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     if co then
       co.singleton = singleton
-      co.offset    = offset
       co.line      = line_no
     end
 
@@ -1031,19 +1022,18 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   def parse_comment_tomdoc container, tk, comment
     return unless signature = RDoc::TomDoc.signature(comment)
-    offset  = tk.seek
+    column  = tk.char_no
     line_no = tk.line_no
 
     name, = signature.split %r%[ \(]%, 2
 
     meth = RDoc::GhostMethod.new get_tkread, name
     record_location meth
-    meth.offset    = offset
     meth.line      = line_no
 
     meth.start_collecting_tokens
     indent = TkSPACE.new 0, 1, 1
-    indent.set_text " " * offset
+    indent.set_text " " * column
 
     position_comment = TkCOMMENT.new 0, line_no, 1
     position_comment.set_text "# File #{@top_level.relative_name}, line #{line_no}"
@@ -1183,7 +1173,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   def parse_meta_method(container, single, tk, comment)
     column  = tk.char_no
-    offset  = tk.seek
     line_no = tk.line_no
 
     start_collecting_tokens
@@ -1200,7 +1189,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     meth = RDoc::MetaMethod.new get_tkread, name
     record_location meth
-    meth.offset = offset
     meth.line   = line_no
     meth.singleton = singleton
 
@@ -1291,7 +1279,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
     added_container = false
     name = nil
     column  = tk.char_no
-    offset  = tk.seek
     line_no = tk.line_no
 
     start_collecting_tokens
@@ -1309,7 +1296,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
     meth.singleton = single == SINGLE ? true : singleton
 
     record_location meth
-    meth.offset = offset
     meth.line   = line_no
 
     meth.start_collecting_tokens
@@ -1328,6 +1314,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     meth.comment = comment
 
+    # after end modifiers
+    read_documentation_modifiers meth, RDoc::METHOD_MODIFIERS
+
     @stats.add_method meth
   end
 
@@ -1519,6 +1508,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
       end
       tk = get_tk
     end
+    @scanner.first_in_method_statement = true
 
     get_tkread_clean(/\s+/, ' ')
   end
@@ -1559,6 +1549,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
     mod.add_comment comment, @top_level
     parse_statements mod
 
+    # after end modifiers
+    read_documentation_modifiers mod, RDoc::CLASS_MODIFIERS
+
     @stats.add_module mod
   end
 
@@ -1732,7 +1725,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
       when TkEND then
         nest -= 1
         if nest == 0 then
-          read_documentation_modifiers container, RDoc::CLASS_MODIFIERS
           container.ongoing_visibility = save_visibility
 
           parse_comment container, tk, comment unless comment.empty?
@@ -2039,7 +2031,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
   def skip_optional_do_after_expression
     skip_tkspace false
     tk = get_tk
-    end_token = get_end_token tk
 
     b_nest = 0
     nest = 0
@@ -2047,23 +2038,18 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     loop do
       case tk
-      when TkSEMICOLON then
+      when TkSEMICOLON, TkNL then
         break if b_nest.zero?
       when TkLPAREN, TkfLPAREN then
         nest += 1
+      when TkRPAREN then
+        nest -= 1
       when TkBEGIN then
         b_nest += 1
       when TkEND then
         b_nest -= 1
       when TkDO
         break if nest.zero?
-      when end_token then
-        if end_token == TkRPAREN
-          nest -= 1
-          break if @scanner.lex_state == :EXPR_END and nest.zero?
-        else
-          break unless @scanner.continue
-        end
       when nil then
         break
       end
diff --git a/lib/rdoc/parser/ruby_tools.rb b/lib/rdoc/parser/ruby_tools.rb
index bbca065b5e..e35b420f30 100644
--- a/lib/rdoc/parser/ruby_tools.rb
+++ b/lib/rdoc/parser/ruby_tools.rb
@@ -33,26 +33,6 @@ module RDoc::Parser::RubyTools
 
     tk = nil if TkEND_OF_SCRIPT === tk
 
-    if TkSYMBEG === tk then
-      set_token_position tk.line_no, tk.char_no
-
-      case tk1 = get_tk
-      when TkId, TkOp, TkSTRING, TkDSTRING, TkSTAR, TkAMPER then
-        if tk1.respond_to?(:name) then
-          tk = Token(TkSYMBOL).set_text(":" + tk1.name)
-        else
-          tk = Token(TkSYMBOL).set_text(":" + tk1.text)
-        end
-
-        # remove the identifier we just read to replace it with a symbol
-        @token_listeners.each do |obj|
-          obj.pop_token
-        end if @token_listeners
-      else
-        tk = tk1
-      end
-    end
-
     # inform any listeners of our shiny new token
     @token_listeners.each do |obj|
       obj.add_token(tk)