* lib/rdoc*: Updated to RDoc 4.0 (pre-release)

* bin/rdoc:  ditto
* test/rdoc:  ditto
* NEWS:  Updated with RDoc 4.0 information


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@37889 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

7 files changed, 591 insertions, 312 deletions

diff --git a/lib/rdoc/parser/c.rb b/lib/rdoc/parser/c.rb
index 3da1820c50..31be27169c 100644
--- a/lib/rdoc/parser/c.rb
+++ b/lib/rdoc/parser/c.rb
@@ -1,6 +1,4 @@
-
-require 'rdoc/parser/ruby'
-require 'rdoc/known_classes'
+require 'tsort'
 
 ##
 # RDoc::Parser::C attempts to parse C extension files.  It looks for
@@ -58,6 +56,13 @@ require 'rdoc/known_classes'
 # [Document-const: +name+]
 #   Documentation for the named +rb_define_const+.
 #
+#   Constant values can be supplied on the first line of the comment like so:
+#
+#     /* 300: The highest possible score in bowling */
+#     rb_define_const(cFoo, "PERFECT", INT2FIX(300));
+#
+#   The value can contain internal colons so long as they are escaped with a \
+#
 # [Document-global: +name+]
 #   Documentation for the named +rb_define_global_const+
 #
@@ -122,27 +127,27 @@ class RDoc::Parser::C < RDoc::Parser
 
   attr_accessor :content
 
+  ##
+  # Dependencies from a missing enclosing class to the classes in
+  # missing_dependencies that depend upon it.
+
+  attr_reader :enclosure_dependencies
 
   ##
-  # Maps C variable names to names of ruby classes (andsingleton classes)
+  # Maps C variable names to names of ruby classes (and singleton classes)
 
   attr_reader :known_classes
 
   ##
-  # Maps C variable names to names of ruby singleton classes
+  # Classes found while parsing the C file that were not yet registered due to
+  # a missing enclosing class.  These are processed by do_missing
 
-  attr_reader :singleton_classes
+  attr_reader :missing_dependencies
 
   ##
-  # Resets cross-file state.  Call when parsing different projects that need
-  # separate documentation.
-
-  def self.reset
-    @@enclosure_classes = {}
-    @@known_bodies = {}
-  end
+  # Maps C variable names to names of ruby singleton classes
 
-  reset
+  attr_reader :singleton_classes
 
   ##
   # Prepare to parse a C file
@@ -155,6 +160,38 @@ class RDoc::Parser::C < RDoc::Parser
     @classes = {}
     @singleton_classes = {}
     @file_dir = File.dirname(@file_name)
+
+    # missing variable => [handle_class_module arguments]
+    @missing_dependencies = {}
+
+    # missing enclosure variable => [dependent handle_class_module arguments]
+    @enclosure_dependencies = Hash.new { |h, k| h[k] = [] }
+    @enclosure_dependencies.instance_variable_set :@missing_dependencies,
+                                                  @missing_dependencies
+
+    @enclosure_dependencies.extend TSort
+
+    def @enclosure_dependencies.tsort_each_node &block
+      each_key(&block)
+    rescue TSort::Cyclic => e
+      cycle_vars = e.message.scan(/"(.*?)"/).flatten
+
+      cycle = cycle_vars.sort.map do |var_name|
+        delete var_name
+
+        var_name, type, mod_name, = @missing_dependencies[var_name]
+
+        "#{type} #{mod_name} (#{var_name})"
+      end.join ', '
+
+      warn "Unable to create #{cycle} due to a cyclic class or module creation"
+
+      retry
+    end
+
+    def @enclosure_dependencies.tsort_each_child node, &block
+      fetch(node, []).each(&block)
+    end
   end
 
   ##
@@ -169,7 +206,7 @@ class RDoc::Parser::C < RDoc::Parser
       class_name = @known_classes[var_name]
 
       unless class_name then
-        warn "Enclosing class/module %p for alias %s %s not known" % [
+        @options.warn "Enclosing class or module %p for alias %s %s is not known" % [
           var_name, new_name, old_name]
         next
       end
@@ -180,7 +217,9 @@ class RDoc::Parser::C < RDoc::Parser
       al.singleton = @singleton_classes.key? var_name
 
       comment = find_alias_comment var_name, new_name, old_name
-      comment = strip_stars comment
+
+      comment.normalize
+
       al.comment = comment
 
       al.record_location @top_level
@@ -214,62 +253,26 @@ class RDoc::Parser::C < RDoc::Parser
   end
 
   ##
-  # Scans #content for rb_define_module, rb_define_class, boot_defclass,
-  # rb_define_module_under, rb_define_class_under and rb_singleton_class
-
-  def do_classes
-    @content.scan(/(\w+)\s* = \s*rb_define_module\s*\(\s*"(\w+)"\s*\)/mx) do
-      |var_name, class_name|
-      handle_class_module(var_name, "module", class_name, nil, nil)
-    end
-
-    # The '.' lets us handle SWIG-generated files
-    @content.scan(/([\w\.]+)\s* = \s*rb_define_class\s*
-              \(
-                 \s*"(\w+)",
-                 \s*(\w+)\s*
-              \)/mx) do |var_name, class_name, parent|
-      handle_class_module(var_name, "class", class_name, parent, nil)
-    end
+  # Scans #content for boot_defclass
 
+  def do_boot_defclass
     @content.scan(/(\w+)\s*=\s*boot_defclass\s*\(\s*"(\w+?)",\s*(\w+?)\s*\)/) do
       |var_name, class_name, parent|
       parent = nil if parent == "0"
-      handle_class_module(var_name, "class", class_name, parent, nil)
-    end
-
-    @content.scan(/(\w+)\s* = \s*rb_define_module_under\s*
-              \(
-                 \s*(\w+),
-                 \s*"(\w+)"
-              \s*\)/mx) do |var_name, in_module, class_name|
-      handle_class_module(var_name, "module", class_name, nil, in_module)
+      handle_class_module(var_name, :class, class_name, parent, nil)
     end
+  end
 
-    @content.scan(/([\w\.]+)\s* =                  # var_name
-                   \s*rb_define_class_under\s*
-                   \(
-                     \s* (\w+),                    # under
-                     \s* "(\w+)",                  # class_name
-                     \s*
-                     (?:
-                       ([\w\*\s\(\)\.\->]+) |      # parent_name
-                       rb_path2class\("([\w:]+)"\) # path
-                     )
-                     \s*
-                   \)
-                  /mx) do |var_name, under, class_name, parent_name, path|
-      parent = path || parent_name
-
-      handle_class_module var_name, 'class', class_name, parent, under
-    end
+  ##
+  # Scans #content for rb_define_class, boot_defclass, rb_define_class_under
+  # and rb_singleton_class
 
-    @content.scan(/([\w\.]+)\s* = \s*rb_singleton_class\s*
-                  \(
-                    \s*(\w+)
-                  \s*\)/mx) do |sclass_var, class_var|
-      handle_singleton sclass_var, class_var
-    end
+  def do_classes
+    do_boot_defclass
+    do_define_class
+    do_define_class_under
+    do_singleton_class
+    do_struct_define_without_accessor
   end
 
   ##
@@ -300,8 +303,82 @@ class RDoc::Parser::C < RDoc::Parser
                   \)
                   \s*;%xm) do |consts|
       const = consts.first
+
       handle_constants 'const', 'mCurses', const, "UINT2NUM(#{const})"
     end
+
+    @content.scan(%r%
+                  \Wrb_file_const
+                  \s*\(
+                    \s*
+                    "([^"]+)",
+                    \s*
+                    (.*?)
+                    \s*
+                  \)
+                  \s*;%xm) do |name, value|
+      handle_constants 'const', 'rb_mFConst', name, value
+    end
+  end
+
+  ##
+  # Scans #content for rb_define_class
+
+  def do_define_class
+    # The '.' lets us handle SWIG-generated files
+    @content.scan(/([\w\.]+)\s* = \s*rb_define_class\s*
+              \(
+                 \s*"(\w+)",
+                 \s*(\w+)\s*
+              \)/mx) do |var_name, class_name, parent|
+      handle_class_module(var_name, :class, class_name, parent, nil)
+    end
+  end
+
+  ##
+  # Scans #content for rb_define_class_under
+
+  def do_define_class_under
+    @content.scan(/([\w\.]+)\s* =                  # var_name
+                   \s*rb_define_class_under\s*
+                   \(
+                     \s* (\w+),                    # under
+                     \s* "(\w+)",                  # class_name
+                     \s*
+                     (?:
+                       ([\w\*\s\(\)\.\->]+) |      # parent_name
+                       rb_path2class\("([\w:]+)"\) # path
+                     )
+                     \s*
+                   \)
+                  /mx) do |var_name, under, class_name, parent_name, path|
+      parent = path || parent_name
+
+      handle_class_module var_name, :class, class_name, parent, under
+    end
+  end
+
+  ##
+  # Scans #content for rb_define_module
+
+  def do_define_module
+    @content.scan(/(\w+)\s* = \s*rb_define_module\s*\(\s*"(\w+)"\s*\)/mx) do
+      |var_name, class_name|
+      handle_class_module(var_name, :module, class_name, nil, nil)
+    end
+  end
+
+  ##
+  # Scans #content for rb_define_module_under
+
+  def do_define_module_under
+    @content.scan(/(\w+)\s* = \s*rb_define_module_under\s*
+              \(
+                 \s*(\w+),
+                 \s*"(\w+)"
+              \s*\)/mx) do |var_name, in_module, class_name|
+      handle_class_module(var_name, :module, class_name, nil, in_module)
+    end
   end
 
   ##
@@ -311,7 +388,9 @@ class RDoc::Parser::C < RDoc::Parser
     @content.scan(/rb_include_module\s*\(\s*(\w+?),\s*(\w+?)\s*\)/) do |c,m|
       if cls = @classes[c]
         m = @known_classes[m] || m
-        incl = cls.add_include RDoc::Include.new(m, "")
+
+        comment = RDoc::Comment.new '', @top_level
+        incl = cls.add_include RDoc::Include.new(m, comment)
         incl.record_location @top_level
       end
     end
@@ -367,6 +446,54 @@ class RDoc::Parser::C < RDoc::Parser
     end
   end
 
+  def do_missing
+    return if @missing_dependencies.empty?
+
+    @enclosure_dependencies.tsort.each do |in_module|
+      arguments = @missing_dependencies.delete in_module
+
+      next unless arguments # dependency on existing class
+
+      handle_class_module(*arguments)
+    end
+  end
+
+  ##
+  # Scans #content for rb_define_module and rb_define_module_under
+
+  def do_modules
+    do_define_module
+    do_define_module_under
+  end
+
+  ##
+  # Scans #content for rb_singleton_class
+
+  def do_singleton_class
+    @content.scan(/([\w\.]+)\s* = \s*rb_singleton_class\s*
+                  \(
+                    \s*(\w+)
+                  \s*\)/mx) do |sclass_var, class_var|
+      handle_singleton sclass_var, class_var
+    end
+  end
+
+  ##
+  # Scans #content for struct_define_without_accessor
+
+  def do_struct_define_without_accessor
+    @content.scan(/([\w\.]+)\s* = \s*rb_struct_define_without_accessor\s*
+              \(
+                 \s*"(\w+)",  # Class name
+                 \s*(\w+),    # Parent class
+                 \s*\w+,      # Allocation function
+                 (\s*"\w+",)* # Attributes
+                 \s*NULL
+              \)/mx) do |var_name, class_name, parent|
+      handle_class_module(var_name, :class, class_name, parent, nil)
+    end
+  end
+
   ##
   # Finds the comment for an alias on +class_name+ from +new_name+ to
   # +old_name+
@@ -377,7 +504,7 @@ class RDoc::Parser::C < RDoc::Parser
                                    \s*"#{Regexp.escape new_name}"\s*,
                                    \s*"#{Regexp.escape old_name}"\s*\);%xm
 
-    $1 || ''
+    RDoc::Comment.new($1 || '', @top_level)
   end
 
   ##
@@ -398,22 +525,24 @@ class RDoc::Parser::C < RDoc::Parser
            /.*?/m
          end
 
-    if @content =~ %r%((?>/\*.*?\*/\s+))
-                      rb_define_attr\((?:\s*#{var_name},)?\s*
-                                      "#{attr_name}"\s*,
-                                      #{rw}\)\s*;%xm then
-      $1
-    elsif @content =~ %r%((?>/\*.*?\*/\s+))
-                         rb_attr\(\s*#{var_name}\s*,
-                                  \s*#{attr_name}\s*,
-                                  #{rw},.*?\)\s*;%xm then
-      $1
-    elsif @content =~ %r%Document-attr:\s#{attr_name}\s*?\n
-                         ((?>.*?\*/))%xm then
-      $1
-    else
-      ''
-    end
+    comment = if @content =~ %r%((?>/\*.*?\*/\s+))
+                                rb_define_attr\((?:\s*#{var_name},)?\s*
+                                                "#{attr_name}"\s*,
+                                                #{rw}\)\s*;%xm then
+                $1
+              elsif @content =~ %r%((?>/\*.*?\*/\s+))
+                                   rb_attr\(\s*#{var_name}\s*,
+                                            \s*#{attr_name}\s*,
+                                            #{rw},.*?\)\s*;%xm then
+                $1
+              elsif @content =~ %r%Document-attr:\s#{attr_name}\s*?\n
+                                   ((?>.*?\*/))%xm then
+                $1
+              else
+                ''
+              end
+
+    RDoc::Comment.new comment, @top_level
   end
 
   ##
@@ -425,11 +554,11 @@ class RDoc::Parser::C < RDoc::Parser
             ((?:(?:static|SWIGINTERN)\s+)?
              (?:intern\s+)?VALUE\s+#{meth_name}
              \s*(\([^)]*\))([^;]|$))%xm then
-      comment = $1
+      comment = RDoc::Comment.new $1, @top_level
       body = $2
-      offset = $~.offset(2).first
+      offset, = $~.offset(2)
 
-      remove_private_comments comment if comment
+      comment.remove_private if comment
 
       # try to find the whole body
       body = $& if /#{Regexp.escape body}[^(]*?\{.*?^\}/m =~ file_content
@@ -443,6 +572,7 @@ class RDoc::Parser::C < RDoc::Parser
       override_comment = find_override_comment class_name, meth_obj
       comment = override_comment if override_comment
 
+      comment.normalize
       find_modifiers comment, meth_obj if comment
 
       #meth_obj.params = params
@@ -450,24 +580,26 @@ class RDoc::Parser::C < RDoc::Parser
       tk = RDoc::RubyToken::Token.new nil, 1, 1
       tk.set_text body
       meth_obj.add_token tk
-      meth_obj.comment = strip_stars comment
+      meth_obj.comment = comment
       meth_obj.offset  = offset
       meth_obj.line    = file_content[0, offset].count("\n") + 1
 
       body
     when %r%((?>/\*.*?\*/\s*))^\s*(\#\s*define\s+#{meth_name}\s+(\w+))%m then
-      comment = $1
+      comment = RDoc::Comment.new $1, @top_level
       body = $2
       offset = $~.offset(2).first
 
       find_body class_name, $3, meth_obj, file_content, true
+
+      comment.normalize
       find_modifiers comment, meth_obj
 
       meth_obj.start_collecting_tokens
       tk = RDoc::RubyToken::Token.new nil, 1, 1
       tk.set_text body
       meth_obj.add_token tk
-      meth_obj.comment = strip_stars(comment) + meth_obj.comment.to_s
+      meth_obj.comment = comment
       meth_obj.offset  = offset
       meth_obj.line    = file_content[0, offset].count("\n") + 1
 
@@ -480,18 +612,19 @@ class RDoc::Parser::C < RDoc::Parser
 
       return body if body
 
-      warn "No definition for #{meth_name}" if @options.verbosity > 1
+      @options.warn "No definition for #{meth_name}"
       false
     else # No body, but might still have an override comment
       comment = find_override_comment class_name, meth_obj
 
       if comment then
+        comment.normalize
         find_modifiers comment, meth_obj
-        meth_obj.comment = strip_stars comment
+        meth_obj.comment = comment
 
         ''
       else
-        warn "No definition for #{meth_name}" if @options.verbosity > 1
+        @options.warn "No definition for #{meth_name}"
         false
       end
     end
@@ -540,7 +673,7 @@ class RDoc::Parser::C < RDoc::Parser
   #    */
   #   VALUE cFoo = rb_define_class("Foo", rb_cObject);
 
-  def find_class_comment(class_name, class_mod)
+  def find_class_comment class_name, class_mod
     comment = nil
 
     if @content =~ %r%
@@ -551,17 +684,21 @@ class RDoc::Parser::C < RDoc::Parser
       comment = $1.sub(%r%Document-(?:class|module):\s+#{class_name}%, '')
     elsif @content =~ %r%Document-(?:class|module):\s+#{class_name}\s*?
                          (?:<\s+[:,\w]+)?\n((?>.*?\*/))%xm then
-      comment = $1
-    elsif @content =~ %r%((?>/\*.*?\*/\s+))
+      comment = "/*\n#{$1}"
+    elsif @content =~ %r%.*((?>/\*.*?\*/\s+))
                          ([\w\.\s]+\s* = \s+)?rb_define_(class|module).*?"(#{class_name})"%xm then
       comment = $1
+    elsif @content =~ %r%.*((?>/\*.*?\*/\s+))
+                         ([\w\.\s]+\s* = \s+)?rb_define_(class|module)_under.*?"(#{class_name.split('::').last})"%xm then
+      comment = $1
+    else
+      comment = ''
     end
 
-    return unless comment
-
-    comment = strip_stars comment
+    comment = RDoc::Comment.new comment, @top_level
+    comment.normalize
 
-    comment = look_for_directives_in class_mod, comment
+    look_for_directives_in class_mod, comment
 
     class_mod.add_comment comment, @top_level
   end
@@ -571,79 +708,33 @@ class RDoc::Parser::C < RDoc::Parser
   # comment or in the matching Document- section.
 
   def find_const_comment(type, const_name, class_name = nil)
-    if @content =~ %r%((?>^\s*/\*.*?\*/\s+))
-                   rb_define_#{type}\((?:\s*(\w+),)?\s*
-                                      "#{const_name}"\s*,
-                                      .*?\)\s*;%xmi then
-      $1
-    elsif class_name and
-          @content =~ %r%Document-(?:const|global|variable):\s
-                         #{class_name}::#{const_name}
-                         \s*?\n((?>.*?\*/))%xm then
-      $1
-    elsif @content =~ %r%Document-(?:const|global|variable):\s#{const_name}
-                         \s*?\n((?>.*?\*/))%xm then
-      $1
-    else
-      ''
-    end
+    comment = if @content =~ %r%((?>^\s*/\*.*?\*/\s+))
+                             rb_define_#{type}\((?:\s*(\w+),)?\s*
+                                                "#{const_name}"\s*,
+                                                .*?\)\s*;%xmi then
+                $1
+              elsif class_name and
+                    @content =~ %r%Document-(?:const|global|variable):\s
+                                   #{class_name}::#{const_name}
+                                   \s*?\n((?>.*?\*/))%xm then
+                "/*\n#{$1}"
+              elsif @content =~ %r%Document-(?:const|global|variable):
+                                   \s#{const_name}
+                                   \s*?\n((?>.*?\*/))%xm then
+                "/*\n#{$1}"
+              else
+                ''
+              end
+
+    RDoc::Comment.new comment, @top_level
   end
 
   ##
   # Handles modifiers in +comment+ and updates +meth_obj+ as appropriate.
-  #
-  # If <tt>:nodoc:</tt> is found, documentation on +meth_obj+ is suppressed.
-  #
-  # If <tt>:yields:</tt> is followed by an argument list it is used for the
-  # #block_params of +meth_obj+.
-  #
-  # If the comment block contains a <tt>call-seq:</tt> section like:
-  #
-  #   call-seq:
-  #      ARGF.readlines(sep=$/)     -> array
-  #      ARGF.readlines(limit)      -> array
-  #      ARGF.readlines(sep, limit) -> array
-  #
-  #      ARGF.to_a(sep=$/)     -> array
-  #      ARGF.to_a(limit)      -> array
-  #      ARGF.to_a(sep, limit) -> array
-  #
-  # it is used for the parameters of +meth_obj+.
 
   def find_modifiers comment, meth_obj
-    # we must handle situations like the above followed by an unindented first
-    # comment.  The difficulty is to make sure not to match lines starting
-    # with ARGF at the same indent, but that are after the first description
-    # paragraph.
-
-    if comment =~ /call-seq:(.*?(?:\S|\*\/?).*?)^\s*(?:\*\/?)?\s*$/m then
-      all_start, all_stop = $~.offset(0)
-      seq_start, seq_stop = $~.offset(1)
-
-      # we get the following lines that start with the leading word at the
-      # same indent, even if they have blank lines before
-      if $1 =~ /(^\s*\*?\s*\n)+^(\s*\*?\s*\w+)/m then
-        leading = $2 # ' *    ARGF' in the example above
-        re = %r%
-          \A(
-             (^\s*\*?\s*\n)+
-             (^#{Regexp.escape leading}.*?\n)+
-            )+
-          ^\s*\*?\s*$
-        %xm
-        if comment[seq_stop..-1] =~ re then
-          all_stop = seq_stop + $~.offset(0).last
-          seq_stop = seq_stop + $~.offset(1).last
-        end
-      end
-
-      seq = comment[seq_start..seq_stop]
-      seq.gsub!(/^(\s*\*?\s*?)(\S|\n)/m, '\2')
-      comment.slice! all_start...all_stop
-      meth_obj.call_seq = seq
-    elsif comment.sub!(/\A\/\*\s*call-seq:(.*?)\*\/\Z/, '') then
-      meth_obj.call_seq = $1.strip
-    end
+    comment.normalize
+    comment.extract_call_seq meth_obj
 
     look_for_directives_in meth_obj, comment
   end
@@ -655,11 +746,18 @@ class RDoc::Parser::C < RDoc::Parser
     name = Regexp.escape meth_obj.name
     prefix = Regexp.escape meth_obj.name_prefix
 
-    if @content =~ %r%Document-method:\s+#{class_name}#{prefix}#{name}\s*?\n((?>.*?\*/))%m then
-      $1
-    elsif @content =~ %r%Document-method:\s#{name}\s*?\n((?>.*?\*/))%m then
-      $1
-    end
+    comment = if @content =~ %r%Document-method:
+                                \s+#{class_name}#{prefix}#{name}
+                                \s*?\n((?>.*?\*/))%xm then
+                "/*#{$1}"
+              elsif @content =~ %r%Document-method:
+                                   \s#{name}\s*?\n((?>.*?\*/))%xm then
+                "/*#{$1}"
+              end
+
+    return unless comment
+
+    RDoc::Comment.new comment, @top_level
   end
 
   ##
@@ -680,7 +778,7 @@ class RDoc::Parser::C < RDoc::Parser
     return unless class_obj
 
     comment = find_attr_comment var_name, attr_name
-    comment = strip_stars comment
+    comment.normalize
 
     name = attr_name.gsub(/rb_intern\("([^"]+)"\)/, '\1')
 
@@ -699,23 +797,26 @@ class RDoc::Parser::C < RDoc::Parser
     parent_name = @known_classes[parent] || parent
 
     if in_module then
-      enclosure = @classes[in_module] || @@enclosure_classes[in_module]
+      enclosure = @classes[in_module] || @store.c_enclosure_classes[in_module]
 
       if enclosure.nil? and enclosure = @known_classes[in_module] then
-        enc_type = /^rb_m/ =~ in_module ? "module" : "class"
+        enc_type = /^rb_m/ =~ in_module ? :module : :class
         handle_class_module in_module, enc_type, enclosure, nil, nil
         enclosure = @classes[in_module]
       end
 
       unless enclosure then
-        warn "Enclosing class/module '#{in_module}' for #{type} #{class_name} not known"
+        @enclosure_dependencies[in_module] << var_name
+        @missing_dependencies[var_name] =
+          [var_name, type, class_name, parent, in_module]
+
         return
       end
     else
       enclosure = @top_level
     end
 
-    if type == "class" then
+    if type == :class then
       full_name = if RDoc::ClassModule === enclosure then
                     enclosure.full_name + "::#{class_name}"
                   else
@@ -743,7 +844,7 @@ class RDoc::Parser::C < RDoc::Parser
     end
 
     @classes[var_name] = cm
-    @@enclosure_classes[var_name] = cm
+    @store.c_enclosure_classes[var_name] = cm
     @known_classes[var_name] = cm.full_name
   end
 
@@ -765,25 +866,20 @@ class RDoc::Parser::C < RDoc::Parser
     class_obj = find_class var_name, class_name
 
     unless class_obj then
-      warn "Enclosing class/module #{const_name.inspect} not known"
+      @options.warn 'Enclosing class or module %p is not known' % [const_name]
       return
     end
 
     comment = find_const_comment type, const_name, class_name
-    comment = strip_stars comment
-    comment = normalize_comment comment
+    comment.normalize
 
     # In the case of rb_define_const, the definition and comment are in
     # "/* definition: comment */" form.  The literal ':' and '\' characters
     # can be escaped with a backslash.
     if type.downcase == 'const' then
-      elements = comment.split ':'
-
-      if elements.nil? or elements.empty? then
-        con = RDoc::Constant.new const_name, definition, comment
-      else
-        new_definition = elements[0..-2].join(':')
+      no_match, new_definition, new_comment = comment.text.split(/(\A.*):/)
 
+      if no_match and no_match.empty? then
         if new_definition.empty? then # Default to literal C definition
           new_definition = definition
         else
@@ -793,13 +889,13 @@ class RDoc::Parser::C < RDoc::Parser
 
         new_definition.sub!(/\A(\s+)/, '')
 
-        new_comment = if $1.nil? then
-                        elements.last.lstrip
-                      else
-                        "#{$1}#{elements.last.lstrip}"
-                      end
+        new_comment = "#{$1}#{new_comment.lstrip}"
+
+        new_comment = RDoc::Comment.new new_comment, @top_level
 
         con = RDoc::Constant.new const_name, new_definition, new_comment
+      else
+        con = RDoc::Constant.new const_name, definition, comment
       end
     else
       con = RDoc::Constant.new const_name, definition, comment
@@ -849,9 +945,9 @@ class RDoc::Parser::C < RDoc::Parser
         file_name = File.join @file_dir, source_file
 
         if File.exist? file_name then
-          file_content = (@@known_bodies[file_name] ||= File.read(file_name))
+          file_content = File.read file_name
         else
-          warn "unknown source #{source_file} for #{meth_name} in #{@file_name}"
+          @options.warn "unknown source #{source_file} for #{meth_name} in #{@file_name}"
         end
       else
         file_content = @content
@@ -1021,20 +1117,16 @@ class RDoc::Parser::C < RDoc::Parser
   end
 
   ##
-  # Removes private comments from +comment+
-
-  def remove_private_comments(comment)
-    comment.gsub!(/\/?\*--\n(.*?)\/?\*\+\+/m, '')
-    comment.sub!(/\/?\*--\n.*/m, '')
-  end
-
-  ##
   # Extracts the classes, modules, methods, attributes, constants and aliases
   # from a C file and returns an RDoc::TopLevel for this file
 
   def scan
     remove_commented_out_lines
+
+    do_modules
     do_classes
+    do_missing
+
     do_constants
     do_methods
     do_includes
diff --git a/lib/rdoc/parser/markdown.rb b/lib/rdoc/parser/markdown.rb
new file mode 100644
index 0000000000..6fd88cf614
--- /dev/null
+++ b/lib/rdoc/parser/markdown.rb
@@ -0,0 +1,23 @@
+##
+# Parse a Markdown format file.  The parsed RDoc::Markup::Document is attached
+# as a file comment.
+
+class RDoc::Parser::Markdown < RDoc::Parser
+
+  include RDoc::Parser::Text
+
+  parse_files_matching(/\.(md|markdown)(?:\.[^.]+)?$/)
+
+  ##
+  # Creates an Markdown-format TopLevel for the given file.
+
+  def scan
+    comment = RDoc::Comment.new @content, @top_level
+    comment.format = 'markdown'
+
+    @top_level.comment = comment
+  end
+
+end
+
+
diff --git a/lib/rdoc/parser/rd.rb b/lib/rdoc/parser/rd.rb
new file mode 100644
index 0000000000..09069ae297
--- /dev/null
+++ b/lib/rdoc/parser/rd.rb
@@ -0,0 +1,22 @@
+##
+# Parse a RD format file.  The parsed RDoc::Markup::Document is attached as a
+# file comment.
+
+class RDoc::Parser::RD < RDoc::Parser
+
+  include RDoc::Parser::Text
+
+  parse_files_matching(/\.rd(?:\.[^.]+)?$/)
+
+  ##
+  # Creates an rd-format TopLevel for the given file.
+
+  def scan
+    comment = RDoc::Comment.new @content, @top_level
+    comment.format = 'rd'
+
+    @top_level.comment = comment
+  end
+
+end
+
diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb
index c9a12a8fe8..0b207fae3b 100644
--- a/lib/rdoc/parser/ruby.rb
+++ b/lib/rdoc/parser/ruby.rb
@@ -7,15 +7,6 @@
 #       by Keiju ISHITSUKA (Nippon Rational Inc.)
 #
 
-require 'rdoc/ruby_token'
-require 'rdoc/ruby_lex'
-
-require 'rdoc/code_objects'
-require 'rdoc/token_stream'
-require 'rdoc/markup/pre_process'
-require 'rdoc/parser'
-require 'rdoc/parser/ruby_tools'
-
 $TOKEN_DEBUG ||= nil
 
 ##
@@ -103,7 +94,7 @@ $TOKEN_DEBUG ||= nil
 # You can force the name of a method using the :method: directive:
 #
 #   ##
-#   # :method: woo_hoo!
+#   # :method: some_method!
 #
 # By default, meta-methods are instance methods.  To indicate that a method is
 # a singleton method instead use the :singleton-method: directive:
@@ -114,7 +105,7 @@ $TOKEN_DEBUG ||= nil
 # You can also use the :singleton-method: directive with a name:
 #
 #   ##
-#   # :singleton-method: woo_hoo!
+#   # :singleton-method: some_method!
 #
 # Additionally you can mark a method as an attribute by
 # using :attr:, :attr_reader:, :attr_writer: or :attr_accessor:.  Just like
@@ -173,6 +164,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
     @scanner = RDoc::RubyLex.new content, @options
     @scanner.exception_on_syntax_error = false
     @prev_seek = nil
+    @markup = @options.markup
 
     @encoding = nil
     @encoding = @options.encoding if Object.const_defined? :Encoding
@@ -213,7 +205,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     unget_tk tk
 
-    comment
+    new_comment comment
   end
 
   ##
@@ -226,22 +218,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
   end
 
   ##
-  # Look for a 'call-seq' in the comment, and override the normal parameter
-  # stuff
-  #--
-  # TODO handle undent
-
-  def extract_call_seq(comment, meth)
-    if comment.sub!(/:?call-seq:(.*?)(^\s*#?\s*$|\z)/m, '') then
-      seq = $1
-      seq.gsub!(/^\s*\#\s*/, '')
-      meth.call_seq = seq
-    end
-
-    meth
-  end
-
-  ##
   # Looks for a true or false token.  Returns false if TkFALSE or TkNIL are
   # found.
 
@@ -264,7 +240,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # with :: separated named) and return the ultimate name, the associated
   # container, and the given name (with the ::).
 
-  def get_class_or_module(container)
+  def get_class_or_module container
     skip_tkspace
     name_t = get_tk
     given_name = ''
@@ -283,14 +259,18 @@ class RDoc::Parser::Ruby < RDoc::Parser
     while TkCOLON2 === peek_tk do
       prev_container = container
       container = container.find_module_named name_t.name
-      unless container then
-        container = prev_container.add_module RDoc::NormalModule, name_t.name
-      end
+      container ||= prev_container.add_module RDoc::NormalModule, name_t.name
+
+      container.ignore unless prev_container.document_children
+
       get_tk
+      skip_tkspace false
       name_t = get_tk
       given_name << '::' << name_t.name
     end
+
     skip_tkspace false
+
     return [container, name_t, given_name]
   end
 
@@ -299,9 +279,10 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   def get_class_specification
     tk = get_tk
-    return "self" if TkSELF === tk
+    return 'self' if TkSELF === tk
+    return ''     if TkGVAR === tk
 
-    res = ""
+    res = ''
     while TkCOLON2 === tk or TkCOLON3 === tk or TkCONSTANT === tk do
       res += tk.name
       tk = get_tk
@@ -412,8 +393,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
            'attr', 'attr_accessor', 'attr_reader', 'attr_writer' then
         false # handled elsewhere
       when 'section' then
-        context.set_current_section param, comment
-        comment.replace ''
+        context.set_current_section param, comment.dup
+        comment.text = ''
         break
       end
     end
@@ -433,6 +414,15 @@ class RDoc::Parser::Ruby < RDoc::Parser
   end
 
   ##
+  # Creates a comment with the correct format
+
+  def new_comment comment
+    c = RDoc::Comment.new comment, @top_level
+    c.format = @markup
+    c
+  end
+
+  ##
   # Creates an RDoc::Attr for the name following +tk+, setting the comment to
   # +comment+.
 
@@ -590,7 +580,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
   ##
   # Parses a class in +context+ with +comment+
 
-  def parse_class(container, single, tk, comment)
+  def parse_class container, single, tk, comment
     offset  = tk.seek
     line_no = tk.line_no
 
@@ -602,6 +592,11 @@ class RDoc::Parser::Ruby < RDoc::Parser
       name = name_t.name
       superclass = '::Object'
 
+      if given_name =~ /^::/ then
+        declaration_context = @top_level
+        given_name = $'
+      end
+
       if TkLT === peek_tk then
         get_tk
         skip_tkspace
@@ -626,17 +621,25 @@ class RDoc::Parser::Ruby < RDoc::Parser
       parse_statements cls
     when TkLSHFT
       case name = get_class_specification
-      when "self", container.name
+      when 'self', container.name
         parse_statements container, SINGLE
       else
-        other = RDoc::TopLevel.find_class_named name
+        other = @store.find_class_named name
 
         unless other then
+          if name =~ /^::/ then
+            name = $'
+            container = @top_level
+          end
+
           other = container.add_module RDoc::NormalModule, name
           other.record_location @top_level
           other.offset  = offset
           other.line    = line_no
 
+          # class << $gvar
+          other.ignore if name.empty?
+
           other.add_comment comment, @top_level
         end
 
@@ -678,6 +681,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
       return false
     end
 
+    value = ''
+    con = RDoc::Constant.new name, value, comment
     nest = 0
     get_tkread
 
@@ -705,13 +710,16 @@ class RDoc::Parser::Ruby < RDoc::Parser
            (@scanner.lex_state == EXPR_END || [email protected]) then
           unget_tk tk
           break
+        else
+          unget_tk tk
+          read_documentation_modifiers con, RDoc::CONSTANT_MODIFIERS
         end
       when TkCONSTANT then
         rhs_name << tk.name
 
         if nest <= 0 and TkNL === peek_tk then
           mod = if rhs_name =~ /^::/ then
-                  RDoc::TopLevel.find_class_or_module rhs_name
+                  @store.find_class_or_module rhs_name
                 else
                   container.find_module_named rhs_name
                 end
@@ -736,14 +744,15 @@ class RDoc::Parser::Ruby < RDoc::Parser
     res = get_tkread.gsub(/^[ \t]+/, '').strip
     res = "" if res == ";"
 
-    con = RDoc::Constant.new name, res, comment
+    value.replace res
     con.record_location @top_level
     con.offset = offset
     con.line   = line_no
     read_documentation_modifiers con, RDoc::CONSTANT_MODIFIERS
 
     @stats.add_constant con
-    container.add_constant con
+    con = container.add_constant con
+
     true
   end
 
@@ -751,15 +760,18 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Generates an RDoc::Method or RDoc::Attr from +comment+ by looking for
   # :method: or :attr: directives in +comment+.
 
-  def parse_comment(container, tk, comment)
+  def parse_comment container, tk, comment
+    return parse_comment_tomdoc container, tk, comment if @markup == 'tomdoc'
     column  = tk.char_no
     offset  = tk.seek
     line_no = tk.line_no
 
-    singleton = !!comment.sub!(/(^# +:?)(singleton-)(method:)/, '\1\3')
+    text = comment.text
+
+    singleton = !!text.sub!(/(^# +:?)(singleton-)(method:)/, '\1\3')
 
     # REFACTOR
-    if comment.sub!(/^# +:?method: *(\S*).*?\n/i, '') then
+    if text.sub!(/^# +:?method: *(\S*).*?\n/i, '') then
       name = $1 unless $1.empty?
 
       meth = RDoc::GhostMethod.new get_tkread, name
@@ -769,16 +781,17 @@ class RDoc::Parser::Ruby < RDoc::Parser
       meth.line      = line_no
 
       meth.start_collecting_tokens
-      indent = TkSPACE.new nil, 1, 1
+      indent = TkSPACE.new 0, 1, 1
       indent.set_text " " * column
 
-      position_comment = TkCOMMENT.new nil, line_no, 1
+      position_comment = TkCOMMENT.new 0, line_no, 1
       position_comment.set_text "# File #{@top_level.absolute_name}, line #{line_no}"
       meth.add_tokens [position_comment, NEWLINE_TOKEN, indent]
 
       meth.params = ''
 
-      extract_call_seq comment, meth
+      comment.normalize
+      comment.extract_call_seq meth
 
       return unless meth.name
 
@@ -787,7 +800,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
       meth.comment = comment
 
       @stats.add_method meth
-    elsif comment.sub!(/# +:?(attr(_reader|_writer|_accessor)?): *(\S*).*?\n/i, '') then
+    elsif text.sub!(/# +:?(attr(_reader|_writer|_accessor)?): *(\S*).*?\n/i, '') then
       rw = case $1
            when 'attr_reader' then 'R'
            when 'attr_writer' then 'W'
@@ -810,6 +823,43 @@ class RDoc::Parser::Ruby < RDoc::Parser
   end
 
   ##
+  # Creates an RDoc::Method on +container+ from +comment+ if there is a
+  # Signature section in the comment
+
+  def parse_comment_tomdoc container, tk, comment
+    return unless signature = RDoc::TomDoc.signature(comment)
+    offset  = tk.seek
+    line_no = tk.line_no
+
+    name, = signature.split %r%[ \(]%, 2
+
+    meth = RDoc::GhostMethod.new get_tkread, name
+    meth.record_location @top_level
+    meth.offset    = offset
+    meth.line      = line_no
+
+    meth.start_collecting_tokens
+    indent = TkSPACE.new 0, 1, 1
+    indent.set_text " " * offset
+
+    position_comment = TkCOMMENT.new 0, line_no, 1
+    position_comment.set_text "# File #{@top_level.absolute_name}, line #{line_no}"
+    meth.add_tokens [position_comment, NEWLINE_TOKEN, indent]
+
+    meth.call_seq = signature
+
+    comment.normalize
+
+    return unless meth.name
+
+    container.add_method meth
+
+    meth.comment = comment
+
+    @stats.add_method meth
+  end
+
+  ##
   # Parses an +include+ in +context+ with +comment+
 
   def parse_include context, comment
@@ -830,6 +880,26 @@ class RDoc::Parser::Ruby < RDoc::Parser
   end
 
   ##
+  # Parses an +extend+ in +context+ with +comment+
+
+  def parse_extend context, comment
+    loop do
+      skip_tkspace_comment
+
+      name = get_constant_with_optional_parens
+
+      unless name.empty? then
+        incl = context.add_extend RDoc::Extend.new(name, comment)
+        incl.record_location @top_level
+      end
+
+      return unless TkCOMMA === peek_tk
+
+      get_tk
+    end
+  end
+
+  ##
   # Parses a meta-programmed attribute and creates an RDoc::Attr.
   #
   # To create foo and bar attributes on class C with comment "My attributes":
@@ -867,7 +937,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
     tmp = RDoc::CodeObject.new
     read_documentation_modifiers tmp, RDoc::ATTR_MODIFIERS
 
-    if comment.sub!(/^# +:?(attr(_reader|_writer|_accessor)?): *(\S*).*?\n/i, '') then
+    if comment.text.sub!(/^# +:?(attr(_reader|_writer|_accessor)?): *(\S*).*?\n/i, '') then
       rw = case $1
            when 'attr_reader' then 'R'
            when 'attr_writer' then 'W'
@@ -892,6 +962,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
         @stats.add_attribute att
       end
     end
+
+    att
   end
 
   ##
@@ -908,9 +980,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     skip_tkspace false
 
-    singleton = !!comment.sub!(/(^# +:?)(singleton-)(method:)/, '\1\3')
+    singleton = !!comment.text.sub!(/(^# +:?)(singleton-)(method:)/, '\1\3')
 
-    if comment.sub!(/^# +:?method: *(\S*).*?\n/i, '') then
+    if comment.text.sub!(/^# +:?method: *(\S*).*?\n/i, '') then
       name = $1 unless $1.empty?
     end
 
@@ -939,10 +1011,10 @@ class RDoc::Parser::Ruby < RDoc::Parser
     remove_token_listener self
 
     meth.start_collecting_tokens
-    indent = TkSPACE.new nil, 1, 1
+    indent = TkSPACE.new 0, 1, 1
     indent.set_text " " * column
 
-    position_comment = TkCOMMENT.new nil, line_no, 1
+    position_comment = TkCOMMENT.new 0, line_no, 1
     position_comment.value = "# File #{@top_level.absolute_name}, line #{line_no}"
     meth.add_tokens [position_comment, NEWLINE_TOKEN, indent]
     meth.add_tokens @token_stream
@@ -950,7 +1022,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
     token_listener meth do
       meth.params = ''
 
-      extract_call_seq comment, meth
+      comment.normalize
+      comment.extract_call_seq meth
 
       container.add_method meth
 
@@ -965,7 +1038,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
         when TkSPACE then
           # expression continues
         when TkDO then
-          unget_tk tk
           parse_statements container, single, meth
           break
         else
@@ -977,6 +1049,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
     meth.comment = comment
 
     @stats.add_method meth
+
+    meth
   end
 
   ##
@@ -1002,15 +1076,23 @@ class RDoc::Parser::Ruby < RDoc::Parser
       meth = nil
       added_container = false
 
-      dot = get_tk
-      if TkDOT === dot or TkCOLON2 === dot then
+      case dot = get_tk
+      when TkDOT, TkCOLON2 then
         @scanner.instance_eval do @lex_state = EXPR_FNAME end
         skip_tkspace
         name_t2 = get_tk
 
         case name_t
         when TkSELF, TkMOD then
-          name = name_t2.name
+          name = case name_t2
+                 # NOTE: work around '[' being consumed early and not being
+                 # re-tokenized as a TkAREF
+                 when TkfLBRACK then
+                   get_tk
+                   '[]'
+                 else
+                   name_t2.name
+                 end
         when TkCONSTANT then
           name = name_t2.name
           prev_container = container
@@ -1039,11 +1121,12 @@ class RDoc::Parser::Ruby < RDoc::Parser
         when TkIDENTIFIER, TkIVAR, TkGVAR then
           dummy = RDoc::Context.new
           dummy.parent = container
+          dummy.store  = container.store
           skip_method dummy
           return
         when TkTRUE, TkFALSE, TkNIL then
           klass_name = "#{name_t.name.capitalize}Class"
-          container = RDoc::TopLevel.find_class_named klass_name
+          container = @store.find_class_named klass_name
           container ||= @top_level.add_class RDoc::NormalClass, klass_name
 
           name = name_t2.name
@@ -1084,10 +1167,10 @@ class RDoc::Parser::Ruby < RDoc::Parser
     meth.line   = line_no
 
     meth.start_collecting_tokens
-    indent = TkSPACE.new nil, 1, 1
+    indent = TkSPACE.new 0, 1, 1
     indent.set_text " " * column
 
-    token = TkCOMMENT.new nil, line_no, 1
+    token = TkCOMMENT.new 0, line_no, 1
     token.set_text "# File #{@top_level.absolute_name}, line #{line_no}"
     meth.add_tokens [token, NEWLINE_TOKEN, indent]
     meth.add_tokens @token_stream
@@ -1118,7 +1201,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
       parse_statements container, single, meth
     end
 
-    extract_call_seq comment, meth
+    comment.normalize
+    comment.extract_call_seq meth
 
     meth.comment = comment
 
@@ -1212,17 +1296,18 @@ class RDoc::Parser::Ruby < RDoc::Parser
   ##
   # Parses an RDoc::NormalModule in +container+ with +comment+
 
-  def parse_module(container, single, tk, comment)
+  def parse_module container, single, tk, comment
     container, name_t, = get_class_or_module container
 
     name = name_t.name
 
     mod = container.add_module RDoc::NormalModule, name
+    mod.ignore unless container.document_children
     mod.record_location @top_level
 
     read_documentation_modifiers mod, RDoc::CLASS_MODIFIERS
     mod.add_comment comment, @top_level
-    parse_statements(mod)
+    parse_statements mod
 
     @top_level.add_to_classes_or_modules mod
     @stats.add_module mod
@@ -1253,7 +1338,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # The core of the ruby parser.
 
   def parse_statements(container, single = NORMAL, current_method = nil,
-                       comment = '')
+                       comment = new_comment(''))
+    raise 'no' unless RDoc::Comment === comment
     comment.force_encoding @encoding if @encoding
 
     nest = 1
@@ -1293,6 +1379,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
             end
           end
 
+          comment = new_comment comment
+
           unless comment.empty? then
             look_for_directives_in container, comment
 
@@ -1306,18 +1394,14 @@ class RDoc::Parser::Ruby < RDoc::Parser
           non_comment_seen = true
         end
 
-        unget_tk tk # TODO peek instead of get then unget
+        unget_tk tk
         keep_comment = true
 
       when TkCLASS then
         parse_class container, single, tk, comment
 
       when TkMODULE then
-        if container.document_children then
-          parse_module container, single, tk, comment
-        else
-          nest += 1
-        end
+        parse_module container, single, tk, comment
 
       when TkDEF then
         parse_method container, single, tk, comment
@@ -1354,6 +1438,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
       when TkCASE, TkDO, TkIF, TkUNLESS, TkBEGIN then
         nest += 1
 
+      when TkSUPER then
+        current_method.calls_super = true if current_method
+
       when TkIDENTIFIER then
         if nest == 1 and current_method.nil? then
           case tk.name
@@ -1370,12 +1457,16 @@ class RDoc::Parser::Ruby < RDoc::Parser
           when 'require', 'include' then
             # ignore
           else
-            if comment =~ /\A#\#$/ then
-              case comment
+            if comment.text =~ /\A#\#$/ then
+              case comment.text
               when /^# +:?attr(_reader|_writer|_accessor)?:/ then
                 parse_meta_attr container, single, tk, comment
               else
-                parse_meta_method container, single, tk, comment
+                method = parse_meta_method container, single, tk, comment
+                method.params = container.params if
+                  container.params
+                method.block_params = container.block_params if
+                  container.block_params
               end
             end
           end
@@ -1386,6 +1477,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
           parse_require container, comment
         when "include" then
           parse_include container, comment
+        when "extend" then
+          parse_extend container, comment
         end
 
       when TkEND then
@@ -1410,8 +1503,10 @@ class RDoc::Parser::Ruby < RDoc::Parser
       end
 
       unless keep_comment then
-        comment = ''
+        comment = new_comment ''
         comment.force_encoding @encoding if @encoding
+        container.params = nil
+        container.block_params = nil
       end
 
       begin
@@ -1419,6 +1514,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
         skip_tkspace false
       end while peek_tk == TkNL
     end
+
+    container.params = nil
+    container.block_params = nil
   end
 
   ##
@@ -1497,8 +1595,11 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   def parse_top_level_statements container
     comment = collect_first_comment
+
     look_for_directives_in container, comment
 
+    @markup = comment.format
+
     # HACK move if to RDoc::Context#comment=
     container.comment = comment if container.document_self unless comment.empty?
 
@@ -1604,28 +1705,44 @@ class RDoc::Parser::Ruby < RDoc::Parser
   #
   #   class MyClass # :nodoc:
   #
-  # We return the directive name and any parameters as a two element array
+  # We return the directive name and any parameters as a two element array if
+  # the name is in +allowed+.  A directive can be found anywhere up to the end
+  # of the current line.
 
   def read_directive allowed
-    tk = get_tk
+    tokens = []
 
-    if TkCOMMENT === tk then
-      return unless tk.text =~ /\s*:?(\w+):\s*(.*)/
+    while tk = get_tk do
+      tokens << tk
 
-      directive = $1.downcase
+      case tk
+      when TkNL      then return
+      when TkCOMMENT then
+        return unless tk.text =~ /\s*:?([\w-]+):\s*(.*)/
 
-      return [directive, $2] if allowed.include? directive
-    else
-      unget_tk tk
+        directive = $1.downcase
+
+        return [directive, $2] if allowed.include? directive
+
+        return
+      end
+    end
+  ensure
+    unless tokens.length == 1 and TkCOMMENT === tokens.first then
+      tokens.reverse_each do |token|
+        unget_tk token
+      end
     end
   end
 
   ##
-  # Handles the directive for +context+ if the directive is listed in +allow+.
-  # This method is called for directives following a definition.
+  # Handles directives following the definition for +context+ (any
+  # RDoc::CodeObject) if the directives are +allowed+ at this point.
+  #
+  # See also RDoc::Markup::PreProcess#handle_directive
 
-  def read_documentation_modifiers context, allow
-    directive, value = read_directive allow
+  def read_documentation_modifiers context, allowed
+    directive, value = read_directive allowed
 
     return unless directive
 
@@ -1640,13 +1757,11 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   ##
   # Removes private comments from +comment+
+  #--
+  # TODO remove
 
-  def remove_private_comments(comment)
-    empty = ''
-    empty.force_encoding comment.encoding if Object.const_defined? :Encoding
-
-    comment.gsub!(/^#--.*?^#\+\+\n?/m, empty)
-    comment.sub!(/^#--.*/m, '')
+  def remove_private_comments comment
+    comment.remove_private
   end
 
   ##
@@ -1658,6 +1773,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
     catch :eof do
       begin
         parse_top_level_statements @top_level
+
       rescue StandardError => e
         bytes = ''
 
@@ -1770,12 +1886,10 @@ class RDoc::Parser::Ruby < RDoc::Parser
   end
 
   ##
-  # Prints +msg+ to +$stderr+ unless we're being quiet
+  # Prints +message+ to +$stderr+ unless we're being quiet
 
-  def warn(msg)
-    return if @options.quiet
-    msg = make_message msg
-    $stderr.puts msg
+  def warn message
+    @options.warn make_message message
   end
 
 end
diff --git a/lib/rdoc/parser/ruby_tools.rb b/lib/rdoc/parser/ruby_tools.rb
index 678f721624..654431ea30 100644
--- a/lib/rdoc/parser/ruby_tools.rb
+++ b/lib/rdoc/parser/ruby_tools.rb
@@ -43,8 +43,7 @@ module RDoc::Parser::RubyTools
           tk = Token(TkSYMBOL).set_text(":" + tk1.text)
         end
 
-        # remove the identifier we just read (we're about to replace it with a
-        # symbol)
+        # remove the identifier we just read to replace it with a symbol
         @token_listeners.each do |obj|
           obj.pop_token
         end if @token_listeners
@@ -70,7 +69,13 @@ module RDoc::Parser::RubyTools
 
     loop do
       tk = get_tk
-      case tk when *tokens then unget_tk tk; break end
+
+      case tk
+      when *tokens then
+        unget_tk tk
+        break
+      end
+
       read << tk
     end
 
diff --git a/lib/rdoc/parser/simple.rb b/lib/rdoc/parser/simple.rb
index 1e82eb5097..65cfc1b2e7 100644
--- a/lib/rdoc/parser/simple.rb
+++ b/lib/rdoc/parser/simple.rb
@@ -4,6 +4,8 @@
 
 class RDoc::Parser::Simple < RDoc::Parser
 
+  include RDoc::Parser::Text
+
   parse_files_matching(//)
 
   attr_reader :content # :nodoc:
@@ -24,26 +26,36 @@ class RDoc::Parser::Simple < RDoc::Parser
 
   def scan
     comment = remove_coding_comment @content
-    comment = remove_private_comments comment
+    comment = remove_private_comment comment
+
+    comment = RDoc::Comment.new comment, @top_level
 
     @top_level.comment = comment
-    @top_level.parser = self.class
     @top_level
   end
 
   ##
-  # Removes comments wrapped in <tt>--/++</tt>
-
-  def remove_private_comments text
-    text.gsub(/^--\n.*?^\+\+/m, '').sub(/^--\n.*/m, '')
-  end
-
-  ##
   # Removes the encoding magic comment from +text+
 
   def remove_coding_comment text
     text.sub(/\A# .*coding[=:].*$/, '')
   end
 
+  ##
+  # Removes private comments.
+  #
+  # Unlike RDoc::Comment#remove_private this implementation only looks for two
+  # dashes at the beginning of the line.  Three or more dashes are considered
+  # to be a rule and ignored.
+
+  def remove_private_comment comment
+    # Workaround for gsub encoding for Ruby 1.9.2 and earlier
+    empty = ''
+    empty.force_encoding comment.encoding if Object.const_defined? :Encoding
+
+    comment = comment.gsub(%r%^--\n.*?^\+\+\n?%m, empty)
+    comment.sub(%r%^--\n.*%m, empty)
+  end
+
 end
 
diff --git a/lib/rdoc/parser/text.rb b/lib/rdoc/parser/text.rb
new file mode 100644
index 0000000000..f973313551
--- /dev/null
+++ b/lib/rdoc/parser/text.rb
@@ -0,0 +1,11 @@
+##
+# Indicates this parser is text and doesn't contain code constructs.
+#
+# Include this module in a RDoc::Parser subclass to make it show up as a file,
+# not as part of a class or module.
+#--
+# This is not named File to avoid overriding ::File
+
+module RDoc::Parser::Text
+end
+