Merge rdoc-6.0.0.beta1.

  This version fixed strange behavior of ruby code parser.
  We will list all of impromovement to Changelog when 6.0.0 releasing.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@59686 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

17 files changed, 456 insertions, 313 deletions

diff --git a/lib/rdoc.rb b/lib/rdoc.rb
index 9d66201fc7..8482e910f5 100644
--- a/lib/rdoc.rb
+++ b/lib/rdoc.rb
@@ -65,7 +65,7 @@ module RDoc
   ##
   # RDoc version you are using
 
-  VERSION = '5.1.0'
+  VERSION = '6.0.0.beta1'
 
   ##
   # Method visibilities
diff --git a/lib/rdoc/code_object.rb b/lib/rdoc/code_object.rb
index 21aa2b09f5..11c682b7fa 100644
--- a/lib/rdoc/code_object.rb
+++ b/lib/rdoc/code_object.rb
@@ -70,13 +70,6 @@ class RDoc::CodeObject
   attr_reader :metadata
 
   ##
-  # Offset in #file where this CodeObject was defined
-  #--
-  # TODO character or byte?
-
-  attr_accessor :offset
-
-  ##
   # Sets the parent CodeObject
 
   attr_writer :parent
diff --git a/lib/rdoc/context.rb b/lib/rdoc/context.rb
index 7cfd9a89b4..7fd6bb0af9 100644
--- a/lib/rdoc/context.rb
+++ b/lib/rdoc/context.rb
@@ -762,7 +762,7 @@ class RDoc::Context < RDoc::CodeObject
     attributes.default = []
 
     sort_sections.each do |section|
-      yield section, constants[section].sort, attributes[section].sort
+      yield section, constants[section].select(&:display?).sort, attributes[section].select(&:display?).sort
     end
   end
 
diff --git a/lib/rdoc/generator/template/darkfish/class.rhtml b/lib/rdoc/generator/template/darkfish/class.rhtml
index b497000112..b232866984 100644
--- a/lib/rdoc/generator/template/darkfish/class.rhtml
+++ b/lib/rdoc/generator/template/darkfish/class.rhtml
@@ -26,8 +26,6 @@
   </section>
 
   <% klass.each_section do |section, constants, attributes| %>
-  <% constants = constants.select { |const| const.display? } %>
-  <% attributes = attributes.select { |attr| attr.display? } %>
   <section id="<%= section.aref %>" class="documentation-section">
     <% if section.title then %>
     <header class="documentation-section-title">
diff --git a/lib/rdoc/i18n.rb b/lib/rdoc/i18n.rb
index 14a3739c1c..20848aad75 100644
--- a/lib/rdoc/i18n.rb
+++ b/lib/rdoc/i18n.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: false
 ##
-# This module provides i18n related features.
+# This module provides i18n realated features.
 
 module RDoc::I18n
 
diff --git a/lib/rdoc/markdown.rb b/lib/rdoc/markdown.rb
index b10f5193da..9b433b3ee6 100644
--- a/lib/rdoc/markdown.rb
+++ b/lib/rdoc/markdown.rb
@@ -15280,7 +15280,7 @@ class RDoc::Markdown
         self.pos = _save
         break
       end
-      @result = begin;
+      @result = begin; 
                ref = [:inline, @note_order.length]
                @footnotes[ref] = paragraph a
 
diff --git a/lib/rdoc/parser/c.rb b/lib/rdoc/parser/c.rb
index 087d56f7b9..0e72df2ecd 100644
--- a/lib/rdoc/parser/c.rb
+++ b/lib/rdoc/parser/c.rb
@@ -670,7 +670,6 @@ class RDoc::Parser::C < RDoc::Parser
       tk.set_text body
       meth_obj.add_token tk
       meth_obj.comment = comment
-      meth_obj.offset  = offset
       meth_obj.line    = file_content[0, offset].count("\n") + 1
 
       body
@@ -689,7 +688,6 @@ class RDoc::Parser::C < RDoc::Parser
       tk.set_text body
       meth_obj.add_token tk
       meth_obj.comment = comment
-      meth_obj.offset  = offset
       meth_obj.line    = file_content[0, offset].count("\n") + 1
 
       body
diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb
index 9562ddbd7a..e7900c0807 100644
--- a/lib/rdoc/parser/ruby.rb
+++ b/lib/rdoc/parser/ruby.rb
@@ -589,7 +589,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # +comment+.
 
   def parse_attr(context, single, tk, comment)
-    offset  = tk.seek
     line_no = tk.line_no
 
     args = parse_symbol_arg 1
@@ -606,7 +605,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
       end
 
       att = create_attr context, single, name, rw, comment
-      att.offset = offset
       att.line   = line_no
 
       read_documentation_modifiers att, RDoc::ATTR_MODIFIERS
@@ -620,7 +618,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # comment for each to +comment+.
 
   def parse_attr_accessor(context, single, tk, comment)
-    offset  = tk.seek
     line_no = tk.line_no
 
     args = parse_symbol_arg
@@ -642,7 +639,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     for name in args
       att = create_attr context, single, name, rw, comment
-      att.offset = offset
       att.line   = line_no
     end
   end
@@ -651,7 +647,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Parses an +alias+ in +context+ with +comment+
 
   def parse_alias(context, single, tk, comment)
-    offset  = tk.seek
     line_no = tk.line_no
 
     skip_tkspace
@@ -680,7 +675,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
     al = RDoc::Alias.new(get_tkread, old_name, new_name, comment,
                          single == SINGLE)
     record_location al
-    al.offset = offset
     al.line   = line_no
 
     read_documentation_modifiers al, RDoc::ATTR_MODIFIERS
@@ -733,7 +727,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Parses a class in +context+ with +comment+
 
   def parse_class container, single, tk, comment
-    offset  = tk.seek
     line_no = tk.line_no
 
     declaration_context = container
@@ -748,7 +741,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
         case name = get_class_specification
         when 'self', container.name
           parse_statements container, SINGLE
-          return # don't update offset or line
+          return # don't update line
         else
           parse_class_singleton container, name, comment
         end
@@ -757,9 +750,11 @@ class RDoc::Parser::Ruby < RDoc::Parser
         return
       end
 
-    cls.offset = offset
     cls.line   = line_no
 
+    # after end modifiers
+    read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS
+
     cls
   end
 
@@ -845,7 +840,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # true, no found constants will be added to RDoc.
 
   def parse_constant container, tk, comment, ignore_constants = false
-    offset  = tk.seek
     line_no = tk.line_no
 
     name = tk.name
@@ -885,7 +879,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     value.replace body
     record_location con
-    con.offset = offset
     con.line   = line_no
     read_documentation_modifiers con, RDoc::CONSTANT_MODIFIERS
 
@@ -950,7 +943,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
   def parse_comment container, tk, comment
     return parse_comment_tomdoc container, tk, comment if @markup == 'tomdoc'
     column  = tk.char_no
-    offset  = tk.seek
     line_no = tk.line_no
 
     text = comment.text
@@ -966,7 +958,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     if co then
       co.singleton = singleton
-      co.offset    = offset
       co.line      = line_no
     end
 
@@ -1031,19 +1022,18 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   def parse_comment_tomdoc container, tk, comment
     return unless signature = RDoc::TomDoc.signature(comment)
-    offset  = tk.seek
+    column  = tk.char_no
     line_no = tk.line_no
 
     name, = signature.split %r%[ \(]%, 2
 
     meth = RDoc::GhostMethod.new get_tkread, name
     record_location meth
-    meth.offset    = offset
     meth.line      = line_no
 
     meth.start_collecting_tokens
     indent = TkSPACE.new 0, 1, 1
-    indent.set_text " " * offset
+    indent.set_text " " * column
 
     position_comment = TkCOMMENT.new 0, line_no, 1
     position_comment.set_text "# File #{@top_level.relative_name}, line #{line_no}"
@@ -1183,7 +1173,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   def parse_meta_method(container, single, tk, comment)
     column  = tk.char_no
-    offset  = tk.seek
     line_no = tk.line_no
 
     start_collecting_tokens
@@ -1200,7 +1189,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     meth = RDoc::MetaMethod.new get_tkread, name
     record_location meth
-    meth.offset = offset
     meth.line   = line_no
     meth.singleton = singleton
 
@@ -1291,7 +1279,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
     added_container = false
     name = nil
     column  = tk.char_no
-    offset  = tk.seek
     line_no = tk.line_no
 
     start_collecting_tokens
@@ -1309,7 +1296,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
     meth.singleton = single == SINGLE ? true : singleton
 
     record_location meth
-    meth.offset = offset
     meth.line   = line_no
 
     meth.start_collecting_tokens
@@ -1328,6 +1314,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     meth.comment = comment
 
+    # after end modifiers
+    read_documentation_modifiers meth, RDoc::METHOD_MODIFIERS
+
     @stats.add_method meth
   end
 
@@ -1519,6 +1508,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
       end
       tk = get_tk
     end
+    @scanner.first_in_method_statement = true
 
     get_tkread_clean(/\s+/, ' ')
   end
@@ -1559,6 +1549,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
     mod.add_comment comment, @top_level
     parse_statements mod
 
+    # after end modifiers
+    read_documentation_modifiers mod, RDoc::CLASS_MODIFIERS
+
     @stats.add_module mod
   end
 
@@ -1732,7 +1725,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
       when TkEND then
         nest -= 1
         if nest == 0 then
-          read_documentation_modifiers container, RDoc::CLASS_MODIFIERS
           container.ongoing_visibility = save_visibility
 
           parse_comment container, tk, comment unless comment.empty?
@@ -2039,7 +2031,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
   def skip_optional_do_after_expression
     skip_tkspace false
     tk = get_tk
-    end_token = get_end_token tk
 
     b_nest = 0
     nest = 0
@@ -2047,23 +2038,18 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     loop do
       case tk
-      when TkSEMICOLON then
+      when TkSEMICOLON, TkNL then
         break if b_nest.zero?
       when TkLPAREN, TkfLPAREN then
         nest += 1
+      when TkRPAREN then
+        nest -= 1
       when TkBEGIN then
         b_nest += 1
       when TkEND then
         b_nest -= 1
       when TkDO
         break if nest.zero?
-      when end_token then
-        if end_token == TkRPAREN
-          nest -= 1
-          break if @scanner.lex_state == :EXPR_END and nest.zero?
-        else
-          break unless @scanner.continue
-        end
       when nil then
         break
       end
diff --git a/lib/rdoc/parser/ruby_tools.rb b/lib/rdoc/parser/ruby_tools.rb
index bbca065b5e..e35b420f30 100644
--- a/lib/rdoc/parser/ruby_tools.rb
+++ b/lib/rdoc/parser/ruby_tools.rb
@@ -33,26 +33,6 @@ module RDoc::Parser::RubyTools
 
     tk = nil if TkEND_OF_SCRIPT === tk
 
-    if TkSYMBEG === tk then
-      set_token_position tk.line_no, tk.char_no
-
-      case tk1 = get_tk
-      when TkId, TkOp, TkSTRING, TkDSTRING, TkSTAR, TkAMPER then
-        if tk1.respond_to?(:name) then
-          tk = Token(TkSYMBOL).set_text(":" + tk1.name)
-        else
-          tk = Token(TkSYMBOL).set_text(":" + tk1.text)
-        end
-
-        # remove the identifier we just read to replace it with a symbol
-        @token_listeners.each do |obj|
-          obj.pop_token
-        end if @token_listeners
-      else
-        tk = tk1
-      end
-    end
-
     # inform any listeners of our shiny new token
     @token_listeners.each do |obj|
       obj.add_token(tk)
diff --git a/lib/rdoc/rd/block_parser.rb b/lib/rdoc/rd/block_parser.rb
index 954e708495..8ac7cad996 100644
--- a/lib/rdoc/rd/block_parser.rb
+++ b/lib/rdoc/rd/block_parser.rb
@@ -1,7 +1,7 @@
 #
 # DO NOT MODIFY!!!!
 # This file is automatically generated by Racc 1.4.14
-# from Racc grammar file "".
+# from Racc grammer file "".
 #
 
 require 'racc/parser.rb'
@@ -677,54 +677,54 @@ Racc_debug_parser = false
 # reduce 0 omitted
 
 def _reduce_1(val, _values, result)
- result = RDoc::Markup::Document.new(*val[0])
+ result = RDoc::Markup::Document.new(*val[0]) 
     result
 end
 
 def _reduce_2(val, _values, result)
- raise ParseError, "file empty"
+ raise ParseError, "file empty" 
     result
 end
 
 def _reduce_3(val, _values, result)
- result = val[0].concat val[1]
+ result = val[0].concat val[1] 
     result
 end
 
 def _reduce_4(val, _values, result)
- result = val[0]
+ result = val[0] 
     result
 end
 
 def _reduce_5(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 def _reduce_6(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 # reduce 7 omitted
 
 def _reduce_8(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 def _reduce_9(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 def _reduce_10(val, _values, result)
- result = [RDoc::Markup::BlankLine.new]
+ result = [RDoc::Markup::BlankLine.new] 
     result
 end
 
 def _reduce_11(val, _values, result)
- result = val[0].parts
+ result = val[0].parts 
     result
 end
 
@@ -732,30 +732,30 @@ def _reduce_12(val, _values, result)
       # val[0] is like [level, title]
       title = @inline_parser.parse(val[0][1])
       result = RDoc::Markup::Heading.new(val[0][0], title)
-
+    
     result
 end
 
 def _reduce_13(val, _values, result)
       result = RDoc::Markup::Include.new val[0], @include_path
-
+    
     result
 end
 
 def _reduce_14(val, _values, result)
       # val[0] is Array of String
       result = paragraph val[0]
-
+    
     result
 end
 
 def _reduce_15(val, _values, result)
- result << val[1].rstrip
+ result << val[1].rstrip 
     result
 end
 
 def _reduce_16(val, _values, result)
- result = [val[0].rstrip]
+ result = [val[0].rstrip] 
     result
 end
 
@@ -766,7 +766,7 @@ def _reduce_17(val, _values, result)
 
       # imform to lexer.
       @in_verbatim = false
-
+    
     result
 end
 
@@ -777,25 +777,25 @@ def _reduce_18(val, _values, result)
 
       # imform to lexer.
       @in_verbatim = false
-
+    
     result
 end
 
 def _reduce_19(val, _values, result)
       result << val[1]
-
+    
     result
 end
 
 def _reduce_20(val, _values, result)
       result.concat val[2]
-
+    
     result
 end
 
 def _reduce_21(val, _values, result)
       result << "\n"
-
+    
     result
 end
 
@@ -803,7 +803,7 @@ def _reduce_22(val, _values, result)
       result = val
       # inform to lexer.
       @in_verbatim = true
-
+    
     result
 end
 
@@ -817,89 +817,89 @@ end
 
 def _reduce_27(val, _values, result)
       result = val[0]
-
+    
     result
 end
 
 def _reduce_28(val, _values, result)
       result = val[1]
-
+    
     result
 end
 
 def _reduce_29(val, _values, result)
       result = val[1].push(val[2])
-
+    
     result
 end
 
 def _reduce_30(val, _values, result)
- result = val[0] << val[1]
+ result = val[0] << val[1] 
     result
 end
 
 def _reduce_31(val, _values, result)
- result = [val[0]]
+ result = [val[0]] 
     result
 end
 
 def _reduce_32(val, _values, result)
       result = RDoc::Markup::List.new :BULLET, *val[0]
-
+    
     result
 end
 
 def _reduce_33(val, _values, result)
- result.push(val[1])
+ result.push(val[1]) 
     result
 end
 
 def _reduce_34(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 def _reduce_35(val, _values, result)
       result = RDoc::Markup::ListItem.new nil, val[0], *val[1]
-
+    
     result
 end
 
 def _reduce_36(val, _values, result)
       result = RDoc::Markup::List.new :NUMBER, *val[0]
-
+    
     result
 end
 
 def _reduce_37(val, _values, result)
- result.push(val[1])
+ result.push(val[1]) 
     result
 end
 
 def _reduce_38(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 def _reduce_39(val, _values, result)
       result = RDoc::Markup::ListItem.new nil, val[0], *val[1]
-
+    
     result
 end
 
 def _reduce_40(val, _values, result)
       result = RDoc::Markup::List.new :NOTE, *val[0]
-
+    
     result
 end
 
 def _reduce_41(val, _values, result)
- result.push(val[1])
+ result.push(val[1]) 
     result
 end
 
 def _reduce_42(val, _values, result)
- result = val
+ result = val 
     result
 end
 
@@ -907,77 +907,77 @@ def _reduce_43(val, _values, result)
       term = @inline_parser.parse val[0].strip
 
       result = RDoc::Markup::ListItem.new term, *val[1]
-
+    
     result
 end
 
 def _reduce_44(val, _values, result)
       result = RDoc::Markup::List.new :LABEL, *val[0]
-
+    
     result
 end
 
 def _reduce_45(val, _values, result)
- result.push(val[1])
+ result.push(val[1]) 
     result
 end
 
 def _reduce_46(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 def _reduce_47(val, _values, result)
       result = RDoc::Markup::ListItem.new "<tt>#{val[0].strip}</tt>", *val[1]
-
+    
     result
 end
 
 def _reduce_48(val, _values, result)
       result = [val[1]].concat(val[2])
-
+    
     result
 end
 
 def _reduce_49(val, _values, result)
       result = [val[1]]
-
+    
     result
 end
 
 def _reduce_50(val, _values, result)
       result = val[2]
-
+    
     result
 end
 
 def _reduce_51(val, _values, result)
       result = []
-
+    
     result
 end
 
 def _reduce_52(val, _values, result)
- result.concat val[1]
+ result.concat val[1] 
     result
 end
 
 # reduce 53 omitted
 
 def _reduce_54(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 def _reduce_55(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 # reduce 56 omitted
 
 def _reduce_57(val, _values, result)
- result = []
+ result = [] 
     result
 end
 
@@ -991,58 +991,58 @@ end
 
 def _reduce_62(val, _values, result)
       result = paragraph [val[0]].concat(val[1])
-
+    
     result
 end
 
 def _reduce_63(val, _values, result)
       result = paragraph [val[0]]
-
+    
     result
 end
 
 def _reduce_64(val, _values, result)
       result = paragraph [val[0]].concat(val[1])
-
+    
     result
 end
 
 def _reduce_65(val, _values, result)
       result = paragraph [val[0]]
-
+    
     result
 end
 
 def _reduce_66(val, _values, result)
       result = [val[0]].concat(val[1])
-
+    
     result
 end
 
 def _reduce_67(val, _values, result)
- result.concat val[1]
+ result.concat val[1] 
     result
 end
 
 def _reduce_68(val, _values, result)
- result = val[1]
+ result = val[1] 
     result
 end
 
 def _reduce_69(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 # reduce 70 omitted
 
 def _reduce_71(val, _values, result)
- result = []
+ result = [] 
     result
 end
 
 def _reduce_72(val, _values, result)
- result = []
+ result = [] 
     result
 end
 
diff --git a/lib/rdoc/rd/inline_parser.rb b/lib/rdoc/rd/inline_parser.rb
index a2792973ab..4a3f3ed7b5 100644
--- a/lib/rdoc/rd/inline_parser.rb
+++ b/lib/rdoc/rd/inline_parser.rb
@@ -1,7 +1,7 @@
 #
 # DO NOT MODIFY!!!!
 # This file is automatically generated by Racc 1.4.14
-# from Racc grammar file "".
+# from Racc grammer file "".
 #
 
 require 'racc/parser.rb'
@@ -704,9 +704,9 @@ Racc_token_to_s_table = [
   "ref_subst_strings_q",
   "ref_subst_strings_first",
   "ref_subst_ele2",
-  "ref_subst_eels",
+  "ref_subst_eles",
   "ref_subst_str_ele_first",
-  "ref_subst_eels_q",
+  "ref_subst_eles_q",
   "ref_subst_ele",
   "ref_subst_ele_q",
   "ref_subst_str_ele",
@@ -732,12 +732,12 @@ Racc_debug_parser = false
 # reduce 1 omitted
 
 def _reduce_2(val, _values, result)
- result.append val[1]
+ result.append val[1] 
     result
 end
 
 def _reduce_3(val, _values, result)
- result = val[0]
+ result = val[0] 
     result
 end
 
@@ -762,28 +762,28 @@ end
 def _reduce_13(val, _values, result)
       content = val[1]
       result = inline "<em>#{content}</em>", content
-
+    
     result
 end
 
 def _reduce_14(val, _values, result)
       content = val[1]
       result = inline "<code>#{content}</code>", content
-
+    
     result
 end
 
 def _reduce_15(val, _values, result)
       content = val[1]
       result = inline "+#{content}+", content
-
+    
     result
 end
 
 def _reduce_16(val, _values, result)
       content = val[1]
       result = inline "<tt>#{content}</tt>", content
-
+    
     result
 end
 
@@ -791,13 +791,13 @@ def _reduce_17(val, _values, result)
       label = val[1]
       @block_parser.add_label label.reference
       result = "<span id=\"label-#{label}\">#{label}</span>"
-
+    
     result
 end
 
 def _reduce_18(val, _values, result)
       result = "{#{val[1]}}[#{val[2].join}]"
-
+    
     result
 end
 
@@ -805,13 +805,13 @@ def _reduce_19(val, _values, result)
       scheme, inline = val[1]
 
       result = "{#{inline}}[#{scheme}#{inline.reference}]"
-
+    
     result
 end
 
 def _reduce_20(val, _values, result)
       result = [nil, inline(val[1])]
-
+    
     result
 end
 
@@ -820,25 +820,25 @@ def _reduce_21(val, _values, result)
         'rdoc-label:',
         inline("#{val[0].reference}/#{val[1].reference}")
       ]
-
+    
     result
 end
 
 def _reduce_22(val, _values, result)
       result = ['rdoc-label:', val[0].reference]
-
+    
     result
 end
 
 def _reduce_23(val, _values, result)
       result = ['rdoc-label:', "#{val[0].reference}/"]
-
+    
     result
 end
 
 def _reduce_24(val, _values, result)
       result = [nil, inline(val[1])]
-
+    
     result
 end
 
@@ -847,92 +847,92 @@ def _reduce_25(val, _values, result)
         'rdoc-label:',
         inline("#{val[0].reference}/#{val[1].reference}")
       ]
-
+    
     result
 end
 
 def _reduce_26(val, _values, result)
       result = ['rdoc-label:', val[0]]
-
+    
     result
 end
 
 def _reduce_27(val, _values, result)
       ref = val[0].reference
       result = ['rdoc-label:', inline(ref, "#{ref}/")]
-
+    
     result
 end
 
 # reduce 28 omitted
 
 def _reduce_29(val, _values, result)
- result = val[1]
+ result = val[1] 
     result
 end
 
 def _reduce_30(val, _values, result)
- result = val[1]
+ result = val[1] 
     result
 end
 
 def _reduce_31(val, _values, result)
       result = inline val[0]
-
+    
     result
 end
 
 def _reduce_32(val, _values, result)
       result = inline "\"#{val[1]}\""
-
+    
     result
 end
 
 def _reduce_33(val, _values, result)
       result = inline val[0]
-
+    
     result
 end
 
 def _reduce_34(val, _values, result)
       result = inline "\"#{val[1]}\""
-
+    
     result
 end
 
 # reduce 35 omitted
 
 def _reduce_36(val, _values, result)
- result = val[1]
+ result = val[1] 
     result
 end
 
 def _reduce_37(val, _values, result)
- result = inline val[1]
+ result = inline val[1] 
     result
 end
 
 def _reduce_38(val, _values, result)
       result = val[0].append val[1]
-
+    
     result
 end
 
 def _reduce_39(val, _values, result)
       result = val[0].append val[1]
-
+    
     result
 end
 
 def _reduce_40(val, _values, result)
       result = val[0]
-
+    
     result
 end
 
 def _reduce_41(val, _values, result)
       result = inline val[0]
-
+    
     result
 end
 
@@ -940,25 +940,25 @@ end
 
 def _reduce_43(val, _values, result)
       result = val[0].append val[1]
-
+    
     result
 end
 
 def _reduce_44(val, _values, result)
       result = inline val[0]
-
+    
     result
 end
 
 def _reduce_45(val, _values, result)
       result = val[0].append val[1]
-
+    
     result
 end
 
 def _reduce_46(val, _values, result)
       result = val[0]
-
+    
     result
 end
 
@@ -984,24 +984,24 @@ end
 
 def _reduce_57(val, _values, result)
       result = val[0]
-
+    
     result
 end
 
 def _reduce_58(val, _values, result)
       result = inline val[0]
-
+    
     result
 end
 
 def _reduce_59(val, _values, result)
       result = inline val[0]
-
+    
     result
 end
 
 def _reduce_60(val, _values, result)
- result << val[1]
+ result << val[1] 
     result
 end
 
@@ -1009,7 +1009,7 @@ end
 
 def _reduce_62(val, _values, result)
       result << val[1]
-
+    
     result
 end
 
@@ -1017,7 +1017,7 @@ end
 
 def _reduce_64(val, _values, result)
       result << val[1]
-
+    
     result
 end
 
@@ -1048,7 +1048,7 @@ end
 # reduce 77 omitted
 
 def _reduce_78(val, _values, result)
- result << val[1]
+ result << val[1] 
     result
 end
 
@@ -1099,13 +1099,13 @@ end
 def _reduce_101(val, _values, result)
       index = @block_parser.add_footnote val[1].rdoc
       result = "{*#{index}}[rdoc-label:foottext-#{index}:footmark-#{index}]"
-
+    
     result
 end
 
 def _reduce_102(val, _values, result)
       result = inline "<tt>#{val[1]}</tt>", val[1]
-
+    
     result
 end
 
@@ -1122,7 +1122,7 @@ end
 # reduce 108 omitted
 
 def _reduce_109(val, _values, result)
- result << val[1]
+ result << val[1] 
     result
 end
 
@@ -1130,24 +1130,24 @@ end
 
 def _reduce_111(val, _values, result)
       result = inline val[0]
-
+    
     result
 end
 
 # reduce 112 omitted
 
 def _reduce_113(val, _values, result)
- result = val[1]
+ result = val[1] 
     result
 end
 
 def _reduce_114(val, _values, result)
- result = val[1]
+ result = val[1] 
     result
 end
 
 def _reduce_115(val, _values, result)
- result = val[1]
+ result = val[1] 
     result
 end
 
@@ -1192,7 +1192,7 @@ end
 # reduce 135 omitted
 
 def _reduce_136(val, _values, result)
- result << val[1]
+ result << val[1] 
     result
 end
 
diff --git a/lib/rdoc/rdoc.gemspec b/lib/rdoc/rdoc.gemspec
index e605560868..1822c27366 100644
--- a/lib/rdoc/rdoc.gemspec
+++ b/lib/rdoc/rdoc.gemspec
@@ -21,7 +21,7 @@ Gem::Specification.new do |s|
 RDoc produces HTML and command-line documentation for Ruby projects.
 RDoc includes the +rdoc+ and +ri+ tools for generating and displaying documentation from the command-line.
   DESCRIPTION
-  s.homepage = "https://rdoc.github.io/rdoc"
+  s.homepage = "https://ruby.github.io/rdoc"
   s.licenses = ["Ruby"]
 
   s.bindir = "exe"
@@ -46,7 +46,7 @@ RDoc includes the +rdoc+ and +ri+ tools for generating and displaying documentat
     TODO.rdoc
   ]
 
-  s.required_ruby_version = Gem::Requirement.new(">= 1.9.3")
+  s.required_ruby_version = Gem::Requirement.new(">= 2.2.2")
   s.rubygems_version = "2.5.2"
   s.required_rubygems_version = Gem::Requirement.new(">= 2.2")
 
diff --git a/lib/rdoc/ri/driver.rb b/lib/rdoc/ri/driver.rb
index 23d24e9a20..8e68f72d53 100644
--- a/lib/rdoc/ri/driver.rb
+++ b/lib/rdoc/ri/driver.rb
@@ -80,7 +80,6 @@ class RDoc::RI::Driver
     options[:interactive] = false
     options[:profile]     = false
     options[:show_all]    = false
-    options[:use_cache]   = true
     options[:use_stdout]  = !$stdout.tty?
     options[:width]       = 72
 
@@ -122,7 +121,7 @@ class RDoc::RI::Driver
       opt.summary_indent = ' ' * 4
 
       opt.banner = <<-EOT
-Usage: #{opt.program_name} [options] [names...]
+Usage: #{opt.program_name} [options] [name ...]
 
 Where name can be:
 
@@ -132,8 +131,8 @@ Where name can be:
 
   gem_name: | gem_name:README | gem_name:History
 
-All class names may be abbreviated to their minimum unambiguous form. If a name
-is ambiguous, all valid options will be listed.
+All class names may be abbreviated to their minimum unambiguous form.
+If a name is ambiguous, all valid options will be listed.
 
 A '.' matches either class or instance methods, while #method
 matches only instance and ::method matches only class methods.
@@ -151,23 +150,23 @@ For example:
     #{opt.program_name} zip
     #{opt.program_name} rdoc:README
 
-Note that shell quoting or escaping may be required for method names containing
-punctuation:
+Note that shell quoting or escaping may be required for method names
+containing punctuation:
 
     #{opt.program_name} 'Array.[]'
     #{opt.program_name} compact\\!
 
-To see the default directories ri will search, run:
+To see the default directories #{opt.program_name} will search, run:
 
     #{opt.program_name} --list-doc-dirs
 
-Specifying the --system, --site, --home, --gems or --doc-dir options will
-limit ri to searching only the specified directories.
+Specifying the --system, --site, --home, --gems, or --doc-dir options
+will limit ri to searching only the specified directories.
 
-ri options may be set in the 'RI' environment variable.
+ri options may be set in the RI environment variable.
 
-The ri pager can be set with the 'RI_PAGER' environment variable or the
-'PAGER' environment variable.
+The ri pager can be set with the RI_PAGER environment variable
+or the PAGER environment variable.
       EOT
 
       opt.separator nil
@@ -199,15 +198,15 @@ The ri pager can be set with the 'RI_PAGER' environment variable or the
       opt.separator nil
 
       opt.on("--[no-]pager",
-             "Send output directly to stdout,",
-             "rather than to a pager.") do |use_pager|
+             "Send output to a pager,",
+             "rather than directly to stdout.") do |use_pager|
         options[:use_stdout] = !use_pager
       end
 
       opt.separator nil
 
       opt.on("-T",
-             "Synonym for --no-pager") do
+             "Synonym for --no-pager.") do
         options[:use_stdout] = true
       end
 
@@ -220,7 +219,7 @@ The ri pager can be set with the 'RI_PAGER' environment variable or the
 
       opt.separator nil
 
-      opt.on("--server [PORT]", Integer,
+      opt.on("--server[=PORT]", Integer,
              "Run RDoc server on the given port.",
              "The default port is 8214.") do |port|
         options[:server] = port || 8214
@@ -235,14 +234,30 @@ The ri pager can be set with the 'RI_PAGER' environment variable or the
       formatters -= %w[html label test] # remove useless output formats
 
       opt.on("--format=NAME", "-f",
-             "Uses the selected formatter. The default",
+             "Use the selected formatter.  The default",
              "formatter is bs for paged output and ansi",
-             "otherwise. Valid formatters are:",
-             formatters.join(' '), formatters) do |value|
+             "otherwise.  Valid formatters are:",
+             "#{formatters.join(', ')}.", formatters) do |value|
         options[:formatter] = RDoc::Markup.const_get "To#{value.capitalize}"
       end
 
       opt.separator nil
+
+      opt.on("--help", "-h",
+             "Show help and exit.") do
+        puts opts
+        exit
+      end
+
+      opt.separator nil
+
+      opt.on("--version", "-v",
+             "Output version information and exit.") do
+        puts "#{opts.program_name} #{opts.version}"
+        exit
+      end
+
+      opt.separator nil
       opt.separator "Data source options:"
       opt.separator nil
 
@@ -273,7 +288,7 @@ The ri pager can be set with the 'RI_PAGER' environment variable or the
              "Do not include documentation from",
              "the Ruby standard library, site_lib,",
              "installed gems, or ~/.rdoc.",
-             "Use with --doc-dir") do
+             "Use with --doc-dir.") do
         options[:use_system] = false
         options[:use_site] = false
         options[:use_gems] = false
@@ -283,8 +298,8 @@ The ri pager can be set with the 'RI_PAGER' environment variable or the
       opt.separator nil
 
       opt.on("--[no-]system",
-             "Include documentation from Ruby's standard",
-             "library.  Defaults to true.") do |value|
+             "Include documentation from Ruby's",
+             "standard library.  Defaults to true.") do |value|
         options[:use_system] = value
       end
 
@@ -318,14 +333,14 @@ The ri pager can be set with the 'RI_PAGER' environment variable or the
       opt.separator nil
 
       opt.on("--[no-]profile",
-             "Run with the ruby profiler") do |value|
+             "Run with the ruby profiler.") do |value|
         options[:profile] = value
       end
 
       opt.separator nil
 
       opt.on("--dump=CACHE", File,
-             "Dumps data from an ri cache or data file") do |value|
+             "Dump data from an ri cache or data file.") do |value|
         options[:dump_path] = value
       end
     end
diff --git a/lib/rdoc/ruby_lex.rb b/lib/rdoc/ruby_lex.rb
index 1fc3c12c4a..e76fdf0414 100644
--- a/lib/rdoc/ruby_lex.rb
+++ b/lib/rdoc/ruby_lex.rb
@@ -45,6 +45,7 @@ class RDoc::RubyLex
 
   attr_accessor :continue
   attr_accessor :lex_state
+  attr_accessor :first_in_method_statement
   attr_reader :reader
 
   class << self
@@ -106,10 +107,15 @@ class RDoc::RubyLex
     @rests = []
     @seek = 0
 
+    @heredoc_queue = []
+
     @indent = 0
     @indent_stack = []
     @lex_state = :EXPR_BEG
     @space_seen = false
+    @escaped_nl = false
+    @first_in_method_statement = false
+    @after_question = false
 
     @continue = false
     @line = ""
@@ -350,6 +356,7 @@ class RDoc::RubyLex
       begin
         tk = @OP.match(self)
         @space_seen = tk.kind_of?(TkSPACE)
+        @first_in_method_statement = false if !@space_seen && @first_in_method_statement
       rescue SyntaxError => e
         raise Error, "syntax error: #{e.message}" if
           @exception_on_syntax_error
@@ -361,6 +368,28 @@ class RDoc::RubyLex
     if @readed_auto_clean_up
       get_readed
     end
+
+    if TkSYMBEG === tk then
+      tk1 = token
+      set_token_position tk.seek, tk.line_no, tk.char_no
+
+      case tk1
+      when TkId, TkOp, TkSTRING, TkDSTRING, TkSTAR, TkAMPER then
+        if tk1.respond_to?(:name) then
+          tk = Token(TkSYMBOL, ":" + tk1.name)
+        else
+          tk = Token(TkSYMBOL, ":" + tk1.text)
+        end
+      else
+        tk = tk1
+      end
+    elsif (TkPLUS === tk or TkMINUS === tk) and peek(0) =~ /\d/ then
+      tk1 = token
+      set_token_position tk.seek, tk.line_no, tk.char_no
+      tk = Token(tk1.class, tk.text + tk1.text)
+    end
+    @after_question = false if @after_question and !(TkQUESTION === tk)
+
     #      Tracer.off
     tk
   end
@@ -380,7 +409,9 @@ class RDoc::RubyLex
     "r" => "/",
     "w" => "]",
     "W" => "]",
-    "s" => ":"
+    "s" => ":",
+    "i" => "]",
+    "I" => "]"
   }
 
   PERCENT_PAREN = {
@@ -430,15 +461,18 @@ class RDoc::RubyLex
                  proc{|op, io| @prev_char_no == 0 && peek(0) =~ /\s/}) do
       |op, io|
       @ltype = "="
-      res = ''
-      nil until getc == "\n"
+      res = op
+      until (ch = getc) == "\n" do
+        res << ch
+      end
+      res << ch
 
       until ( peek_equal?("=end") && peek(4) =~ /\s/ ) do
         (ch = getc)
         res << ch
       end
 
-      gets # consume =end
+      res << gets # consume =end
 
       @ltype = nil
       Token(TkRD_COMMENT, res)
@@ -446,42 +480,90 @@ class RDoc::RubyLex
 
     @OP.def_rule("\n") do |op, io|
       print "\\n\n" if RDoc::RubyLex.debug?
+      unless @heredoc_queue.empty?
+        info = @heredoc_queue[0]
+        if !info[:started] # "\n"
+          info[:started] = true
+          ungetc "\n"
+        elsif info[:heredoc_end].nil? # heredoc body
+          tk, heredoc_end = identify_here_document_body(info[:quoted], info[:lt], info[:indent])
+          info[:heredoc_end] = heredoc_end
+          ungetc "\n"
+        else # heredoc end
+          @heredoc_queue.shift
+          @lex_state = :EXPR_BEG
+          tk = Token(TkHEREDOCEND, info[:heredoc_end])
+          if !@heredoc_queue.empty?
+            @heredoc_queue[0][:started] = true
+            ungetc "\n"
+          end
+        end
+      end
+      unless tk
+        case @lex_state
+        when :EXPR_BEG, :EXPR_FNAME, :EXPR_DOT
+          @continue = true
+        else
+          @continue = false
+          @lex_state = :EXPR_BEG unless @escaped_nl
+          until (@indent_stack.empty? ||
+                 [TkLPAREN, TkLBRACK, TkLBRACE,
+                   TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
+            @indent_stack.pop
+          end
+        end
+        @current_readed = @readed
+        @here_readed.clear
+        tk = Token(TkNL)
+      end
+      @escaped_nl = false
+      tk
+    end
+
+    @OP.def_rules("=") do
+      |op, io|
       case @lex_state
-      when :EXPR_BEG, :EXPR_FNAME, :EXPR_DOT
-        @continue = true
+      when :EXPR_FNAME, :EXPR_DOT
+        @lex_state = :EXPR_ARG
       else
-        @continue = false
         @lex_state = :EXPR_BEG
-        until (@indent_stack.empty? ||
-               [TkLPAREN, TkLBRACK, TkLBRACE,
-                 TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
-          @indent_stack.pop
-        end
       end
-      @current_readed = @readed
-      @here_readed.clear
-      Token(TkNL)
+      Token(op)
     end
 
     @OP.def_rules("*", "**",
-                  "=", "==", "===",
+                  "==", "===",
                   "=~", "<=>",
                   "<", "<=",
-                  ">", ">=", ">>") do
+                  ">", ">=", ">>", "=>") do
       |op, io|
       case @lex_state
       when :EXPR_FNAME, :EXPR_DOT
+        tk = Token(TkId, op)
         @lex_state = :EXPR_ARG
       else
+        tk = Token(op)
         @lex_state = :EXPR_BEG
       end
+      tk
+    end
+
+    @OP.def_rules("->") do
+      |op, io|
+      @lex_state = :EXPR_ENDFN
       Token(op)
     end
 
     @OP.def_rules("!", "!=", "!~") do
       |op, io|
-      @lex_state = :EXPR_BEG
-      Token(op)
+      case @lex_state
+      when :EXPR_FNAME, :EXPR_DOT
+        @lex_state = :EXPR_ARG
+        Token(TkId, op)
+      else
+        @lex_state = :EXPR_BEG
+        Token(op)
+      end
     end
 
     @OP.def_rules("<<") do
@@ -490,16 +572,17 @@ class RDoc::RubyLex
       if @lex_state != :EXPR_END && @lex_state != :EXPR_CLASS &&
          (@lex_state != :EXPR_ARG || @space_seen)
         c = peek(0)
-        if /\S/ =~ c && (/["'`]/ =~ c || /\w/ =~ c || c == "-")
-          tk = identify_here_document
+        if /\S/ =~ c && (/["'`]/ =~ c || /\w/ =~ c || c == "-" || c == "~")
+          tk = identify_here_document(op)
         end
       end
       unless tk
-        tk = Token(op)
         case @lex_state
         when :EXPR_FNAME, :EXPR_DOT
+          tk = Token(TkId, op)
           @lex_state = :EXPR_ARG
         else
+          tk = Token(op)
           @lex_state = :EXPR_BEG
         end
       end
@@ -513,9 +596,9 @@ class RDoc::RubyLex
 
     @OP.def_rules("`") do
       |op, io|
-      if @lex_state == :EXPR_FNAME
-        @lex_state = :EXPR_END
-        Token(op)
+      if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
+        @lex_state = :EXPR_ARG
+        Token(TkId, op)
       else
         identify_string(op)
       end
@@ -525,6 +608,7 @@ class RDoc::RubyLex
       |op, io|
       if @lex_state == :EXPR_END
         @lex_state = :EXPR_BEG
+        @after_question = true
         Token(TkQUESTION)
       else
         ch = getc
@@ -534,17 +618,31 @@ class RDoc::RubyLex
           Token(TkQUESTION)
         else
           @lex_state = :EXPR_END
+          ch << getc if "\\" == ch
           Token(TkCHAR, "?#{ch}")
         end
       end
     end
 
-    @OP.def_rules("&", "&&", "|", "||") do
+    @OP.def_rules("&&", "||") do
       |op, io|
       @lex_state = :EXPR_BEG
       Token(op)
     end
 
+    @OP.def_rules("&", "|") do
+      |op, io|
+      case @lex_state
+      when :EXPR_FNAME, :EXPR_DOT
+        tk = Token(TkId, op)
+        @lex_state = :EXPR_ARG
+      else
+        tk = Token(op)
+        @lex_state = :EXPR_BEG
+      end
+      tk
+    end
+
     @OP.def_rules("+=", "-=", "*=", "**=",
                   "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do
       |op, io|
@@ -556,19 +654,22 @@ class RDoc::RubyLex
     @OP.def_rule("+@", proc{|op, io| @lex_state == :EXPR_FNAME}) do
       |op, io|
       @lex_state = :EXPR_ARG
-      Token(op)
+      Token(TkId, op)
     end
 
     @OP.def_rule("-@", proc{|op, io| @lex_state == :EXPR_FNAME}) do
       |op, io|
       @lex_state = :EXPR_ARG
-      Token(op)
+      Token(TkId, op)
     end
 
     @OP.def_rules("+", "-") do
       |op, io|
       catch(:RET) do
-        if @lex_state == :EXPR_ARG
+        if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
+          tk = Token(TkId, op)
+          @lex_state = :EXPR_ARG
+        elsif @lex_state == :EXPR_ARG
           if @space_seen and peek(0) =~ /[0-9]/
             throw :RET, identify_number(op)
           else
@@ -579,20 +680,21 @@ class RDoc::RubyLex
         else
           @lex_state = :EXPR_BEG
         end
-        Token(op)
+        tk = Token(op) unless tk
+        tk
       end
     end
 
-    @OP.def_rule(".") do
+    @OP.def_rules(".", "&.") do
       |op, io|
       @lex_state = :EXPR_BEG
       if peek(0) =~ /[0-9]/
         ungetc
         identify_number
       else
-        # for "obj.if" etc.
+        # for "obj.if" or "obj&.if" etc.
         @lex_state = :EXPR_DOT
-        Token(TkDOT)
+        Token(op)
       end
     end
 
@@ -639,7 +741,10 @@ class RDoc::RubyLex
 
     @OP.def_rule("/") do
       |op, io|
-      if @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
+      if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
+        @lex_state = :EXPR_ARG
+        Token(TkId, op)
+      elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID || @first_in_method_statement
         identify_string(op)
       elsif peek(0) == '='
         getc
@@ -655,8 +760,15 @@ class RDoc::RubyLex
 
     @OP.def_rules("^") do
       |op, io|
-      @lex_state = :EXPR_BEG
-      Token("^")
+      case @lex_state
+      when :EXPR_FNAME, :EXPR_DOT
+        tk = Token(TkId, op)
+        @lex_state = :EXPR_ARG
+      else
+        tk = Token(op)
+        @lex_state = :EXPR_BEG
+      end
+      tk
     end
 
     #       @OP.def_rules("^=") do
@@ -683,8 +795,14 @@ class RDoc::RubyLex
 
     @OP.def_rule("~") do
       |op, io|
-      @lex_state = :EXPR_BEG
-      Token("~")
+      case @lex_state
+      when :EXPR_FNAME, :EXPR_DOT
+        @lex_state = :EXPR_ARG
+        Token(TkId, op)
+      else
+        @lex_state = :EXPR_BEG
+        Token(op)
+      end
     end
 
     @OP.def_rule("~@", proc{|op, io| @lex_state == :EXPR_FNAME}) do
@@ -710,17 +828,18 @@ class RDoc::RubyLex
     @OP.def_rule("[]", proc{|op, io| @lex_state == :EXPR_FNAME}) do
       |op, io|
       @lex_state = :EXPR_ARG
-      Token("[]")
+      Token(TkId, op)
     end
 
     @OP.def_rule("[]=", proc{|op, io| @lex_state == :EXPR_FNAME}) do
       |op, io|
       @lex_state = :EXPR_ARG
-      Token("[]=")
+      Token(TkId, op)
     end
 
     @OP.def_rule("[") do
       |op, io|
+      text = nil
       @indent += 1
       if @lex_state == :EXPR_FNAME
         tk_c = TkfLBRACK
@@ -729,13 +848,25 @@ class RDoc::RubyLex
           tk_c = TkLBRACK
         elsif @lex_state == :EXPR_ARG && @space_seen
           tk_c = TkLBRACK
+        elsif @lex_state == :EXPR_DOT
+          if peek(0) == "]"
+            tk_c = TkIDENTIFIER
+            getc
+            if peek(0) == "="
+              text = "[]="
+            else
+              text = "[]"
+            end
+          else
+            tk_c = TkOp
+          end
         else
           tk_c = TkfLBRACK
         end
         @lex_state = :EXPR_BEG
       end
       @indent_stack.push tk_c
-      Token(tk_c)
+      Token(tk_c, text)
     end
 
     @OP.def_rule("{") do
@@ -753,23 +884,25 @@ class RDoc::RubyLex
 
     @OP.def_rule('\\') do
       |op, io|
-      if getc == "\n"
+      if peek(0) == "\n"
         @space_seen = true
         @continue = true
-        Token(TkSPACE)
-      else
-        ungetc
-        Token("\\")
+        @escaped_nl = true
       end
+      Token("\\")
     end
 
     @OP.def_rule('%') do
       |op, io|
-      if @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
+      if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
+        @lex_state = :EXPR_ARG
+        Token(TkId, op)
+      elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
         identify_quotation
       elsif peek(0) == '='
         getc
-        Token(TkOPASGN, :%)
+        @lex_state = :EXPR_BEG
+        Token(TkOPASGN, '%')
       elsif @lex_state == :EXPR_ARG and @space_seen and peek(0) !~ /\s/
         identify_quotation
       else
@@ -871,7 +1004,7 @@ class RDoc::RubyLex
 
     ungetc
 
-    if (ch == "!" || ch == "?") && token[0,1] =~ /\w/ && peek(0) != "="
+    if ((ch == "!" && peek(1) != "=") || ch == "?") && token[0,1] =~ /\w/
       token.concat getc
     end
 
@@ -944,44 +1077,63 @@ class RDoc::RubyLex
             @lex_state = :EXPR_END
           end
         end
+        if token_c.ancestors.include?(TkId) and peek(0) == ':' and !peek_match?(/^::/)
+          token.concat getc
+          token_c = TkSYMBOL
+        end
         return Token(token_c, token)
       end
     end
 
     if @lex_state == :EXPR_FNAME
       @lex_state = :EXPR_END
-      if peek(0) == '='
+      if peek(0) == '=' and peek(1) != '>'
         token.concat getc
       end
     elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_DOT ||
-          @lex_state == :EXPR_ARG
+          @lex_state == :EXPR_ARG || @lex_state == :EXPR_MID
       @lex_state = :EXPR_ARG
     else
       @lex_state = :EXPR_END
     end
 
     if token[0, 1] =~ /[A-Z]/
-      return Token(TkCONSTANT, token)
+      if token[-1] =~ /[!?]/
+        token_c = TkIDENTIFIER
+      else
+        token_c = TkCONSTANT
+      end
     elsif token[token.size - 1, 1] =~ /[!?]/
-      return Token(TkFID, token)
+      token_c = TkFID
+    else
+      token_c = TkIDENTIFIER
+    end
+    if peek(0) == ':' and !peek_match?(/^::/)
+      token.concat getc
+      return Token(TkSYMBOL, token)
     else
-      return Token(TkIDENTIFIER, token)
+      return Token(token_c, token)
     end
   end
 
-  def identify_here_document
+  def identify_here_document(op)
     ch = getc
+    start_token = op
     #    if lt = PERCENT_LTYPE[ch]
-    if ch == "-"
+    if ch == "-" or ch == "~"
+      start_token.concat ch
       ch = getc
       indent = true
     end
     if /['"`]/ =~ ch
+      start_token.concat ch
       user_quote = lt = ch
       quoted = ""
       while (c = getc) && c != lt
         quoted.concat c
       end
+      start_token.concat quoted
+      start_token.concat lt
     else
       user_quote = nil
       lt = '"'
@@ -989,57 +1141,38 @@ class RDoc::RubyLex
       while (c = getc) && c =~ /\w/
         quoted.concat c
       end
+      start_token.concat quoted
       ungetc
     end
 
-    ltback, @ltype = @ltype, lt
-    reserve = []
-    while ch = getc
-      reserve.push ch
-      if ch == "\\"
-        reserve.push ch = getc
-      elsif ch == "\n"
-        break
-      end
-    end
-
-    output_heredoc = reserve.join =~ /\A\r?\n\z/
+    @heredoc_queue << {
+      quoted: quoted,
+      lt: lt,
+      indent: indent,
+      started: false
+    }
+    @lex_state = :EXPR_END
+    Token(RDoc::RubyLex::TkHEREDOCBEG, start_token)
+  end
 
-    if output_heredoc then
-      doc = '<<'
-      doc << '-' if indent
-      doc << "#{user_quote}#{quoted}#{user_quote}\n"
-    else
-      doc = '"'
-    end
+  def identify_here_document_body(quoted, lt, indent)
+    ltback, @ltype = @ltype, lt
 
-    @current_readed = @readed
+    doc = ""
+    heredoc_end = nil
     while l = gets
       l = l.sub(/(:?\r)?\n\z/, "\n")
       if (indent ? l.strip : l.chomp) == quoted
+        heredoc_end = l
         break
       end
       doc << l
     end
+    raise Error, "Missing terminating #{quoted} for string" unless heredoc_end
 
-    if output_heredoc then
-      raise Error, "Missing terminating #{quoted} for string" unless l
-
-      doc << l.chomp
-    else
-      doc << '"'
-    end
-
-    @current_readed = @here_readed
-    @here_readed.concat reserve
-    while ch = reserve.pop
-      ungetc ch
-    end
-
-    token_class = output_heredoc ? RDoc::RubyLex::TkHEREDOC : Ltype2Token[lt]
     @ltype = ltback
-    @lex_state = :EXPR_END
-    Token(token_class, doc)
+    @lex_state = :EXPR_BEG
+    [Token(RDoc::RubyLex::TkHEREDOC, doc), heredoc_end]
   end
 
   def identify_quotation
@@ -1066,7 +1199,7 @@ class RDoc::RubyLex
 
     num = op
 
-    if peek(0) == "0" && peek(1) !~ /[.eE]/
+    if peek(0) == "0" && peek(1) !~ /[.eEri]/
       num << getc
 
       case peek(0)
@@ -1125,6 +1258,7 @@ class RDoc::RubyLex
     type = TkINTEGER
     allow_point = true
     allow_e = true
+    allow_ri = true
     non_digit = false
     while ch = getc
       num << ch
@@ -1154,8 +1288,25 @@ class RDoc::RubyLex
           num << getc
         end
         allow_e = false
+        allow_ri = false
         allow_point = false
         non_digit = ch
+      when allow_ri && "r"
+        if non_digit
+          raise Error, "trailing `#{non_digit}' in number"
+        end
+        type = TkRATIONAL
+        if peek(0) == 'i'
+          type = TkIMAGINARY
+          num << getc
+        end
+        break
+      when allow_ri && "i"
+        if non_digit && non_digit != "r"
+          raise Error, "trailing `#{non_digit}' in number"
+        end
+        type = TkIMAGINARY
+        break
       else
         if non_digit
           raise Error, "trailing `#{non_digit}' in number"
@@ -1174,10 +1325,10 @@ class RDoc::RubyLex
     @ltype = ltype
     @quoted = quoted
 
-    str = if ltype == quoted and %w[" ' /].include? ltype then
+    str = if ltype == quoted and %w[" ' / `].include? ltype and type.nil? then
             ltype.dup
           else
-            "%#{type or PERCENT_LTYPE.key ltype}#{PERCENT_PAREN_REV[quoted]||quoted}"
+            "%#{type}#{PERCENT_PAREN_REV[quoted]||quoted}"
           end
 
     subtype = nil
@@ -1191,21 +1342,21 @@ class RDoc::RubyLex
           break
         elsif @ltype != "'" && @ltype != "]" && @ltype != ":" and ch == "#"
           ch = getc
-          subtype = true
           if ch == "{" then
+            subtype = true
             str << ch << skip_inner_expression
             next
           else
             ungetc
           end
         elsif ch == '\\'
-          if %w[' /].include? @ltype then
+          case @ltype
+          when "'" then
             case ch = getc
-            when "\\", "\n", "'"
-            when @ltype
+            when "'", '\\' then
               str << ch
             else
-              ungetc
+              str << ch
             end
           else
             str << read_escape
@@ -1227,7 +1378,10 @@ class RDoc::RubyLex
         end
       end
 
-      if subtype
+      if peek(0) == ':' and !peek_match?(/^::/) and :EXPR_BEG == @lex_state and !@after_question
+        str.concat getc
+        return Token(TkSYMBOL, str)
+      elsif subtype
         Token(DLtype2Token[ltype], str)
       else
         Token(Ltype2Token[ltype], str)
diff --git a/lib/rdoc/ruby_token.rb b/lib/rdoc/ruby_token.rb
index d923e24b18..baab3fe836 100644
--- a/lib/rdoc/ruby_token.rb
+++ b/lib/rdoc/ruby_token.rb
@@ -26,7 +26,8 @@ module RDoc::RubyToken
     Symbol = Integer
   end
 
-  def set_token_position(line, char)
+  def set_token_position(seek, line, char)
+    @prev_seek = seek
     @prev_line_no = line
     @prev_char_no = char
   end
@@ -302,8 +303,8 @@ module RDoc::RubyToken
     [:TkIN,         TkKW,  "in",     :EXPR_BEG],
     [:TkDO,         TkKW,  "do",     :EXPR_BEG],
     [:TkRETURN,     TkKW,  "return", :EXPR_MID],
-    [:TkYIELD,      TkKW,  "yield",  :EXPR_END],
-    [:TkSUPER,      TkKW,  "super",  :EXPR_END],
+    [:TkYIELD,      TkKW,  "yield",  :EXPR_ARG],
+    [:TkSUPER,      TkKW,  "super",  :EXPR_ARG],
     [:TkSELF,       TkKW,  "self",   :EXPR_END],
     [:TkNIL,        TkKW,  "nil",    :EXPR_END],
     [:TkTRUE,       TkKW,  "true",   :EXPR_END],
@@ -316,11 +317,12 @@ module RDoc::RubyToken
     [:TkWHILE_MOD,  TkKW],
     [:TkUNTIL_MOD,  TkKW],
     [:TkALIAS,      TkKW,  "alias",    :EXPR_FNAME],
-    [:TkDEFINED,    TkKW,  "defined?", :EXPR_END],
+    [:TkDEFINED,    TkKW,  "defined?", :EXPR_ARG],
     [:TklBEGIN,     TkKW,  "BEGIN",    :EXPR_END],
     [:TklEND,       TkKW,  "END",      :EXPR_END],
     [:Tk__LINE__,   TkKW,  "__LINE__", :EXPR_END],
     [:Tk__FILE__,   TkKW,  "__FILE__", :EXPR_END],
+    [:Tk__ENCODING__,TkKW, "__ENCODING__", :EXPR_END],
 
     [:TkIDENTIFIER, TkId],
     [:TkFID,        TkId],
@@ -328,9 +330,13 @@ module RDoc::RubyToken
     [:TkCVAR,       TkId],
     [:TkIVAR,       TkId],
     [:TkCONSTANT,   TkId],
+    [:TkHEREDOCBEG, TkId],
+    [:TkHEREDOCEND, TkId],
 
     [:TkINTEGER,    TkVal],
     [:TkFLOAT,      TkVal],
+    [:TkRATIONAL,   TkVal],
+    [:TkIMAGINARY,  TkVal],
     [:TkSTRING,     TkVal],
     [:TkHEREDOC,    TkVal],
     [:TkXSTRING,    TkVal],
@@ -353,6 +359,7 @@ module RDoc::RubyToken
     [:TkNEQ,        TkOp,   "!="],
     [:TkGEQ,        TkOp,   ">="],
     [:TkLEQ,        TkOp,   "<="],
+    [:TkHASHROCKET, TkOp,   "=>"],
     [:TkANDOP,      TkOp,   "&&"],
     [:TkOROP,       TkOp,   "||"],
     [:TkMATCH,      TkOp,   "=~"],
@@ -367,6 +374,7 @@ module RDoc::RubyToken
     [:TkCOLON3,     TkOp,   '::'],
     #[:OPASGN,       TkOp],               # +=, -=  etc. #
     [:TkASSOC,      TkOp,   "=>"],
+    [:TkLAMBDA,     TkOp,   "->"],
     [:TkQUESTION,   TkOp,   "?"], #?
     [:TkCOLON,      TkOp,   ":"],        #:
 
@@ -394,6 +402,7 @@ module RDoc::RubyToken
 
     [:TkASSIGN,     Token,  "="],
     [:TkDOT,        Token,  "."],
+    [:TkSAFENAV,    Token,  "&."],
     [:TkLPAREN,     Token,  "("],  #(exp)
     [:TkLBRACK,     Token,  "["],  #[arry]
     [:TkLBRACE,     Token,  "{"],  #{hash}
diff --git a/lib/rdoc/test_case.rb b/lib/rdoc/test_case.rb
index aa7120cb59..594fb31272 100644
--- a/lib/rdoc/test_case.rb
+++ b/lib/rdoc/test_case.rb
@@ -6,7 +6,7 @@ rescue NoMethodError, Gem::LoadError
 end
 
 require 'minitest/autorun'
-require 'minitest/benchmark' if ENV['BENCHMARK']
+require 'minitest/benchmark' unless ENV['NOBENCHMARK']
 
 require 'fileutils'
 require 'pp'
diff --git a/lib/rdoc/token_stream.rb b/lib/rdoc/token_stream.rb
index b0035227fa..0cfa2f1384 100644
--- a/lib/rdoc/token_stream.rb
+++ b/lib/rdoc/token_stream.rb
@@ -36,17 +36,27 @@ module RDoc::TokenStream
               when RDoc::RubyToken::TkIVAR     then 'ruby-ivar'
               when RDoc::RubyToken::TkOp       then 'ruby-operator'
               when RDoc::RubyToken::TkId       then 'ruby-identifier'
+              when RDoc::RubyToken::TkREGEXP   then 'ruby-regexp'
+              when RDoc::RubyToken::TkDREGEXP  then 'ruby-regexp'
               when RDoc::RubyToken::TkNode     then 'ruby-node'
               when RDoc::RubyToken::TkCOMMENT  then 'ruby-comment'
-              when RDoc::RubyToken::TkREGEXP   then 'ruby-regexp'
+              when RDoc::RubyToken::TkXSTRING  then 'ruby-string'
               when RDoc::RubyToken::TkSTRING   then 'ruby-string'
               when RDoc::RubyToken::TkVal      then 'ruby-value'
               end
 
-      text = CGI.escapeHTML t.text
+      comment_with_nl = false
+      case t
+      when RDoc::RubyToken::TkRD_COMMENT, RDoc::RubyToken::TkHEREDOCEND
+        comment_with_nl = true if t.text =~ /\n$/
+        text = t.text.rstrip
+      else
+        text = t.text
+      end
+      text = CGI.escapeHTML text
 
       if style then
-        "<span class=\"#{style}\">#{text}</span>"
+        "<span class=\"#{style}\">#{text}</span>#{"\n" if comment_with_nl}"
       else
         text
       end