Import RDoc 2.5

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@27147 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 28 insertions, 9 deletions

diff --git a/lib/rdoc/tokenstream.rb b/lib/rdoc/tokenstream.rb
index 0a1eb9130b..b1e86543f7 100644
--- a/lib/rdoc/tokenstream.rb
+++ b/lib/rdoc/tokenstream.rb
@@ -9,25 +9,44 @@ module RDoc; end
 
 module RDoc::TokenStream
 
-  def token_stream
-    @token_stream
+  ##
+  # Adds +tokens+ to the collected tokens
+
+  def add_tokens(*tokens)
+    tokens.flatten.each { |token| @token_stream << token }
   end
 
-  def start_collecting_tokens
+  alias add_token add_tokens
+
+  ##
+  # Starts collecting tokens
+
+  def collect_tokens
     @token_stream = []
   end
 
-  def add_token(tk)
-    @token_stream << tk
-  end
+  alias start_collecting_tokens collect_tokens
 
-  def add_tokens(tks)
-    tks.each  {|tk| add_token(tk)}
-  end
+  ##
+  # Remove the last token from the collected tokens
 
   def pop_token
     @token_stream.pop
   end
 
+  ##
+  # Current token stream
+
+  def token_stream
+    @token_stream
+  end
+
+  ##
+  # Returns a string representation of the token stream
+
+  def tokens_to_s
+    token_stream.map { |token| token.text }.join ''
+  end
+
 end