From 2ef9c50c6e405717d06362787c4549ca4f1c6485 Mon Sep 17 00:00:00 2001
From: drbrain <drbrain@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>
Date: Mon, 20 Dec 2010 03:22:49 +0000
Subject: Import RDoc 3

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@30249 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
---
 lib/rdoc/ruby_lex.rb | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

(limited to 'lib/rdoc/ruby_lex.rb')

diff --git a/lib/rdoc/ruby_lex.rb b/lib/rdoc/ruby_lex.rb
index fe289fb1a2..cbe3ec9061 100644
--- a/lib/rdoc/ruby_lex.rb
+++ b/lib/rdoc/ruby_lex.rb
@@ -92,9 +92,9 @@ class RDoc::RubyLex
   end
 
   def inspect # :nodoc:
-    "#<%s:0x%x lex_state %p space_seen %p>" % [
+    "#<%s:0x%x pos %d lex_state %p space_seen %p>" % [
       self.class, object_id,
-      @lex_state, @space_seen,
+      @io.pos, @lex_state, @space_seen,
     ]
   end
 
@@ -149,6 +149,7 @@ class RDoc::RubyLex
     else
       @char_no += 1
     end
+
     c
   end
 
@@ -674,7 +675,7 @@ class RDoc::RubyLex
         tk_c = TkLPAREN
       end
       @indent_stack.push tk_c
-      Token(tk_c)
+      Token tk_c
     end
 
     @OP.def_rule("[]", proc{|op, io| @lex_state == EXPR_FNAME}) do
@@ -822,6 +823,12 @@ class RDoc::RubyLex
     end
   end
 
+  IDENT_RE = if defined? Encoding then
+               /[\w\u0080-\uFFFF]/u
+             else
+               /[\w\x80-\xFF]/
+             end
+
   def identify_identifier
     token = ""
     if peek(0) =~ /[$@]/
@@ -831,15 +838,7 @@ class RDoc::RubyLex
       end
     end
 
-    # HACK to avoid a warning the regexp is hidden behind an eval
-    # HACK need a better way to detect oniguruma
-    @identifier_re ||= if defined? Encoding then
-                         eval '/[\p{Alnum}_]/u'
-                       else
-                         eval '/[\w\x80-\xff]/'
-                       end
-
-    while (ch = getc) =~ @identifier_re
+    while (ch = getc) =~ IDENT_RE do
       print " :#{ch}: " if RDoc::RubyLex.debug?
       token.concat ch
     end
-- 
cgit v1.2.3