diff options
Diffstat (limited to 'lib/rdoc/ruby_lex.rb')
-rw-r--r-- | lib/rdoc/ruby_lex.rb | 23 |
1 files changed, 11 insertions, 12 deletions
diff --git a/lib/rdoc/ruby_lex.rb b/lib/rdoc/ruby_lex.rb index fe289fb1a2..cbe3ec9061 100644 --- a/lib/rdoc/ruby_lex.rb +++ b/lib/rdoc/ruby_lex.rb @@ -92,9 +92,9 @@ class RDoc::RubyLex end def inspect # :nodoc: - "#<%s:0x%x lex_state %p space_seen %p>" % [ + "#<%s:0x%x pos %d lex_state %p space_seen %p>" % [ self.class, object_id, - @lex_state, @space_seen, + @io.pos, @lex_state, @space_seen, ] end @@ -149,6 +149,7 @@ class RDoc::RubyLex else @char_no += 1 end + c end @@ -674,7 +675,7 @@ class RDoc::RubyLex tk_c = TkLPAREN end @indent_stack.push tk_c - Token(tk_c) + Token tk_c end @OP.def_rule("[]", proc{|op, io| @lex_state == EXPR_FNAME}) do @@ -822,6 +823,12 @@ class RDoc::RubyLex end end + IDENT_RE = if defined? Encoding then + /[\w\u0080-\uFFFF]/u + else + /[\w\x80-\xFF]/ + end + def identify_identifier token = "" if peek(0) =~ /[$@]/ @@ -831,15 +838,7 @@ class RDoc::RubyLex end end - # HACK to avoid a warning the regexp is hidden behind an eval - # HACK need a better way to detect oniguruma - @identifier_re ||= if defined? Encoding then - eval '/[\p{Alnum}_]/u' - else - eval '/[\w\x80-\xff]/' - end - - while (ch = getc) =~ @identifier_re + while (ch = getc) =~ IDENT_RE do print " :#{ch}: " if RDoc::RubyLex.debug? token.concat ch end |