Colorize error characters

* lib/irb/color.rb (IRB::Color.scan): ignore "incomplete end of
  input" error only, to colorize invalid characters, e.g., control
  characters, and invalid symbols, as errors.

2 files changed, 20 insertions, 37 deletions

diff --git a/lib/irb/color.rb b/lib/irb/color.rb
index 5e58c2509f..c2d0084446 100644
--- a/lib/irb/color.rb
+++ b/lib/irb/color.rb
@@ -131,28 +131,23 @@ module IRB # :nodoc:
       private
 
       def scan(code, detect_compile_error:)
-        if detect_compile_error
-          pos = [1, 0]
-
-          Ripper::Lexer.new(code).scan.each do |elem|
-            str = elem.tok
-            next if ([elem.pos[0], elem.pos[1] + str.bytesize] <=> pos) <= 0
-
-            str.each_line do |line|
-              if line.end_with?("\n")
-                pos[0] += 1
-                pos[1] = 0
-              else
-                pos[1] += line.bytesize
-              end
-            end
+        pos = [1, 0]
 
-            yield(elem.event, str, elem.state)
-          end
-        else
-          ParseErrorLexer.new(code).parse.sort_by(&:pos).each do |elem|
-            yield(elem.event, elem.tok, elem.state)
+        Ripper::Lexer.new(code).scan.each do |elem|
+          str = elem.tok
+          next if !detect_compile_error and elem.message&.end_with?("meets end of file")
+          next if ([elem.pos[0], elem.pos[1] + str.bytesize] <=> pos) <= 0
+
+          str.each_line do |line|
+            if line.end_with?("\n")
+              pos[0] += 1
+              pos[1] = 0
+            else
+              pos[1] += line.bytesize
+            end
           end
+
+          yield(elem.event, str, elem.state)
         end
       end
 
@@ -171,15 +166,6 @@ module IRB # :nodoc:
       end
     end
 
-    class ParseErrorLexer < Ripper::Lexer
-      if method_defined?(:token)
-        def on_parse_error(mesg)
-          @buf.push Elem.new([lineno(), column()], __callee__, token(), state())
-        end
-      end
-    end
-    private_constant :ParseErrorLexer
-
     # A class to manage a state to know whether the current token is for Symbol or not.
     class SymbolState
       def initialize
diff --git a/test/irb/test_color.rb b/test/irb/test_color.rb
index ebae790b71..d327d617fa 100644
--- a/test/irb/test_color.rb
+++ b/test/irb/test_color.rb
@@ -41,6 +41,7 @@ module TestIRB
         "'a\nb'" => "#{RED}'#{CLEAR}#{RED}a#{CLEAR}\n#{RED}b#{CLEAR}#{RED}'#{CLEAR}",
         "4.5.6" => "#{MAGENTA}#{BOLD}4.5#{CLEAR}#{RED}#{REVERSE}.6#{CLEAR}",
         "[1]]]" => "[1]]]",
+        "\e[0m\n" => "#{RED}#{REVERSE}^[#{CLEAR}[#{BLUE}#{BOLD}0#{CLEAR}m\n",
         "%w[a b]" => "#{RED}%w[#{CLEAR}#{RED}a#{CLEAR} #{RED}b#{CLEAR}#{RED}]#{CLEAR}",
         "%i[c d]" => "#{RED}%i[#{CLEAR}#{RED}c#{CLEAR} #{RED}d#{CLEAR}#{RED}]#{CLEAR}",
         "{'a': 1}" => "{#{RED}'#{CLEAR}#{RED}a#{CLEAR}#{RED}':#{CLEAR} #{BLUE}#{BOLD}1#{CLEAR}}",
@@ -52,6 +53,8 @@ module TestIRB
         "[__FILE__, __LINE__]" => "[#{CYAN}#{BOLD}__FILE__#{CLEAR}, #{CYAN}#{BOLD}__LINE__#{CLEAR}]",
         ":self" => "#{YELLOW}:#{CLEAR}#{YELLOW}self#{CLEAR}",
         ":class" => "#{YELLOW}:#{CLEAR}#{YELLOW}class#{CLEAR}",
+        ":@1" => "#{YELLOW}:#{CLEAR}#{RED}#{REVERSE}@1#{CLEAR}",
+        "@@1" => "#{RED}#{REVERSE}@@1#{CLEAR}",
         "[:end, 2]" => "[#{YELLOW}:#{CLEAR}#{YELLOW}end#{CLEAR}, #{BLUE}#{BOLD}2#{CLEAR}]",
         "[:>, 3]" => "[#{YELLOW}:#{CLEAR}#{YELLOW}>#{CLEAR}, #{BLUE}#{BOLD}3#{CLEAR}]",
         ":Hello ? world : nil" => "#{YELLOW}:#{CLEAR}#{YELLOW}Hello#{CLEAR} ? world : #{CYAN}#{BOLD}nil#{CLEAR}",
@@ -76,12 +79,9 @@ module TestIRB
     end
 
     def test_colorize_code_complete_true
-      # `complete: true` behaviors. Warn compile_error.
+      # `complete: true` behaviors. Warn end-of-file.
       {
-        "\e[0m\n" => "#{RED}#{REVERSE}^[#{CLEAR}[#{BLUE}#{BOLD}0#{CLEAR}m\n",
         "'foo' + 'bar" => "#{RED}'#{CLEAR}#{RED}foo#{CLEAR}#{RED}'#{CLEAR} + #{RED}'#{CLEAR}#{RED}#{REVERSE}bar#{CLEAR}",
-        ":@1" => "#{YELLOW}:#{CLEAR}#{RED}#{REVERSE}@1#{CLEAR}",
-        "@@1" => "#{RED}#{REVERSE}@@1#{CLEAR}",
       }.each do |code, result|
         actual = with_term { IRB::Color.colorize_code(code, complete: true) }
         assert_equal(result, actual, "Case: colorize_code(#{code.dump}, complete: true)\nResult: #{humanized_literal(actual)}")
@@ -89,12 +89,9 @@ module TestIRB
     end
 
     def test_colorize_code_complete_false
-      # `complete: false` behaviors. Do not warn compile_error.
+      # `complete: false` behaviors. Do not warn end-of-file
       {
-        "\e[0m\n" => "#{CLEAR}\n",
         "'foo' + 'bar" => "#{RED}'#{CLEAR}#{RED}foo#{CLEAR}#{RED}'#{CLEAR} + #{RED}'#{CLEAR}#{RED}bar#{CLEAR}",
-        ":@1" => "#{YELLOW}:#{CLEAR}#{YELLOW}@1#{CLEAR}",
-        "@@1" => "@@1",
       }.each do |code, result|
         actual = with_term { IRB::Color.colorize_code(code, complete: false) }
         assert_equal(result, actual, "Case: colorize_code(#{code.dump}, complete: false)\nResult: #{humanized_literal(actual)}")