[ruby/syntax_suggest] Initial support for the prism parser

Prism will be the parser in Ruby 3.3. We need to support 3.0+ so we will have to "dual boot" both parsers.

Todo:

- LexAll to support Prism lex output
- Add tests that exercise both Ripper and prism codepaths on CI
- Handle https://github.com/ruby/prism/issues/1972 in `ripper_errors.rb`
- Update docs to not mention Ripper explicitly
- Consider different/cleaner APIs for separating out Ripper and Prism

https://github.com/ruby/syntax_suggest/commit/a7d6991cc4

3 files changed, 70 insertions, 12 deletions

diff --git a/lib/syntax_suggest/api.rb b/lib/syntax_suggest/api.rb
index 74e53c2563..e8b39f29f7 100644
--- a/lib/syntax_suggest/api.rb
+++ b/lib/syntax_suggest/api.rb
@@ -5,7 +5,22 @@ require_relative "version"
 require "tmpdir"
 require "stringio"
 require "pathname"
-require "ripper"
+
+# rubocop:disable Style/IdenticalConditionalBranches
+if ENV["SYNTAX_SUGGEST_DISABLE_PRISM"] # For testing dual ripper/prism support
+  require "ripper"
+else
+  # TODO remove require
+  # Allow both to be loaded to enable more atomic commits
+  require "ripper"
+  begin
+    require "prism"
+  rescue LoadError
+    require "ripper"
+  end
+end
+# rubocop:enable Style/IdenticalConditionalBranches
+
 require "timeout"
 
 module SyntaxSuggest
@@ -16,6 +31,14 @@ module SyntaxSuggest
   class Error < StandardError; end
   TIMEOUT_DEFAULT = ENV.fetch("SYNTAX_SUGGEST_TIMEOUT", 1).to_i
 
+  # SyntaxSuggest.use_prism_parser? [Private]
+  #
+  # Tells us if the prism parser is available for use
+  # or if we should fallback to `Ripper`
+  def self.use_prism_parser?
+    defined?(Prism)
+  end
+
   # SyntaxSuggest.handle_error [Public]
   #
   # Takes a `SyntaxError` exception, uses the
@@ -129,11 +152,20 @@ module SyntaxSuggest
   # SyntaxSuggest.invalid? [Private]
   #
   # Opposite of `SyntaxSuggest.valid?`
-  def self.invalid?(source)
-    source = source.join if source.is_a?(Array)
-    source = source.to_s
+  if defined?(Prism)
+    def self.invalid?(source)
+      source = source.join if source.is_a?(Array)
+      source = source.to_s
+
+      Prism.parse(source).failure?
+    end
+  else
+    def self.invalid?(source)
+      source = source.join if source.is_a?(Array)
+      source = source.to_s
 
-    Ripper.new(source).tap(&:parse).error?
+      Ripper.new(source).tap(&:parse).error?
+    end
   end
 
   # SyntaxSuggest.valid? [Private]
@@ -191,7 +223,9 @@ require_relative "lex_all"
 require_relative "code_line"
 require_relative "code_block"
 require_relative "block_expand"
-require_relative "ripper_errors"
+if !SyntaxSuggest.use_prism_parser?
+  require_relative "ripper_errors"
+end
 require_relative "priority_queue"
 require_relative "unvisited_lines"
 require_relative "around_block_scan"
diff --git a/lib/syntax_suggest/explain_syntax.rb b/lib/syntax_suggest/explain_syntax.rb
index 142ed2e269..8de962c157 100644
--- a/lib/syntax_suggest/explain_syntax.rb
+++ b/lib/syntax_suggest/explain_syntax.rb
@@ -3,6 +3,16 @@
 require_relative "left_right_lex_count"
 
 module SyntaxSuggest
+  class GetParseErrors
+    def self.errors(source)
+      if SyntaxSuggest.use_prism_parser?
+        Prism.parse(source).errors.map(&:message)
+      else
+        RipperErrors.new(source).call.errors
+      end
+    end
+  end
+
   # Explains syntax errors based on their source
   #
   # example:
@@ -94,7 +104,7 @@ module SyntaxSuggest
     # on the original ripper error messages
     def errors
       if missing.empty?
-        return RipperErrors.new(@code_lines.map(&:original).join).call.errors
+        return GetParseErrors.errors(@code_lines.map(&:original).join)
       end
 
       missing.map { |miss| why(miss) }
diff --git a/lib/syntax_suggest/lex_all.rb b/lib/syntax_suggest/lex_all.rb
index 962d0d5a93..b197118774 100644
--- a/lib/syntax_suggest/lex_all.rb
+++ b/lib/syntax_suggest/lex_all.rb
@@ -11,8 +11,8 @@ module SyntaxSuggest
     include Enumerable
 
     def initialize(source:, source_lines: nil)
-      @lex = Ripper::Lexer.new(source, "-", 1).parse.sort_by(&:pos)
-      lineno = @lex.last.pos.first + 1
+      @lex = self.class.lex(source, 1)
+      lineno = @lex.last[0][0] + 1
       source_lines ||= source.lines
       last_lineno = source_lines.length
 
@@ -20,17 +20,31 @@ module SyntaxSuggest
         lines = source_lines[lineno..]
 
         @lex.concat(
-          Ripper::Lexer.new(lines.join, "-", lineno + 1).parse.sort_by(&:pos)
+          self.class.lex(lines.join, lineno + 1)
         )
-        lineno = @lex.last.pos.first + 1
+
+        lineno = @lex.last[0].first + 1
       end
 
       last_lex = nil
       @lex.map! { |elem|
-        last_lex = LexValue.new(elem.pos.first, elem.event, elem.tok, elem.state, last_lex)
+        last_lex = LexValue.new(elem[0].first, elem[1], elem[2], elem[3], last_lex)
       }
     end
 
+    # rubocop:disable Style/IdenticalConditionalBranches
+    if SyntaxSuggest.use_prism_parser?
+      def self.lex(source, line_number)
+        # Prism.lex_compat(source, line: line_number).value.sort_by {|values| values[0] }
+        Ripper::Lexer.new(source, "-", line_number).parse.sort_by(&:pos)
+      end
+    else
+      def self.lex(source, line_number)
+        Ripper::Lexer.new(source, "-", line_number).parse.sort_by(&:pos)
+      end
+    end
+    # rubocop:enable Style/IdenticalConditionalBranches
+
     def to_a
       @lex
     end