[ruby/syntax_suggest] Support lexing with Prism

https://github.com/ruby/syntax_suggest/commit/7f4176a914

4 files changed, 33 insertions, 19 deletions

diff --git a/lib/syntax_suggest/api.rb b/lib/syntax_suggest/api.rb
index e8b39f29f7..59f47ee4f3 100644
--- a/lib/syntax_suggest/api.rb
+++ b/lib/syntax_suggest/api.rb
@@ -5,23 +5,27 @@ require_relative "version"
 require "tmpdir"
 require "stringio"
 require "pathname"
+require "timeout"
 
-# rubocop:disable Style/IdenticalConditionalBranches
-if ENV["SYNTAX_SUGGEST_DISABLE_PRISM"] # For testing dual ripper/prism support
-  require "ripper"
+# We need Ripper loaded for `Prism.lex_compat` even if we're using Prism
+# for lexing and parsing
+require "ripper"
+
+# Prism is the new parser, replacing Ripper
+#
+# We need to "dual boot" both for now because syntax_suggest
+# supports older rubies that do not ship with syntax suggest.
+#
+# We also need the ability to control loading of this library
+# so we can test that both modes work correctly in CI.
+if (value = ENV["SYNTAX_SUGGEST_DISABLE_PRISM"])
+  warn "Skipping loading prism due to SYNTAX_SUGGEST_DISABLE_PRISM=#{value}"
 else
-  # TODO remove require
-  # Allow both to be loaded to enable more atomic commits
-  require "ripper"
   begin
     require "prism"
   rescue LoadError
-    require "ripper"
   end
 end
-# rubocop:enable Style/IdenticalConditionalBranches
-
-require "timeout"
 
 module SyntaxSuggest
   # Used to indicate a default value that cannot
diff --git a/lib/syntax_suggest/code_line.rb b/lib/syntax_suggest/code_line.rb
index a20f34afa4..58197e95d0 100644
--- a/lib/syntax_suggest/code_line.rb
+++ b/lib/syntax_suggest/code_line.rb
@@ -180,12 +180,19 @@ module SyntaxSuggest
     #     EOM
     #     expect(lines.first.trailing_slash?).to eq(true)
     #
-    def trailing_slash?
-      last = @lex.last
-      return false unless last
-      return false unless last.type == :on_sp
+    if SyntaxSuggest.use_prism_parser?
+      def trailing_slash?
+        last = @lex.last
+        last&.type == :on_tstring_end
+      end
+    else
+      def trailing_slash?
+        last = @lex.last
+        return false unless last
+        return false unless last.type == :on_sp
 
-      last.token == TRAILING_SLASH
+        last.token == TRAILING_SLASH
+      end
     end
 
     # Endless method detection
diff --git a/lib/syntax_suggest/lex_all.rb b/lib/syntax_suggest/lex_all.rb
index b197118774..e9509c4c3e 100644
--- a/lib/syntax_suggest/lex_all.rb
+++ b/lib/syntax_suggest/lex_all.rb
@@ -32,18 +32,15 @@ module SyntaxSuggest
       }
     end
 
-    # rubocop:disable Style/IdenticalConditionalBranches
     if SyntaxSuggest.use_prism_parser?
       def self.lex(source, line_number)
-        # Prism.lex_compat(source, line: line_number).value.sort_by {|values| values[0] }
-        Ripper::Lexer.new(source, "-", line_number).parse.sort_by(&:pos)
+        Prism.lex_compat(source, line: line_number).value.sort_by { |values| values[0] }
       end
     else
       def self.lex(source, line_number)
         Ripper::Lexer.new(source, "-", line_number).parse.sort_by(&:pos)
       end
     end
-    # rubocop:enable Style/IdenticalConditionalBranches
 
     def to_a
       @lex
diff --git a/spec/syntax_suggest/unit/api_spec.rb b/spec/syntax_suggest/unit/api_spec.rb
index 079a91e46d..e900b9e10b 100644
--- a/spec/syntax_suggest/unit/api_spec.rb
+++ b/spec/syntax_suggest/unit/api_spec.rb
@@ -8,6 +8,12 @@ end
 
 module SyntaxSuggest
   RSpec.describe "Top level SyntaxSuggest api" do
+    it "doesn't load prism if env var is set" do
+      skip("SYNTAX_SUGGEST_DISABLE_PRISM not set") unless ENV["SYNTAX_SUGGEST_DISABLE_PRISM"]
+
+      expect(SyntaxSuggest.use_prism_parser?).to be_falsey
+    end
+
     it "has a `handle_error` interface" do
       fake_error = Object.new
       def fake_error.message