Initial support for the prism parser

Prism will be the parser in Ruby 3.3. We need to support 3.0+ so we will have to "dual boot" both parsers.

Todo:

- LexAll to support Prism lex output
- Add tests that exercise both Ripper and prism codepaths on CI
- Handle ruby/prism#1972 in `ripper_errors.rb`
- Update docs to not mention Ripper explicitly
- Consider different/cleaner APIs for separating out Ripper and Prism

Author: schneems

Gemfile

@@ -12,3 +12,4 @@ gem "standard"
 gem "ruby-prof"
 
 gem "benchmark-ips"
+gem "prism"

Gemfile.lock

@@ -7,33 +7,34 @@ GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.2)
-    benchmark-ips (2.9.2)
-    diff-lcs (1.4.4)
+    benchmark-ips (2.12.0)
+    diff-lcs (1.5.0)
     json (2.7.0)
     language_server-protocol (3.17.0.3)
     lint_roller (1.1.0)
     parallel (1.23.0)
     parser (3.2.2.4)
       ast (~> 2.4.1)
       racc
+    prism (0.18.0)
     racc (1.7.3)
     rainbow (3.1.1)
     rake (12.3.3)
     regexp_parser (2.8.3)
     rexml (3.2.6)
-    rspec (3.10.0)
-      rspec-core (~> 3.10.0)
-      rspec-expectations (~> 3.10.0)
-      rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.0)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.0)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.2)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.3)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.0)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.6)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-support (3.10.0)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.1)
     rubocop (1.57.2)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
@@ -50,9 +51,9 @@ GEM
     rubocop-performance (1.19.1)
       rubocop (>= 1.7.0, < 2.0)
       rubocop-ast (>= 0.4.0)
-    ruby-prof (1.4.3)
+    ruby-prof (1.6.3)
     ruby-progressbar (1.13.0)
-    stackprof (0.2.16)
+    stackprof (0.2.25)
     standard (1.32.1)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.0)
@@ -72,6 +73,7 @@ PLATFORMS
 
 DEPENDENCIES
   benchmark-ips
+  prism
   rake (~> 12.0)
   rspec (~> 3.0)
   ruby-prof
@@ -80,4 +82,4 @@ DEPENDENCIES
   syntax_suggest!
 
 BUNDLED WITH
-   2.3.14
+   2.4.21

lib/syntax_suggest/api.rb

@@ -5,7 +5,22 @@
 require "tmpdir"
 require "stringio"
 require "pathname"
-require "ripper"
+
+# rubocop:disable Style/IdenticalConditionalBranches
+if ENV["SYNTAX_SUGGEST_DISABLE_PRISM"] # For testing dual ripper/prism support
+  require "ripper"
+else
+  # TODO remove require
+  # Allow both to be loaded to enable more atomic commits
+  require "ripper"
+  begin
+    require "prism"
+  rescue LoadError
+    require "ripper"
+  end
+end
+# rubocop:enable Style/IdenticalConditionalBranches
+
 require "timeout"
 
 module SyntaxSuggest
@@ -16,6 +31,14 @@ module SyntaxSuggest
   class Error < StandardError; end
   TIMEOUT_DEFAULT = ENV.fetch("SYNTAX_SUGGEST_TIMEOUT", 1).to_i
 
+  # SyntaxSuggest.use_prism_parser? [Private]
+  #
+  # Tells us if the prism parser is available for use
+  # or if we should fallback to `Ripper`
+  def self.use_prism_parser?
+    defined?(Prism)
+  end
+
   # SyntaxSuggest.handle_error [Public]
   #
   # Takes a `SyntaxError` exception, uses the
@@ -129,11 +152,20 @@ def self.valid_without?(without_lines:, code_lines:)
   # SyntaxSuggest.invalid? [Private]
   #
   # Opposite of `SyntaxSuggest.valid?`
-  def self.invalid?(source)
-    source = source.join if source.is_a?(Array)
-    source = source.to_s
+  if defined?(Prism)
+    def self.invalid?(source)
+      source = source.join if source.is_a?(Array)
+      source = source.to_s
+
+      Prism.parse(source).failure?
+    end
+  else
+    def self.invalid?(source)
+      source = source.join if source.is_a?(Array)
+      source = source.to_s
 
-    Ripper.new(source).tap(&:parse).error?
+      Ripper.new(source).tap(&:parse).error?
+    end
   end
 
   # SyntaxSuggest.valid? [Private]
@@ -191,7 +223,9 @@ def self.valid?(source)
 require_relative "code_line"
 require_relative "code_block"
 require_relative "block_expand"
-require_relative "ripper_errors"
+if !SyntaxSuggest.use_prism_parser?
+  require_relative "ripper_errors"
+end
 require_relative "priority_queue"
 require_relative "unvisited_lines"
 require_relative "around_block_scan"

lib/syntax_suggest/explain_syntax.rb

@@ -3,6 +3,16 @@
 require_relative "left_right_lex_count"
 
 module SyntaxSuggest
+  class GetParseErrors
+    def self.errors(source)
+      if SyntaxSuggest.use_prism_parser?
+        Prism.parse(source).errors.map(&:message)
+      else
+        RipperErrors.new(source).call.errors
+      end
+    end
+  end
+
   # Explains syntax errors based on their source
   #
   # example:
@@ -94,7 +104,7 @@ def why(miss)
     # on the original ripper error messages
     def errors
       if missing.empty?
-        return RipperErrors.new(@code_lines.map(&:original).join).call.errors
+        return GetParseErrors.errors(@code_lines.map(&:original).join)
       end
 
       missing.map { |miss| why(miss) }

lib/syntax_suggest/lex_all.rb

@@ -11,26 +11,40 @@ class LexAll
     include Enumerable
 
     def initialize(source:, source_lines: nil)
-      @lex = Ripper::Lexer.new(source, "-", 1).parse.sort_by(&:pos)
-      lineno = @lex.last.pos.first + 1
+      @lex = self.class.lex(source, 1)
+      lineno = @lex.last[0][0] + 1
       source_lines ||= source.lines
       last_lineno = source_lines.length
 
       until lineno >= last_lineno
         lines = source_lines[lineno..]
 
         @lex.concat(
-          Ripper::Lexer.new(lines.join, "-", lineno + 1).parse.sort_by(&:pos)
+          self.class.lex(lines.join, lineno + 1)
         )
-        lineno = @lex.last.pos.first + 1
+
+        lineno = @lex.last[0].first + 1
       end
 
       last_lex = nil
       @lex.map! { |elem|
-        last_lex = LexValue.new(elem.pos.first, elem.event, elem.tok, elem.state, last_lex)
+        last_lex = LexValue.new(elem[0].first, elem[1], elem[2], elem[3], last_lex)
       }
     end
 
+    # rubocop:disable Style/IdenticalConditionalBranches
+    if SyntaxSuggest.use_prism_parser?
+      def self.lex(source, line_number)
+        # Prism.lex_compat(source, line: line_number).value.sort_by {|values| values[0] }
+        Ripper::Lexer.new(source, "-", line_number).parse.sort_by(&:pos)
+      end
+    else
+      def self.lex(source, line_number)
+        Ripper::Lexer.new(source, "-", line_number).parse.sort_by(&:pos)
+      end
+    end
+    # rubocop:enable Style/IdenticalConditionalBranches
+
     def to_a
       @lex
     end

spec/unit/explain_syntax_spec.rb

@@ -14,7 +14,11 @@ module SyntaxSuggest
       ).call
 
       expect(explain.missing).to eq([])
-      expect(explain.errors.join).to include("unterminated string")
+      if SyntaxSuggest.use_prism_parser?
+        expect(explain.errors.join).to include("Expected a closing delimiter for the interpolated string")
+      else
+        expect(explain.errors.join).to include("unterminated string")
+      end
     end
 
     it "handles %w[]" do
@@ -191,7 +195,7 @@ def meow
       ).call
 
       expect(explain.missing).to eq([])
-      expect(explain.errors).to eq(RipperErrors.new(source).call.errors)
+      expect(explain.errors).to eq(GetParseErrors.errors(source))
     end
 
     it "handles an unexpected rescue" do