Get YARP building and parsing some basic calls

This works:

    $ rake

    $ bin/natalie --ast -e "p 1+2"
    s(:block, s(:block, s(:call, nil, :p, s(:call, s(:lit, 1), :+, s(:lit, 2)))))

    $ bin/natalie -e "p 1+2"
    3

.gitmodules

@@ -10,3 +10,6 @@
 [submodule "ext/zlib"]
 	path = ext/zlib
 	url = https://github.com/madler/zlib
+[submodule "ext/yarp"]
+	path = ext/yarp
+	url = https://github.com/ruby/yarp

Rakefile

@@ -13,12 +13,12 @@ task :build do
 end
 
 desc 'Build Natalie with no optimization and all warnings (default)'
-task build_debug: %i[set_build_debug libnatalie parser_c_ext ctags] do
+task build_debug: %i[set_build_debug libnatalie parser_c_ext yarp_c_ext ctags] do
   puts 'Build mode: debug'
 end
 
 desc 'Build Natalie with release optimizations enabled and warnings off'
-task build_release: %i[set_build_release libnatalie parser_c_ext] do
+task build_release: %i[set_build_release libnatalie parser_c_ext yarp_c_ext] do
   puts 'Build mode: release'
 end
 
@@ -36,6 +36,9 @@ task :clean do
   rm_rf 'build/libnatalie_parser.a'
   rm_rf "build/natalie_parser.#{SO_EXT}"
   rm_rf 'build/natalie_parser.bundle'
+  rm_rf 'build/yarp'
+  rm_rf "build/librubyparser.a"
+  rm_rf "build/librubyparser.#{SO_EXT}"
   rm_rf Rake::FileList['build/*.o']
 end
 
@@ -99,6 +102,9 @@ task bootstrap: [:build, 'bin/nat']
 desc 'Build MRI C Extension for the Natalie Parser'
 task parser_c_ext: ["build/natalie_parser.#{SO_EXT}", "build/libnatalie_parser.#{SO_EXT}"]
 
+desc 'Build MRI C Extension for YARP'
+task yarp_c_ext: ["build/librubyparser.#{SO_EXT}"]
+
 desc 'Show line counts for the project'
 task :cloc do
   sh 'cloc include lib src test'
@@ -257,6 +263,8 @@ task libnatalie: [
   'build/zlib/libz.a',
   'build/onigmo/lib/libonigmo.a',
   'build/libnatalie_parser.a',
+  "build/librubyparser.a",
+  "build/librubyparser.#{SO_EXT}",
   'build/generated/numbers.rb',
   :primary_objects,
   :ruby_objects,
@@ -415,6 +423,21 @@ file "build/libnatalie_parser.#{SO_EXT}" => "build/natalie_parser.#{SO_EXT}" do
   sh "cp #{build_dir}/ext/natalie_parser/natalie_parser.#{SO_EXT} #{File.expand_path('build', __dir__)}/libnatalie_parser.#{SO_EXT}"
 end
 
+file "build/librubyparser.a" do
+  build_dir = File.expand_path('build/yarp', __dir__)
+  rm_rf build_dir
+  cp_r 'ext/yarp', build_dir
+  sh <<-SH
+    cd #{build_dir} && \
+    bundle install && \
+    rake compile && \
+    cp #{build_dir}/build/librubyparser.a #{File.expand_path('build', __dir__)} && \
+    cp #{build_dir}/build/librubyparser.#{SO_EXT} #{File.expand_path('build', __dir__)}
+  SH
+end
+
+file "build/librubyparser.#{SO_EXT}" => ["build/librubyparser.a"]
+
 task :tidy_internal do
   sh "clang-tidy --warnings-as-errors='*' #{PRIMARY_SOURCES.exclude('src/dtoa.c')}"
 end
@@ -480,5 +503,6 @@ def include_paths
     File.expand_path('build', __dir__),
     File.expand_path('build/onigmo/include', __dir__),
     File.expand_path('build/natalie_parser/include', __dir__),
+    File.expand_path('build/yarp/include', __dir__),
   ]
 end

bin/natalie

@@ -295,7 +295,7 @@ class Runner
   end
 
   def parser
-    @parser ||= Natalie::Parser.new(code, source_path)
+    @parser ||= Natalie::Parser.new(code, source_path, old_parser: !!options[:write_obj_path])
   end
 
   def ast

lib/natalie/parser.rb

@@ -1,45 +1,112 @@
+$LOAD_PATH << File.expand_path('../../build/yarp/lib', __dir__)
+$LOAD_PATH << File.expand_path('../../build/yarp/ext', __dir__)
+require 'yarp'
+
+class SexpVisitor < ::YARP::BasicVisitor
+  def initialize(path)
+    @path = path
+  end
+
+  def visit_call_node(node)
+    arguments = node.child_nodes[1].child_nodes
+    s(:call,
+      visit(node.child_nodes.first),
+      node.name.to_sym,
+      *arguments.map { |n| visit(n) },
+      location: node.location)
+  end
+
+  def visit_integer_node(node)
+    s(:lit, node.value, location: node.location)
+  end
+
+  def visit_statements_node(node)
+    s(:block, *node.child_nodes.map { |n| visit(n) }, location: node.location)
+  end
+
+  alias visit_program_node visit_statements_node
+
+  def visit_alias_node(node)
+    s(:alias, visit(node.new_name), visit(node.old_name), location: node.location)
+  end
+
+  private
+
+  def s(*items, location: nil)
+    Sexp.new(*items, location:, file: @path)
+  end
+end
+
+class Sexp < Array
+  def initialize(*parts, location: nil, file: nil)
+    super(parts.size)
+    parts.each_with_index do |part, index|
+      self[index] = part
+      self.file = file
+      self.line = location.start_line
+      self.column = location.start_column
+    end
+  end
+
+  attr_accessor :file, :line, :column
+
+  def inspect
+    "s(#{map(&:inspect).join(', ')})"
+  end
+
+  def pretty_print(q)
+    q.group(1, 's(', ')') do
+      q.seplist(self) { |v| q.pp(v) }
+    end
+  end
+
+  def sexp_type
+    first
+  end
+
+  def new(*parts)
+    Sexp.new(*self).tap do |sexp|
+      sexp.file = file
+      sexp.line = line
+      sexp.column = column
+    end
+  end
+end
+
 module Natalie
   class Parser
     class IncompleteExpression < StandardError
     end
 
-    def initialize(code_str, path)
-      so_ext = RUBY_PLATFORM =~ /darwin/ ? 'bundle' : 'so'
-
-      if RUBY_ENGINE != 'natalie'
-        begin
-          $LOAD_PATH << File.expand_path('../../build/natalie_parser/lib', __dir__)
-          $LOAD_PATH << File.expand_path('../../build/natalie_parser/ext', __dir__)
-          require 'natalie_parser'
-        rescue LoadError => e
-          if e.message =~ /incompatible library version/
-            # NOTE: It's actually NatalieParser that was built against a different Ruby version,
-            # but saying that might be more confusing.
-            puts 'Error: Natalie was built with a different version of Ruby.'
-            puts "Please switch your current Ruby version or rebuild Natalie with version #{RUBY_VERSION}."
-          else
-            puts 'Error: You must build natalie_parser.so by running: rake parser_c_ext'
-          end
-          exit 1
-        end
-      end
+    def initialize(code_str, path, old_parser: false)
+      raise 'TODO' if RUBY_ENGINE == 'natalie'
 
       @code_str = code_str
       @path = path
+      @old_parser = old_parser
     end
 
     def ast
-      result = ::NatalieParser.parse(@code_str, @path)
-      if result.is_a?(Sexp) && result.sexp_type == :block
-        result
-      else
-        result.new(:block, result)
-      end
-    rescue SyntaxError => e
-      if e.message =~ /unexpected end-of-input/
-        raise IncompleteExpression, e.message
+      if @old_parser
+        $LOAD_PATH << File.expand_path('../../build/natalie_parser/lib', __dir__)
+        $LOAD_PATH << File.expand_path('../../build/natalie_parser/ext', __dir__)
+        require 'natalie_parser'
+        begin
+          result = NatalieParser.parse(@code_str, @path)
+          if result.is_a?(Sexp) && result.sexp_type == :block
+            result
+          else
+            result.new(:block, result)
+          end
+        rescue SyntaxError => e
+          if e.message =~ /unexpected end-of-input/
+            raise IncompleteExpression, e.message
+          else
+            raise
+          end
+        end
       else
-        raise
+        YARP.parse(@code_str, @path).value.accept(SexpVisitor.new(@path))
       end
     end
   end