Skip to content

Commit

Permalink
+ ruby-next/lexer.rl: upgrade lexer
Browse files Browse the repository at this point in the history
  • Loading branch information
palkan committed Dec 4, 2023
1 parent 70779a0 commit de4ad28
Show file tree
Hide file tree
Showing 9 changed files with 38 additions and 21 deletions.
6 changes: 2 additions & 4 deletions .dockerdev/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
ARG RUBY_VERSION
FROM ruby:${RUBY_VERSION}-slim

ARG BUNDLER_VERSION

# Install system dependencies
RUN apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get -yq dist-upgrade && \
DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \
Expand All @@ -27,8 +25,8 @@ ENV BUNDLE_APP_CONFIG=${BUNDLE_PATH} \
BUNDLE_BIN=${BUNDLE_PATH}/bin
ENV PATH /app/bin:${BUNDLE_BIN}:${PATH}

# Upgrade RubyGems and install required Bundler version
# Upgrade RubyGems and install latest Bundler version
RUN gem update --system && \
gem install bundler:${BUNDLER_VERSION}
gem install bundler

WORKDIR /app
3 changes: 1 addition & 2 deletions .dockerdev/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@ services:
context: ../
dockerfile: .dockerdev/Dockerfile
args:
RUBY_VERSION: '2.6'
BUNDLER_VERSION: '2.0.2'
RUBY_VERSION: '3.1'
environment:
- HISTFILE=/bundle/.bash_history
- bundler_data:/bundle
Expand Down
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -39,4 +39,5 @@ lib/parser/ruby33.rb
lib/parser/macruby.rb
lib/parser/rubymotion.rb
lib/parser/rubynext.rb
lib/parser/ruby-next/lexer.rb
lib/parser/ruby-next/lexer-F0.rb
lib/parser/ruby-next/lexer-F1.rb
11 changes: 10 additions & 1 deletion Rakefile
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ GENERATED_FILES = %w(lib/parser/lexer-F0.rb
lib/parser/macruby.rb
lib/parser/rubymotion.rb
lib/parser/rubynext.rb
lib/parser/ruby-next/lexer.rb)
lib/parser/ruby-next/lexer-F0.rb
lib/parser/ruby-next/lexer-F1.rb)

CLEAN.include(GENERATED_FILES)

Expand Down Expand Up @@ -163,6 +164,14 @@ file 'lib/parser/lexer-F0.rb' => 'lib/parser/lexer.rl' do |t|
sh "ragel -F0 -R #{t.source} -o #{t.name}"
end

file 'lib/parser/ruby-next/lexer-F1.rb' => 'lib/parser/ruby-next/lexer.rl' do |t|
sh "ragel -F1 -R #{t.source} -o #{t.name}"
end

file 'lib/parser/ruby-next/lexer-F0.rb' => 'lib/parser/ruby-next/lexer.rl' do |t|
sh "ragel -F0 -R #{t.source} -o #{t.name}"
end

file 'lib/parser/lexer-strings.rb' => 'lib/parser/lexer-strings.rl' do |t|
sh "ragel -F0 -R #{t.source} -o #{t.name}"
end
Expand Down
10 changes: 5 additions & 5 deletions lib/parser/ruby-next/lexer-F0.rb
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ class Parser::Lexer
class Next


# line 86 "lib/parser/ruby-next/lexer-F0.rb"
# line 83 "lib/parser/ruby-next/lexer-F0.rb"
class << self
attr_accessor :_lex_actions
private :_lex_actions, :_lex_actions=
Expand Down Expand Up @@ -8598,7 +8598,7 @@ def advance
@command_start = false


# line 8601 "lib/parser/ruby-next/lexer-F0.rb"
# line 8596 "lib/parser/ruby-next/lexer-F0.rb"
begin # ragel flat
testEof = false
_slen, _trans, _keys, _inds, _acts, _nacts = nil
Expand Down Expand Up @@ -8633,7 +8633,7 @@ def advance
begin
@ts = p
end
# line 8636 "lib/parser/ruby-next/lexer-F0.rb"
# line 8631 "lib/parser/ruby-next/lexer-F0.rb"
end # from state action switch
end
if _trigger_goto
Expand Down Expand Up @@ -12652,7 +12652,7 @@ def advance

end
end
# line 12655 "lib/parser/ruby-next/lexer-F0.rb"
# line 12650 "lib/parser/ruby-next/lexer-F0.rb"
end # action switch
end
end
Expand All @@ -12672,7 +12672,7 @@ def advance
# line 1 "NONE"
begin
@ts = nil; end
# line 12675 "lib/parser/ruby-next/lexer-F0.rb"
# line 12670 "lib/parser/ruby-next/lexer-F0.rb"
end # to state action switch
end
if _trigger_goto
Expand Down
10 changes: 5 additions & 5 deletions lib/parser/ruby-next/lexer-F1.rb
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ class Parser::Lexer
class Next


# line 86 "lib/parser/ruby-next/lexer-F1.rb"
# line 83 "lib/parser/ruby-next/lexer-F1.rb"
class << self
attr_accessor :_lex_trans_keys
private :_lex_trans_keys, :_lex_trans_keys=
Expand Down Expand Up @@ -8463,7 +8463,7 @@ def advance
@command_start = false


# line 8466 "lib/parser/ruby-next/lexer-F1.rb"
# line 8461 "lib/parser/ruby-next/lexer-F1.rb"
begin
testEof = false
_slen, _trans, _keys, _inds, _acts, _nacts = nil
Expand Down Expand Up @@ -8491,7 +8491,7 @@ def advance
begin
@ts = p
end
# line 8494 "lib/parser/ruby-next/lexer-F1.rb"
# line 8489 "lib/parser/ruby-next/lexer-F1.rb"
end
_keys = @cs << 1
_inds = _lex_index_offsets[ @cs]
Expand Down Expand Up @@ -14675,7 +14675,7 @@ def advance
# line 1799 "lib/parser/ruby-next/lexer.rl"
begin
@act = 115; end
# line 14678 "lib/parser/ruby-next/lexer-F1.rb"
# line 14673 "lib/parser/ruby-next/lexer-F1.rb"
end
end
end
Expand All @@ -14685,7 +14685,7 @@ def advance
# line 1 "NONE"
begin
@ts = nil; end
# line 14688 "lib/parser/ruby-next/lexer-F1.rb"
# line 14683 "lib/parser/ruby-next/lexer-F1.rb"
end

if @cs == 0
Expand Down
2 changes: 1 addition & 1 deletion lib/parser/ruby-next/lexer.rl
Original file line number Diff line number Diff line change
Expand Up @@ -490,7 +490,7 @@ class Next
'=>' => :tASSOC, '::' => :tCOLON2, '===' => :tEQQ,
'<=>' => :tCMP, '[]' => :tAREF, '[]=' => :tASET,
'{' => :tLCURLY, '}' => :tRCURLY, '`' => :tBACK_REF2,
'!@' => :tBANG, '&.' => :tANDDOT,
'!@' => :tBANG, '&.' => :tANDDOT, '.:' => :tMETHREF,
}

PUNCTUATION_BEGIN = {
Expand Down
7 changes: 6 additions & 1 deletion lib/parser/ruby-next/parser_ext.rb
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
# frozen_string_literal: true

require_relative "lexer"

if RUBY_ENGINE == 'truffleruby'
require_relative "lexer-F0"
else
require_relative "lexer-F1"
end
require_relative "builder"
require_relative "ast/processor"

Expand Down
7 changes: 6 additions & 1 deletion test/ruby-next/test_lexer.rb
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,12 @@
require 'helper'
require 'complex'

require 'parser/ruby-next/lexer'

if RUBY_ENGINE == 'truffleruby'
require 'parser/ruby-next/lexer-F0'
else
require 'parser/ruby-next/lexer-F1'
end

class TestLexerNext < Minitest::Test
def setup_lexer(version)
Expand Down

0 comments on commit de4ad28

Please sign in to comment.