Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Handle multibyte characters in indexing #2619

Merged
merged 4 commits into from
Sep 30, 2024
Merged

Handle multibyte characters in indexing #2619

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c54b0fb
Add test for indexing multibyte characters
NotFounds Sep 24, 2024
63df3c1
Modify to consider encoding in indexing
NotFounds Sep 24, 2024
57f4f9f
Fix type error
NotFounds Sep 26, 2024
d15ede6
Format code
NotFounds Sep 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions lib/ruby_indexer/lib/ruby_indexer/configuration.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,13 @@ class Configuration
sig { params(workspace_path: String).void }
attr_writer :workspace_path

sig { returns(Encoding) }
attr_accessor :encoding

sig { void }
def initialize
@workspace_path = T.let(Dir.pwd, String)
@encoding = T.let(Encoding::UTF_8, Encoding)
@excluded_gems = T.let(initial_excluded_gems, T::Array[String])
@included_gems = T.let([], T::Array[String])
@excluded_patterns = T.let([File.join("**", "*_test.rb"), File.join("tmp", "**", "*")], T::Array[String])
Expand Down
74 changes: 66 additions & 8 deletions lib/ruby_indexer/lib/ruby_indexer/declaration_listener.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ def on_class_node_enter(node)
node.location,
constant_path.location,
comments,
@index.configuration.encoding,
vinistock marked this conversation as resolved.
Show resolved Hide resolved
parent_class,
)

Expand All @@ -132,7 +133,14 @@ def on_module_node_enter(node)

comments = collect_comments(node)

entry = Entry::Module.new(actual_nesting(name), @file_path, node.location, constant_path.location, comments)
entry = Entry::Module.new(
actual_nesting(name),
@file_path,
node.location,
constant_path.location,
comments,
@index.configuration.encoding,
)

@owner_stack << entry
@index.add(entry)
Expand Down Expand Up @@ -161,14 +169,20 @@ def on_singleton_class_node_enter(node)

if existing_entries
entry = T.must(existing_entries.first)
entry.update_singleton_information(node.location, expression.location, collect_comments(node))
entry.update_singleton_information(
node.location,
expression.location,
collect_comments(node),
@index.configuration.encoding,
)
else
entry = Entry::SingletonClass.new(
real_nesting,
@file_path,
node.location,
expression.location,
collect_comments(node),
@index.configuration.encoding,
nil,
)
@index.add(entry, skip_prefix_tree: true)
Expand Down Expand Up @@ -329,6 +343,7 @@ def on_def_node_enter(node)
node.location,
node.name_loc,
comments,
@index.configuration.encoding,
[Entry::Signature.new(list_params(node.parameters))],
current_visibility,
@owner_stack.last,
Expand All @@ -345,6 +360,7 @@ def on_def_node_enter(node)
node.location,
node.name_loc,
comments,
@index.configuration.encoding,
[Entry::Signature.new(list_params(node.parameters))],
current_visibility,
singleton,
Expand Down Expand Up @@ -403,6 +419,7 @@ def on_alias_method_node_enter(node)
@file_path,
node.new_name.location,
comments,
@index.configuration.encoding,
),
)
end
Expand Down Expand Up @@ -433,7 +450,14 @@ def handle_instance_variable(node, loc)
owner = @index.existing_or_new_singleton_class(owner.name)
end

@index.add(Entry::InstanceVariable.new(name, @file_path, loc, collect_comments(node), owner))
@index.add(Entry::InstanceVariable.new(
name,
@file_path,
loc,
collect_comments(node),
@index.configuration.encoding,
owner,
))
end

sig { params(node: Prism::CallNode).void }
Expand Down Expand Up @@ -496,6 +520,7 @@ def handle_alias_method(node)
@file_path,
new_name.location,
comments,
@index.configuration.encoding,
),
)
end
Expand Down Expand Up @@ -525,19 +550,43 @@ def add_constant(node, name, value = nil)
@index.add(
case value
when Prism::ConstantReadNode, Prism::ConstantPathNode
Entry::UnresolvedConstantAlias.new(value.slice, @stack.dup, name, @file_path, node.location, comments)
Entry::UnresolvedConstantAlias.new(
value.slice,
@stack.dup,
name,
@file_path,
node.location,
comments,
@index.configuration.encoding,
)
when Prism::ConstantWriteNode, Prism::ConstantAndWriteNode, Prism::ConstantOrWriteNode,
Prism::ConstantOperatorWriteNode

# If the right hand side is another constant assignment, we need to visit it because that constant has to be
# indexed too
Entry::UnresolvedConstantAlias.new(value.name.to_s, @stack.dup, name, @file_path, node.location, comments)
Entry::UnresolvedConstantAlias.new(
value.name.to_s,
@stack.dup,
name,
@file_path,
node.location,
comments,
@index.configuration.encoding,
)
when Prism::ConstantPathWriteNode, Prism::ConstantPathOrWriteNode, Prism::ConstantPathOperatorWriteNode,
Prism::ConstantPathAndWriteNode

Entry::UnresolvedConstantAlias.new(value.target.slice, @stack.dup, name, @file_path, node.location, comments)
Entry::UnresolvedConstantAlias.new(
value.target.slice,
@stack.dup,
name,
@file_path,
node.location,
comments,
@index.configuration.encoding,
)
else
Entry::Constant.new(name, @file_path, node.location, comments)
Entry::Constant.new(name, @file_path, node.location, comments, @index.configuration.encoding)
end,
)
end
Expand Down Expand Up @@ -600,7 +649,15 @@ def handle_attribute(node, reader:, writer:)
next unless name && loc

if reader
@index.add(Entry::Accessor.new(name, @file_path, loc, comments, current_visibility, @owner_stack.last))
@index.add(Entry::Accessor.new(
name,
@file_path,
loc,
comments,
@index.configuration.encoding,
current_visibility,
@owner_stack.last,
))
end

next unless writer
Expand All @@ -610,6 +667,7 @@ def handle_attribute(node, reader:, writer:)
@file_path,
loc,
comments,
@index.configuration.encoding,
current_visibility,
@owner_stack.last,
))
Expand Down
80 changes: 48 additions & 32 deletions lib/ruby_indexer/lib/ruby_indexer/entry.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,10 @@ class Visibility < T::Enum
file_path: String,
location: T.any(Prism::Location, RubyIndexer::Location),
comments: T.nilable(String),
encoding: Encoding,
).void
end
def initialize(name, file_path, location, comments)
def initialize(name, file_path, location, comments, encoding)
@name = name
@file_path = file_path
@comments = comments
Expand All @@ -46,8 +47,8 @@ def initialize(name, file_path, location, comments)
Location.new(
location.start_line,
location.end_line,
location.start_column,
location.end_column,
location.start_code_units_column(encoding),
location.end_code_units_column(encoding),
)
else
location
Expand Down Expand Up @@ -150,22 +151,23 @@ class Namespace < Entry
location: T.any(Prism::Location, RubyIndexer::Location),
name_location: T.any(Prism::Location, Location),
comments: T.nilable(String),
encoding: Encoding,
).void
end
def initialize(nesting, file_path, location, name_location, comments)
def initialize(nesting, file_path, location, name_location, comments, encoding) # rubocop:disable Metrics/ParameterLists
@name = T.let(nesting.join("::"), String)
# The original nesting where this namespace was discovered
@nesting = nesting

super(@name, file_path, location, comments)
super(@name, file_path, location, comments, encoding)

@name_location = T.let(
if name_location.is_a?(Prism::Location)
Location.new(
name_location.start_line,
name_location.end_line,
name_location.start_column,
name_location.end_column,
name_location.start_code_units_column(encoding),
name_location.end_code_units_column(encoding),
)
else
name_location
Expand Down Expand Up @@ -211,11 +213,12 @@ class Class < Namespace
location: T.any(Prism::Location, RubyIndexer::Location),
name_location: T.any(Prism::Location, Location),
comments: T.nilable(String),
encoding: Encoding,
parent_class: T.nilable(String),
).void
end
def initialize(nesting, file_path, location, name_location, comments, parent_class) # rubocop:disable Metrics/ParameterLists
super(nesting, file_path, location, name_location, comments)
def initialize(nesting, file_path, location, name_location, comments, encoding, parent_class) # rubocop:disable Metrics/ParameterLists
super(nesting, file_path, location, name_location, comments, encoding)
@parent_class = parent_class
end

Expand All @@ -228,20 +231,27 @@ def ancestor_hash
class SingletonClass < Class
extend T::Sig

sig { params(location: Prism::Location, name_location: Prism::Location, comments: T.nilable(String)).void }
def update_singleton_information(location, name_location, comments)
sig do
params(
location: Prism::Location,
name_location: Prism::Location,
comments: T.nilable(String),
encoding: Encoding,
).void
end
def update_singleton_information(location, name_location, comments, encoding)
# Create a new RubyIndexer::Location object from the Prism location
@location = Location.new(
location.start_line,
location.end_line,
location.start_column,
location.end_column,
location.start_code_units_column(encoding),
location.end_code_units_column(encoding),
)
@name_location = Location.new(
name_location.start_line,
name_location.end_line,
name_location.start_column,
name_location.end_column,
name_location.start_code_units_column(encoding),
name_location.end_code_units_column(encoding),
)
(@comments ||= +"") << comments if comments
end
Expand Down Expand Up @@ -361,12 +371,13 @@ class Member < Entry
file_path: String,
location: T.any(Prism::Location, RubyIndexer::Location),
comments: T.nilable(String),
encoding: Encoding,
visibility: Visibility,
owner: T.nilable(Entry::Namespace),
).void
end
def initialize(name, file_path, location, comments, visibility, owner) # rubocop:disable Metrics/ParameterLists
super(name, file_path, location, comments)
def initialize(name, file_path, location, comments, encoding, visibility, owner) # rubocop:disable Metrics/ParameterLists
super(name, file_path, location, comments, encoding)
@visibility = visibility
@owner = owner
end
Expand Down Expand Up @@ -429,21 +440,22 @@ class Method < Member
location: T.any(Prism::Location, RubyIndexer::Location),
name_location: T.any(Prism::Location, Location),
comments: T.nilable(String),
encoding: Encoding,
signatures: T::Array[Signature],
visibility: Visibility,
owner: T.nilable(Entry::Namespace),
).void
end
def initialize(name, file_path, location, name_location, comments, signatures, visibility, owner) # rubocop:disable Metrics/ParameterLists
super(name, file_path, location, comments, visibility, owner)
def initialize(name, file_path, location, name_location, comments, encoding, signatures, visibility, owner) # rubocop:disable Metrics/ParameterLists
super(name, file_path, location, comments, encoding, visibility, owner)
@signatures = signatures
@name_location = T.let(
if name_location.is_a?(Prism::Location)
Location.new(
name_location.start_line,
name_location.end_line,
name_location.start_column,
name_location.end_column,
name_location.start_code_units_column(encoding),
name_location.end_code_units_column(encoding),
)
else
name_location
Expand Down Expand Up @@ -480,10 +492,11 @@ class UnresolvedConstantAlias < Entry
file_path: String,
location: T.any(Prism::Location, RubyIndexer::Location),
comments: T.nilable(String),
encoding: Encoding,
).void
end
def initialize(target, nesting, name, file_path, location, comments) # rubocop:disable Metrics/ParameterLists
super(name, file_path, location, comments)
def initialize(target, nesting, name, file_path, location, comments, encoding) # rubocop:disable Metrics/ParameterLists
super(name, file_path, location, comments, encoding)

@target = target
@nesting = nesting
Expand All @@ -497,9 +510,9 @@ class ConstantAlias < Entry
sig { returns(String) }
attr_reader :target

sig { params(target: String, unresolved_alias: UnresolvedConstantAlias).void }
def initialize(target, unresolved_alias)
super(unresolved_alias.name, unresolved_alias.file_path, unresolved_alias.location, unresolved_alias.comments)
sig { params(target: String, unresolved_alias: UnresolvedConstantAlias, encoding: Encoding).void }
def initialize(target, unresolved_alias, encoding)
super(unresolved_alias.name, unresolved_alias.file_path, unresolved_alias.location, unresolved_alias.comments, encoding)

@visibility = unresolved_alias.visibility
@target = target
Expand All @@ -517,11 +530,12 @@ class InstanceVariable < Entry
file_path: String,
location: T.any(Prism::Location, RubyIndexer::Location),
comments: T.nilable(String),
encoding: Encoding,
owner: T.nilable(Entry::Namespace),
).void
end
def initialize(name, file_path, location, comments, owner)
super(name, file_path, location, comments)
def initialize(name, file_path, location, comments, encoding, owner)
super(name, file_path, location, comments, encoding)
@owner = owner
end
end
Expand All @@ -546,10 +560,11 @@ class UnresolvedMethodAlias < Entry
file_path: String,
location: T.any(Prism::Location, RubyIndexer::Location),
comments: T.nilable(String),
encoding: Encoding,
).void
end
def initialize(new_name, old_name, owner, file_path, location, comments) # rubocop:disable Metrics/ParameterLists
super(new_name, file_path, location, comments)
def initialize(new_name, old_name, owner, file_path, location, comments, encoding) # rubocop:disable Metrics/ParameterLists
super(new_name, file_path, location, comments, encoding)

@new_name = new_name
@old_name = old_name
Expand All @@ -567,8 +582,8 @@ class MethodAlias < Entry
sig { returns(T.nilable(Entry::Namespace)) }
attr_reader :owner

sig { params(target: T.any(Member, MethodAlias), unresolved_alias: UnresolvedMethodAlias).void }
def initialize(target, unresolved_alias)
sig { params(target: T.any(Member, MethodAlias), unresolved_alias: UnresolvedMethodAlias, encoding: Encoding).void }
def initialize(target, unresolved_alias, encoding)
full_comments = +"Alias for #{target.name}\n"
full_comments << "#{unresolved_alias.comments}\n"
full_comments << target.comments
Expand All @@ -578,6 +593,7 @@ def initialize(target, unresolved_alias)
unresolved_alias.file_path,
unresolved_alias.location,
full_comments,
encoding
)

@target = target
Expand Down
Loading
Loading