Handle multibyte characters in indexing (#2619)

* Add test for indexing multibyte characters

* Modify to consider encoding in indexing

* Fix type error

* Format code

lib/ruby_indexer/lib/ruby_indexer/configuration.rb

@@ -19,9 +19,13 @@ class Configuration
     sig { params(workspace_path: String).void }
     attr_writer :workspace_path
 
+    sig { returns(Encoding) }
+    attr_accessor :encoding
+
     sig { void }
     def initialize
       @workspace_path = T.let(Dir.pwd, String)
+      @encoding = T.let(Encoding::UTF_8, Encoding)
       @excluded_gems = T.let(initial_excluded_gems, T::Array[String])
       @included_gems = T.let([], T::Array[String])
       @excluded_patterns = T.let([File.join("**", "*_test.rb"), File.join("tmp", "**", "*")], T::Array[String])

lib/ruby_indexer/lib/ruby_indexer/declaration_listener.rb

@@ -109,6 +109,7 @@ def on_class_node_enter(node)
         node.location,
         constant_path.location,
         comments,
+        @index.configuration.encoding,
         parent_class,
       )
 
@@ -132,7 +133,14 @@ def on_module_node_enter(node)
 
       comments = collect_comments(node)
 
-      entry = Entry::Module.new(actual_nesting(name), @file_path, node.location, constant_path.location, comments)
+      entry = Entry::Module.new(
+        actual_nesting(name),
+        @file_path,
+        node.location,
+        constant_path.location,
+        comments,
+        @index.configuration.encoding,
+      )
 
       @owner_stack << entry
       @index.add(entry)
@@ -161,14 +169,20 @@ def on_singleton_class_node_enter(node)
 
         if existing_entries
           entry = T.must(existing_entries.first)
-          entry.update_singleton_information(node.location, expression.location, collect_comments(node))
+          entry.update_singleton_information(
+            node.location,
+            expression.location,
+            collect_comments(node),
+            @index.configuration.encoding,
+          )
         else
           entry = Entry::SingletonClass.new(
             real_nesting,
             @file_path,
             node.location,
             expression.location,
             collect_comments(node),
+            @index.configuration.encoding,
             nil,
           )
           @index.add(entry, skip_prefix_tree: true)
@@ -329,6 +343,7 @@ def on_def_node_enter(node)
           node.location,
           node.name_loc,
           comments,
+          @index.configuration.encoding,
           [Entry::Signature.new(list_params(node.parameters))],
           current_visibility,
           @owner_stack.last,
@@ -345,6 +360,7 @@ def on_def_node_enter(node)
             node.location,
             node.name_loc,
             comments,
+            @index.configuration.encoding,
             [Entry::Signature.new(list_params(node.parameters))],
             current_visibility,
             singleton,
@@ -403,6 +419,7 @@ def on_alias_method_node_enter(node)
           @file_path,
           node.new_name.location,
           comments,
+          @index.configuration.encoding,
         ),
       )
     end
@@ -433,7 +450,14 @@ def handle_instance_variable(node, loc)
         owner = @index.existing_or_new_singleton_class(owner.name)
       end
 
-      @index.add(Entry::InstanceVariable.new(name, @file_path, loc, collect_comments(node), owner))
+      @index.add(Entry::InstanceVariable.new(
+        name,
+        @file_path,
+        loc,
+        collect_comments(node),
+        @index.configuration.encoding,
+        owner,
+      ))
     end
 
     sig { params(node: Prism::CallNode).void }
@@ -496,6 +520,7 @@ def handle_alias_method(node)
           @file_path,
           new_name.location,
           comments,
+          @index.configuration.encoding,
         ),
       )
     end
@@ -525,19 +550,43 @@ def add_constant(node, name, value = nil)
       @index.add(
         case value
         when Prism::ConstantReadNode, Prism::ConstantPathNode
-          Entry::UnresolvedConstantAlias.new(value.slice, @stack.dup, name, @file_path, node.location, comments)
+          Entry::UnresolvedConstantAlias.new(
+            value.slice,
+            @stack.dup,
+            name,
+            @file_path,
+            node.location,
+            comments,
+            @index.configuration.encoding,
+          )
         when Prism::ConstantWriteNode, Prism::ConstantAndWriteNode, Prism::ConstantOrWriteNode,
         Prism::ConstantOperatorWriteNode
 
           # If the right hand side is another constant assignment, we need to visit it because that constant has to be
           # indexed too
-          Entry::UnresolvedConstantAlias.new(value.name.to_s, @stack.dup, name, @file_path, node.location, comments)
+          Entry::UnresolvedConstantAlias.new(
+            value.name.to_s,
+            @stack.dup,
+            name,
+            @file_path,
+            node.location,
+            comments,
+            @index.configuration.encoding,
+          )
         when Prism::ConstantPathWriteNode, Prism::ConstantPathOrWriteNode, Prism::ConstantPathOperatorWriteNode,
         Prism::ConstantPathAndWriteNode
 
-          Entry::UnresolvedConstantAlias.new(value.target.slice, @stack.dup, name, @file_path, node.location, comments)
+          Entry::UnresolvedConstantAlias.new(
+            value.target.slice,
+            @stack.dup,
+            name,
+            @file_path,
+            node.location,
+            comments,
+            @index.configuration.encoding,
+          )
         else
-          Entry::Constant.new(name, @file_path, node.location, comments)
+          Entry::Constant.new(name, @file_path, node.location, comments, @index.configuration.encoding)
         end,
       )
     end
@@ -600,7 +649,15 @@ def handle_attribute(node, reader:, writer:)
         next unless name && loc
 
         if reader
-          @index.add(Entry::Accessor.new(name, @file_path, loc, comments, current_visibility, @owner_stack.last))
+          @index.add(Entry::Accessor.new(
+            name,
+            @file_path,
+            loc,
+            comments,
+            @index.configuration.encoding,
+            current_visibility,
+            @owner_stack.last,
+          ))
         end
 
         next unless writer
@@ -610,6 +667,7 @@ def handle_attribute(node, reader:, writer:)
           @file_path,
           loc,
           comments,
+          @index.configuration.encoding,
           current_visibility,
           @owner_stack.last,
         ))